diff --git a/README.md b/README.md index 78c1daf..24c996d 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,12 @@ From a fresh clone: docker compose up --build ``` -The Compose entrypoint generates and persists a random creator-authority encryption +The Compose entrypoint generates and persists a random runtime master key in the private `lock-home` volume. Set -`PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY` before startup only when you need to supply -your own 32-byte base64url key. +`PUBKY_LOCK_RUNTIME_MASTER_KEY` before startup only when you need to supply +your own 32-byte unpadded-base64url key. A supplied override is atomically +persisted to that volume, so a later startup without the environment variable +continues using the same key rather than silently reverting to an older key. Verified browser-facing defaults for the basic `docker-compose.yml` stack are: diff --git a/docker-compose.yml b/docker-compose.yml index 12413c8..c5faf4a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,7 +49,7 @@ services: network_mode: service:pubky-testnet environment: PUBKY_LOCK_DATABASE_URL: postgres://locks:locks@postgres:5432/locks_test - PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY: ${PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY:-} + PUBKY_LOCK_RUNTIME_MASTER_KEY: ${PUBKY_LOCK_RUNTIME_MASTER_KEY:-} volumes: - lock-home:/var/lib/pubky-lock command: ["locks-server-compose-entrypoint.sh"] diff --git a/docker/locks-server-compose-entrypoint.sh b/docker/locks-server-compose-entrypoint.sh index 7467e98..0c98acc 100644 --- a/docker/locks-server-compose-entrypoint.sh +++ b/docker/locks-server-compose-entrypoint.sh @@ -5,26 +5,75 @@ service_home="${LOCKS_SERVICE_HOME:-/var/lib/pubky-lock/.pubky-lock}" generated_config="$service_home/config.toml" compose_config="${LOCKS_COMPOSE_CONFIG:-/var/lib/pubky-lock/config.compose.toml}" secret_path="$service_home/secret.sess" -creator_authority_key_path="$service_home/creator-authority-encryption-key" +runtime_master_key_path="$service_home/runtime-master-key" +retired_creator_authority_key_path="$service_home/creator-authority-encryption-key" public_config="${LOCKS_PUBLIC_CONFIG:-/run/locks-public/config.toml}" mkdir -p "$service_home" -if [ -z "${PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY:-}" ]; then - if [ ! -f "$creator_authority_key_path" ]; then - echo "[locks-compose] generating creator-authority encryption key" +if [ -f "$retired_creator_authority_key_path" ]; then + echo "[locks-compose] retired creator-authority key detected: $retired_creator_authority_key_path" >&2 + echo "[locks-compose] stop the stack, discard and reacquire creator authority rows or recreate the local database, remove the retired key file, then restart" >&2 + exit 1 +fi + +if [ -n "${PUBKY_LOCK_RUNTIME_MASTER_KEY:-}" ]; then + if ! printf '%s' "$PUBKY_LOCK_RUNTIME_MASTER_KEY" | grep -Eq '^[A-Za-z0-9_-]{43}$'; then + echo "[locks-compose] PUBKY_LOCK_RUNTIME_MASTER_KEY must be an unpadded base64url-encoded 32-byte key" >&2 + exit 1 + fi + umask 077 + decoded_key_path="$runtime_master_key_path.decoded.$$" + cleanup_decoded_key() { + rm -f "$decoded_key_path" + } + trap cleanup_decoded_key EXIT HUP INT TERM + if ! printf '%s=' "$PUBKY_LOCK_RUNTIME_MASTER_KEY" \ + | tr '_-' '/+' \ + | base64 -d > "$decoded_key_path" 2>/dev/null \ + || [ "$(wc -c < "$decoded_key_path" | tr -d ' ')" -ne 32 ]; then + echo "[locks-compose] PUBKY_LOCK_RUNTIME_MASTER_KEY must be an unpadded base64url-encoded 32-byte key" >&2 + exit 1 + fi + canonical_runtime_master_key="$( + base64 < "$decoded_key_path" \ + | tr '+/' '-_' \ + | tr -d '=\n' + )" + if [ "$canonical_runtime_master_key" != "$PUBKY_LOCK_RUNTIME_MASTER_KEY" ]; then + echo "[locks-compose] PUBKY_LOCK_RUNTIME_MASTER_KEY must be an unpadded base64url-encoded 32-byte key" >&2 + exit 1 + fi + cleanup_decoded_key + trap - EXIT HUP INT TERM + if [ -f "$runtime_master_key_path" ]; then + persisted_runtime_master_key="$(cat "$runtime_master_key_path")" + if [ "$persisted_runtime_master_key" != "$PUBKY_LOCK_RUNTIME_MASTER_KEY" ]; then + echo "[locks-compose] PUBKY_LOCK_RUNTIME_MASTER_KEY does not match the persisted runtime master key" >&2 + echo "[locks-compose] rotate only through an explicit data migration or reset that handles encrypted state" >&2 + exit 1 + fi + else + temporary_key_path="$runtime_master_key_path.tmp.$$" + printf '%s' "$PUBKY_LOCK_RUNTIME_MASTER_KEY" > "$temporary_key_path" + chmod 600 "$temporary_key_path" + mv "$temporary_key_path" "$runtime_master_key_path" + fi +else + if [ ! -f "$runtime_master_key_path" ]; then + echo "[locks-compose] generating runtime master key" umask 077 - temporary_key_path="$creator_authority_key_path.tmp.$$" + temporary_key_path="$runtime_master_key_path.tmp.$$" head -c 32 /dev/urandom \ | base64 \ | tr '+/' '-_' \ | tr -d '=\n' > "$temporary_key_path" chmod 600 "$temporary_key_path" - mv "$temporary_key_path" "$creator_authority_key_path" + mv "$temporary_key_path" "$runtime_master_key_path" fi - PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY="$(cat "$creator_authority_key_path")" - export PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY + PUBKY_LOCK_RUNTIME_MASTER_KEY="$(cat "$runtime_master_key_path")" + export PUBKY_LOCK_RUNTIME_MASTER_KEY fi if [ ! -f "$generated_config" ] || [ ! -f "$secret_path" ]; then @@ -83,7 +132,14 @@ frontend_session_code_ttl_seconds = 120 allowed_return_origins = ["http://127.0.0.1:8080", "http://localhost:8080"] [secrets] -creator_authority_key_env = "PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY" +runtime_master_key_env = "PUBKY_LOCK_RUNTIME_MASTER_KEY" + +[deletion] +retry_max_attempts = 10 +retry_initial_backoff_seconds = 1 +retry_max_backoff_seconds = 300 +final_credential_issuance_window_seconds = 900 +final_read_window_seconds = 900 [logging] level = "info,pubky::actors::session=warn" diff --git a/docs/LOCAL_OPERATOR_DEMO.md b/docs/LOCAL_OPERATOR_DEMO.md index f2236e6..4767b79 100644 --- a/docs/LOCAL_OPERATOR_DEMO.md +++ b/docs/LOCAL_OPERATOR_DEMO.md @@ -108,7 +108,7 @@ If `/creator/lock-service-config` or `/creator/priv-resources/content/` re ## Prerequisites - A Postgres database reachable through `PUBKY_LOCK_DATABASE_URL`. -- A 32-byte base64url creator-authority encryption key in `PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY`. +- A 32-byte base64url runtime master key in `PUBKY_LOCK_RUNTIME_MASTER_KEY`. - `curl`, `jq`, and `python3` available in your shell. - A generated/default Lock Server config and secret under `~/.pubky-lock/`. @@ -118,10 +118,10 @@ The database URL below is a local development example. Real credentials must com export PUBKY_LOCK_DATABASE_URL='postgres://locks:locks@localhost:55433/locks_test' ``` -Generate a local creator-authority encryption key for this shell before starting the server: +Generate a local runtime master key for this shell before starting the server: ```bash -export PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY="$(python3 - <<'PY' +export PUBKY_LOCK_RUNTIME_MASTER_KEY="$(python3 - <<'PY' import base64, os print(base64.urlsafe_b64encode(os.urandom(32)).decode().rstrip('=')) PY diff --git a/docs/RUNTIME.md b/docs/RUNTIME.md index c1a55ee..6ae082d 100644 --- a/docs/RUNTIME.md +++ b/docs/RUNTIME.md @@ -182,14 +182,33 @@ Operator-facing readiness uses semantic storage labels: Postgres is private runtime storage for verification tasks, task claiming, access credentials, frontend sessions, and creator-granted homeserver session material. It is not storage for Pubky-owned content locks, guarded resources, Lock Service Pointers, or verified proof bundles. -Creator-granted session material is encrypted before storage. The server-side encryption key comes from an env var named by config: +Sensitive runtime material is encrypted before storage. A root key comes from an env var named by config: ```toml [secrets] -creator_authority_key_env = "PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY" +runtime_master_key_env = "PUBKY_LOCK_RUNTIME_MASTER_KEY" ``` -The named env var must contain a 32-byte key encoded as base64url without padding: +The named env var must contain a 32-byte key encoded as base64url without padding. Locks derives separate fixed-domain keys for creator-authority material and replayable final deletion credentials; the root key is never used directly as an AEAD key. Rotating it requires an explicit data migration. In the local Compose flow, an explicitly supplied `PUBKY_LOCK_RUNTIME_MASTER_KEY` is validated and persisted only when `.pubky-lock/runtime-master-key` does not yet exist. Once persisted, an override must match those exact bytes or startup fails closed; later starts without the override reuse the persisted key. Changing the key therefore requires an explicit encrypted-data migration or an intentional reset that discards the dependent ciphertext and its old key together. + +Migration `0016_content_lock_access_drains` fails closed when an older database contains a resumable `queued`, `running`, or `failed` deletion job, because Task 7 rows do not contain enough information to reconstruct cutoff credential classification safely. Failed jobs are resumable by graceful-deletion replay, so they cannot be treated as terminal for this upgrade. Before upgrading, stop new writes and check: + +```sql +SELECT job_id, state, phase +FROM content_lock_deletion_jobs +WHERE state IN ('queued', 'running', 'failed'); +``` + +If rows are returned, resume/retry them on the pre-0016 release until every deletion reaches `completed` and then retry the upgrade, or explicitly reset the pre-production environment: stop the stack, back up anything needed, recreate the Locks PostgreSQL database/volume, reconcile or republish any public tombstones/content locks, and reacquire creator authority. Do not bypass the guard by deleting only the job rows; that can strand external Pubky state and accepted obligations. + +The runtime-master-key cutover intentionally cannot decrypt creator-authority rows written with the retired Compose `creator-authority-encryption-key`. An existing Compose volume containing that file fails startup instead of silently stranding encrypted authority. To upgrade a local stack: + +1. Stop the stack and back up any data that must be retained. +2. Either discard the existing creator-authority rows and reacquire authority after restart, or recreate the local PostgreSQL database/volume. +3. Remove `.pubky-lock/creator-authority-encryption-key` from the Locks service volume. +4. Restart. Compose creates `.pubky-lock/runtime-master-key`; keep that file stable with the database. + +Removing only the retired key file while retaining its encrypted creator-authority rows is unsupported. ```bash python3 - <<'PY' @@ -198,6 +217,19 @@ print(base64.urlsafe_b64encode(os.urandom(32)).decode().rstrip('=')) PY ``` +Deletion retry and final-access bounds are a closed configuration section: + +```toml +[deletion] +retry_max_attempts = 10 +retry_initial_backoff_seconds = 1 +retry_max_backoff_seconds = 300 +final_credential_issuance_window_seconds = 900 +final_read_window_seconds = 900 +``` + +All values must be positive, initial backoff cannot exceed maximum backoff, and both final-access windows must be at most 3600 seconds. + ## Development integration shape ```toml diff --git a/docs/plans/2026-08-10-graceful-content-lock-deletion.md b/docs/plans/2026-08-10-graceful-content-lock-deletion.md index 0b57a08..c0855e1 100644 --- a/docs/plans/2026-08-10-graceful-content-lock-deletion.md +++ b/docs/plans/2026-08-10-graceful-content-lock-deletion.md @@ -14,7 +14,7 @@ ## Status and provenance -- Plan status: **accepted product design; Tasks 1, 2, 4, 5, and 6 committed; the omitted Task 3 prerequisite is implemented pending commit; Tasks 7–10 remain**. +- Plan status: **accepted product design; Tasks 1–7 committed; Tasks 8–11 remain**. - Repository inspected: `/home/u/Projects/Synonym/Pubky/locks-public`. - Planning base when written: clean `master` at `ba49a77`. - There has been no production deployment. New persistence may require a clean pre-production database; no historical backfill is required. @@ -66,6 +66,11 @@ 32. `force=true` against an active graceful job persists `force_requested`, revokes the current claim token/lease, requeues the same frozen job, and returns `202`; a fresh worker claim escalates asynchronously under exclusive action ownership, skips drains, deletes tombstone then content, and finishes forced. 33. Graceful job insertion/resume and permanent force-receipt establishment acquire the same canonical per-lock PostgreSQL fence. The durable result is either an active graceful job or a permanent force receipt, never both. Failed graceful replay requeues the same job and frozen manifest. Force against a terminal job atomically replaces that operational row with the permanent receipt before synchronous external deletion. 34. Any Content Lock fetched from Pubky for deletion must hash to the requested Lock ID and name the authenticated creator before its manifest is frozen or used for resource deletion. +35. Runtime encryption uses one environment-only 32-byte unpadded-base64url master key selected by `secrets.runtime_master_key_env`. Creator-authority and final-credential encryption keys are derived from it with distinct fixed domain labels. The retired `creator_authority_key_env` key is rejected as unknown configuration; no compatibility alias is retained. +36. The closed `[deletion]` configuration contract is `retry_max_attempts = 10`, `retry_initial_backoff_seconds = 1`, `retry_max_backoff_seconds = 300`, `final_credential_issuance_window_seconds = 900`, and `final_read_window_seconds = 900` by default. All values are positive; initial backoff cannot exceed maximum backoff; both credential windows are bounded to at most 3600 seconds. Retry jitter remains an implementation policy rather than a configurable field. +37. Deletion admission immutably records whether each paid snapshot Bundle had any active credential at cutoff and enrolls every such ordinary credential with its original expiry. Enrolled ordinary credentials remain reusable against the frozen manifest until that expiry; they do not acquire one-shot resource-read rows. When the claimed job first enters `issue_final_credentials`, it persists `final_issuance_started_at`, `final_credential_issuance_deadline = final_issuance_started_at + final_credential_issuance_window`, and `final_read_deadline = final_credential_issuance_deadline + final_read_window` once; replay and later config changes never extend them. A paid snapshot resolved completed without an active ordinary credential at cutoff becomes durably final-credential eligible and receives exactly one encrypted replayable final credential expiring at `final_read_deadline`. Every final credential receives one claimable row per frozen manifest path. Final-read claims precede Pubky fetch, are released on pre-response failure, expire for crash recovery, and are consumed only after the complete HTTP response is constructed; consumption is permanent. Phase advancement waits until every enrolled ordinary credential is expired and every final resource is consumed or its credential/read window is expired. +38. Ordinary credential insertion and deletion admission acquire the same canonical per-lock fence. Deletion-first rejects the insert; insertion-first is attached and classified at cutoff. Database lock order is canonical per-lock fence, deletion job row, snapshot/credential row, then resource-read row. No transaction spans Pubky I/O. Final read claims use fixed 30-second leases clamped to credential expiry; stale claim tokens cannot consume or release a reclaimed row. +39. This pre-production migration intentionally has no creator-authority ciphertext compatibility path. Moving the same bytes to `runtime_master_key_env` changes the derived creator-authority key; existing local encrypted authority rows must be discarded and reacquired or the local database recreated. ### Source-derived constraints @@ -458,7 +463,7 @@ cargo test --workspace --no-run **RED:** Cover exact encrypted replay, wrong-key/corrupt/version rejection, no secret Debug/log output, issuance/read deadlines, no deadline extension, existing/final access through the frozen manifest while the public path is a tombstone, denial outside the persisted drain, one concurrent success per path, claim release before response construction, consumption after construction, and automatic revocation when complete/expired. -**GREEN:** Use versioned AEAD and domain-separated key derivation; retain lookup hashes. Resolve draining reads from the frozen manifest rather than parsing the tombstone. Do not store plaintext bearer. +**GREEN:** Snapshot and enroll active credentials atomically at deletion admission, and initialize final-window timestamps once when entering final issuance under the deletion lease. Fence ordinary insertion against deletion admission. Use versioned AEAD and domain-separated key derivation; retain lookup hashes. Enroll cutoff-active credentials at their original expiry and create one final credential only for an eligible completed snapshot without one. Resolve draining reads from the frozen manifest rather than parsing the tombstone. Claim each credential/path before fetch, release on pre-response failure, consume only after the server constructs the complete response, and allow only expired claims to be reclaimed. Do not store plaintext bearer. **Suggested commit:** `feat(access): drain final deletion credentials` @@ -560,9 +565,7 @@ Cross-service acceptance must additionally prove: ## Remaining implementation-contract gates -These do not reopen accepted product semantics, but code must not start for the affected slice until both plans are patched identically: - -1. Exact configuration keys for retry attempts/backoff and final credential windows, within the accepted defaults/maxima. +None. The exact Locks-only deletion configuration and runtime-master-key contracts are fixed above. Paykit Server has no corresponding credential or deletion-worker configuration. ## Out of scope diff --git a/examples/js-sdk/README.md b/examples/js-sdk/README.md index cc838f0..b599b11 100644 --- a/examples/js-sdk/README.md +++ b/examples/js-sdk/README.md @@ -96,10 +96,10 @@ For a containerized local stack from the repository root: docker compose up --build ``` -On first startup, the Lock Server entrypoint generates a random creator-authority -encryption key and persists it in the private `lock-home` volume. Later starts reuse -that key. To supply your own 32-byte base64url key instead, export -`PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY` before running Compose. +On first startup, the Lock Server entrypoint generates a random runtime master key +and persists it in the private `lock-home` volume. Later starts reuse that key. To +supply your own 32-byte base64url key instead, export +`PUBKY_LOCK_RUNTIME_MASTER_KEY` before running Compose. The compose stack starts: @@ -112,8 +112,8 @@ The compose stack starts: The Pubky testnet image is built from the public `pubky/pubky-homeserver` repository at the revision pinned in `docker-compose.yml`; no sibling checkout is required. -Compose keeps the Lock Server identity, config, and generated creator-authority -encryption key in the `lock-home` Docker volume and Postgres data in `postgres-data`. +Compose keeps the Lock Server identity, config, and generated runtime master key in +the `lock-home` Docker volume and Postgres data in `postgres-data`. To reset everything: ```bash diff --git a/locks-e2e/tests/postgres_runtime.rs b/locks-e2e/tests/postgres_runtime.rs index af93f1e..c03a8be 100644 --- a/locks-e2e/tests/postgres_runtime.rs +++ b/locks-e2e/tests/postgres_runtime.rs @@ -18,9 +18,9 @@ use locks_core::lock_policy::{ }; use locks_core::verification::{Proof, SUBMITTED_PROOF_BUNDLE_VERSION, SubmittedProofBundle}; use locks_server::api::routes::router; -use locks_server::app_state::{AppState, ReaderPubkyResolver}; +use locks_server::app_state::{AppState, ReaderPubkyResolver, RuntimeSecretCiphers}; use locks_server::config::{ - ContentLocksConfig, CreatorAuthorityAcquisitionConfig, DatabaseConfig, + ContentLocksConfig, CreatorAuthorityAcquisitionConfig, DatabaseConfig, DeletionConfig, FilesystemLockServerIdentityProvider, LockServerCredentialsConfig, LockServerIdentityProvider, LockServerRuntimeConfig, LoggingConfig, PaykitConfig, PkdnsConfig, PubkyConfig, RateLimitsConfig, RuntimeConfig, RuntimeEnvironment, SecretsConfig, WorkerConfig, @@ -34,6 +34,7 @@ use locks_service::application::models::{ use locks_service::application::ports::{ ContentLockDeletionRepository, VerificationTaskRepository, }; +use locks_service::infrastructure::final_credentials::FinalCredentialCipher; use locks_service::infrastructure::memory::{ content_locks::InMemoryContentLockRepository, entitlements::InMemoryEntitlementRepository, guarded_resources::InMemoryGuardedResourceRepository, @@ -276,7 +277,10 @@ async fn deletion_first_proof_submission_returns_409_without_calling_paykit() { let state = AppState::new_with_postgres_runtime_and_creator_repositories( config, database.pool().clone(), - CreatorAuthoritySecretCipher::new([7; 32]), + RuntimeSecretCiphers::new( + CreatorAuthoritySecretCipher::new([7; 32]), + FinalCredentialCipher::new([8; 32]), + ), Arc::new(InMemoryContentLockRepository::new()), Arc::new(InMemoryGuardedResourceRepository::new()), Arc::new(InMemoryLockServicePointerRepository::new()), @@ -368,7 +372,10 @@ async fn snapshotted_unready_paykit_replay_ignores_tombstoned_lock_and_reader_re let initial_state = AppState::new_with_postgres_runtime_and_creator_repositories( config.clone(), database.pool().clone(), - CreatorAuthoritySecretCipher::new([7; 32]), + RuntimeSecretCiphers::new( + CreatorAuthoritySecretCipher::new([7; 32]), + FinalCredentialCipher::new([8; 32]), + ), Arc::new(InMemoryContentLockRepository::new()), Arc::new(InMemoryGuardedResourceRepository::new()), Arc::new(InMemoryLockServicePointerRepository::new()), @@ -403,7 +410,10 @@ async fn snapshotted_unready_paykit_replay_ignores_tombstoned_lock_and_reader_re let tombstoned_state = AppState::new_with_postgres_runtime_and_creator_repositories( config, database.pool().clone(), - CreatorAuthoritySecretCipher::new([7; 32]), + RuntimeSecretCiphers::new( + CreatorAuthoritySecretCipher::new([7; 32]), + FinalCredentialCipher::new([8; 32]), + ), Arc::new(InMemoryContentLockRepository::new()), Arc::new(InMemoryGuardedResourceRepository::new()), Arc::new(InMemoryLockServicePointerRepository::new()), @@ -522,7 +532,10 @@ fn app_state(pool: PgPool) -> AppState { AppState::new_with_postgres_runtime_and_creator_repositories( test_config(), pool, - CreatorAuthoritySecretCipher::new([7; 32]), + RuntimeSecretCiphers::new( + CreatorAuthoritySecretCipher::new([7; 32]), + FinalCredentialCipher::new([8; 32]), + ), std::sync::Arc::new(InMemoryContentLockRepository::new()), std::sync::Arc::new(InMemoryGuardedResourceRepository::new()), std::sync::Arc::new(InMemoryLockServicePointerRepository::new()), @@ -560,6 +573,7 @@ fn test_config() -> LockServerRuntimeConfig { max_connections: 10, run_migrations_on_startup: true, }, + deletion: DeletionConfig::default(), worker: WorkerConfig { enabled: true, poll_interval_ms: 250, diff --git a/locks-server/config/example.dev.postgres.toml b/locks-server/config/example.dev.postgres.toml index 63789a7..2bdfdea 100644 --- a/locks-server/config/example.dev.postgres.toml +++ b/locks-server/config/example.dev.postgres.toml @@ -33,7 +33,14 @@ frontend_session_code_ttl_seconds = 120 allowed_return_origins = ["http://localhost:3000"] [secrets] -creator_authority_key_env = "PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY" +runtime_master_key_env = "PUBKY_LOCK_RUNTIME_MASTER_KEY" + +[deletion] +retry_max_attempts = 10 +retry_initial_backoff_seconds = 1 +retry_max_backoff_seconds = 300 +final_credential_issuance_window_seconds = 900 +final_read_window_seconds = 900 [logging] level = "info" diff --git a/locks-server/src/api/access.rs b/locks-server/src/api/access.rs index 8c7615a..b091397 100644 --- a/locks-server/src/api/access.rs +++ b/locks-server/src/api/access.rs @@ -8,7 +8,7 @@ use locks_service::application::use_cases::issue_access_credential::{ IssueAccessCredentialRequest, IssueAccessCredentialUseCase, }; use locks_service::application::use_cases::proxy_read_guarded_resource::{ - ProxyReadGuardedResourceRequest, ProxyReadGuardedResourceUseCase, + ProxiedGuardedResource, ProxyReadGuardedResourceRequest, ProxyReadGuardedResourceUseCase, }; use crate::api::dtos::{IssueAccessCredentialHttpRequest, IssueAccessCredentialHttpResponse}; @@ -58,6 +58,18 @@ pub(super) async fn proxy_read_guarded_resource( let proxied = use_case .execute(ProxyReadGuardedResourceRequest { credential, path }) .await?; + let response = match build_proxy_read_response(&proxied) { + Ok(response) => response, + Err(error) => { + use_case.release_prepared_deletion_read(&proxied).await?; + return Err(error); + } + }; + use_case.consume_prepared_deletion_read(&proxied).await?; + Ok(response) +} + +fn build_proxy_read_response(proxied: &ProxiedGuardedResource) -> Result { let content_type = HeaderValue::from_str(&proxied.content_type).map_err(|_| { ApiError::new( ApiErrorCode::InternalError, @@ -88,7 +100,7 @@ pub(super) async fn proxy_read_guarded_resource( .map_err(|_| ApiError::new(ApiErrorCode::InternalError, "invalid etag"))?, ), ], - Body::from(proxied.bytes), + Body::from(proxied.bytes.clone()), ) .into_response()) } diff --git a/locks-server/src/api/errors.rs b/locks-server/src/api/errors.rs index f707397..3fb1326 100644 --- a/locks-server/src/api/errors.rs +++ b/locks-server/src/api/errors.rs @@ -250,6 +250,7 @@ impl From for ApiError { Self::new(ApiErrorCode::RateLimited, "rate limit exceeded") } ApplicationError::Storage { .. } + | ApplicationError::FinalCredentialSecret { .. } | ApplicationError::InvalidContentLockDeletionState { .. } | ApplicationError::Verifier { .. } | ApplicationError::CredentialGeneration { .. } diff --git a/locks-server/src/api/routes/tests.rs b/locks-server/src/api/routes/tests.rs index 7d13ca2..152e190 100644 --- a/locks-server/src/api/routes/tests.rs +++ b/locks-server/src/api/routes/tests.rs @@ -22,12 +22,15 @@ use locks_core::lock_policy::{ use locks_core::verification::{Proof, SUBMITTED_PROOF_BUNDLE_VERSION, SubmittedProofBundle}; use locks_service::application::errors::ApplicationError; use locks_service::application::models::{ - ContentLockDeletionFailureCode, ContentLockDeletionState, CreatorAuthorityAuthKind, - CreatorAuthorityRecord, CreatorAuthoritySecret, CreatorConnectAuthorizationUrl, - CreatorConnectFlowId, FrontendSessionRecord, FrontendSessionToken, GuardedResourceRecord, + AccessCredentialLookupKey, AccessCredentialRecord, ContentLockDeletionFailureCode, + ContentLockDeletionState, CreatorAuthorityAuthKind, CreatorAuthorityRecord, + CreatorAuthoritySecret, CreatorConnectAuthorizationUrl, CreatorConnectFlowId, + DeletionReadAuthorization, FrontendSessionRecord, FrontendSessionToken, GuardedResourceRecord, LegacyCreatorConnectFlowApproval, PendingCreatorConnectFlowRecord, }; -use locks_service::application::ports::{Clock, LegacyCreatorConnectFlowClient}; +use locks_service::application::ports::{ + AccessCredentialStore, Clock, GuardedResourceRepository, LegacyCreatorConnectFlowClient, +}; use pubky_common::crypto::Keypair; use serde_json::{Value, json}; use sqlx::postgres::PgPoolOptions; @@ -81,6 +84,156 @@ impl ReaderPubkyResolver for AlwaysResolvesReader { } } +struct ResponseBoundaryAccessCredentialStore { + content_type: String, + claim_token: Uuid, + consume_succeeds: bool, + releases: AtomicUsize, + consumes: AtomicUsize, +} + +impl ResponseBoundaryAccessCredentialStore { + fn new(content_type: &str) -> Self { + Self { + content_type: content_type.to_owned(), + claim_token: Uuid::new_v4(), + consume_succeeds: true, + releases: AtomicUsize::new(0), + consumes: AtomicUsize::new(0), + } + } + + fn losing_consume(content_type: &str) -> Self { + Self { + consume_succeeds: false, + ..Self::new(content_type) + } + } +} + +#[async_trait] +impl AccessCredentialStore for ResponseBoundaryAccessCredentialStore { + async fn insert_access_credential( + &self, + _lock_id: &LockId, + _lookup_key: AccessCredentialLookupKey, + _record: AccessCredentialRecord, + ) -> Result<(), ApplicationError> { + unreachable!("response-boundary fake does not issue credentials") + } + + async fn get_access_credential( + &self, + _lookup_key: &AccessCredentialLookupKey, + ) -> Result, ApplicationError> { + Ok(None) + } + + async fn delete_access_credential( + &self, + _lookup_key: &AccessCredentialLookupKey, + ) -> Result<(), ApplicationError> { + Ok(()) + } + + async fn prepare_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + path: &str, + _now: time::OffsetDateTime, + _claim_expires_at: time::OffsetDateTime, + ) -> Result, ApplicationError> { + Ok(Some(DeletionReadAuthorization { + claim_token: Some(self.claim_token), + creator: creator(), + resource: GuardedResource { + path: path.to_owned(), + hash: GuardedResourceHash::from_bytes([7; 32]), + content_type: self.content_type.clone(), + size: 13, + }, + })) + } + + async fn deletion_credential_enrolled( + &self, + _lookup_key: &AccessCredentialLookupKey, + ) -> Result { + Ok(true) + } + + async fn release_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + _path: &str, + claim_token: Uuid, + _now: time::OffsetDateTime, + ) -> Result { + assert_eq!(claim_token, self.claim_token); + self.releases.fetch_add(1, Ordering::SeqCst); + Ok(true) + } + + async fn consume_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + _path: &str, + claim_token: Uuid, + _now: time::OffsetDateTime, + ) -> Result { + assert_eq!(claim_token, self.claim_token); + self.consumes.fetch_add(1, Ordering::SeqCst); + Ok(self.consume_succeeds) + } +} + +struct ResponseBoundaryGuardedResourceRepository { + content_type: String, +} + +#[async_trait] +impl GuardedResourceRepository for ResponseBoundaryGuardedResourceRepository { + async fn upsert_guarded_resource( + &self, + _guarded_resource: GuardedResourceRecord, + ) -> Result { + unreachable!("response-boundary fake does not store resources") + } + + async fn get_guarded_resource( + &self, + creator: &CreatorPubky, + path: &str, + hash: &GuardedResourceHash, + ) -> Result, ApplicationError> { + Ok(Some(GuardedResourceRecord { + creator: creator.clone(), + path: path.to_owned(), + hash: *hash, + content_type: self.content_type.clone(), + size: 13, + bytes: b"guarded bytes".to_vec(), + })) + } + + async fn get_current_guarded_resource( + &self, + creator: &CreatorPubky, + path: &str, + ) -> Result, ApplicationError> { + self.get_guarded_resource(creator, path, &GuardedResourceHash::from_bytes([7; 32])) + .await + } + + async fn delete_guarded_resource( + &self, + _creator: &CreatorPubky, + _path: &str, + ) -> Result { + Ok(false) + } +} + #[derive(Debug)] struct RecordingPaykitSetupStatusProvider { result: Result, @@ -888,6 +1041,7 @@ async fn readyz_returns_not_ready_for_persisted_runtime_when_pool_ping_fails() { test_config(RuntimeEnvironment::Development, true), pool, CreatorAuthoritySecretCipher::new([7; 32]), + locks_service::infrastructure::final_credentials::FinalCredentialCipher::new([8; 32]), ); let response = router(state) @@ -2444,6 +2598,72 @@ async fn proxy_read_with_valid_bearer_credential_returns_raw_guarded_resource_by assert_eq!(response_bytes(response).await, b"guarded bytes".to_vec()); } +#[tokio::test] +async fn deletion_proxy_read_releases_claim_when_http_response_construction_fails() { + let store = Arc::new(ResponseBoundaryAccessCredentialStore::new( + "invalid\ncontent-type", + )); + let state = test_state() + .with_access_credentials(store.clone()) + .with_guarded_resources(Arc::new(ResponseBoundaryGuardedResourceRepository { + content_type: "invalid\ncontent-type".to_owned(), + })); + + let response = router(state) + .oneshot(auth_request( + "GET", + "/priv-resources/content/hello.txt", + "Bearer deletion-credential", + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(store.releases.load(Ordering::SeqCst), 1); + assert_eq!(store.consumes.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn deletion_proxy_read_consumes_claim_before_returning_http_200() { + let store = Arc::new(ResponseBoundaryAccessCredentialStore::new("text/plain")); + let state = test_state().with_access_credentials(store.clone()); + seed_response_boundary_resource(&state, "text/plain").await; + + let response = router(state) + .oneshot(auth_request( + "GET", + "/priv-resources/content/hello.txt", + "Bearer deletion-credential", + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(store.consumes.load(Ordering::SeqCst), 1); + assert_eq!(store.releases.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn deletion_proxy_read_does_not_return_constructed_response_when_consume_loses() { + let store = Arc::new(ResponseBoundaryAccessCredentialStore::losing_consume( + "text/plain", + )); + let state = test_state().with_access_credentials(store.clone()); + seed_response_boundary_resource(&state, "text/plain").await; + + let response = router(state) + .oneshot(auth_request( + "GET", + "/priv-resources/content/hello.txt", + "Bearer deletion-credential", + )) + .await + .unwrap(); + + assert_ne!(response.status(), StatusCode::OK); + assert_eq!(store.consumes.load(Ordering::SeqCst), 1); +} + #[tokio::test] async fn proxy_read_accepts_bearer_scheme_case_insensitively() { let state = test_state(); @@ -4069,6 +4289,7 @@ fn test_config( pkdns: crate::config::PkdnsConfig::default(), rate_limits: RateLimitsConfig::default(), content_locks: ContentLocksConfig::default(), + deletion: crate::config::DeletionConfig::default(), paykit: None, } } @@ -4281,6 +4502,21 @@ async fn seed_content_lock(state: &AppState, content_lock: ContentLock) { .unwrap(); } +async fn seed_response_boundary_resource(state: &AppState, content_type: &str) { + state + .guarded_resources() + .upsert_guarded_resource(GuardedResourceRecord { + creator: creator(), + path: "/priv/locks.app/content/hello.txt".to_owned(), + hash: GuardedResourceHash::from_bytes([7; 32]), + content_type: content_type.to_owned(), + size: 13, + bytes: b"guarded bytes".to_vec(), + }) + .await + .unwrap(); +} + async fn seed_guarded_resource(state: &AppState, content_lock: &ContentLock, bytes: Vec) { let guarded_resource = content_lock.primary_resource.as_ref().unwrap(); state diff --git a/locks-server/src/app_state/mod.rs b/locks-server/src/app_state/mod.rs index 3162996..c264665 100644 --- a/locks-server/src/app_state/mod.rs +++ b/locks-server/src/app_state/mod.rs @@ -26,6 +26,7 @@ use locks_service::{ }, }, infrastructure::{ + final_credentials::FinalCredentialCipher, memory::{ access_credentials::InMemoryAccessCredentialStore, content_lock_deletions::InMemoryContentLockDeletionRepository, @@ -200,6 +201,24 @@ pub struct AppState { paykit_setup_status_provider: Option>, } +/// Purpose-separated runtime ciphers derived from the configured master key. +pub struct RuntimeSecretCiphers { + creator_authority: CreatorAuthoritySecretCipher, + final_credential: FinalCredentialCipher, +} + +impl RuntimeSecretCiphers { + pub fn new( + creator_authority: CreatorAuthoritySecretCipher, + final_credential: FinalCredentialCipher, + ) -> Self { + Self { + creator_authority, + final_credential, + } + } +} + impl std::fmt::Debug for AppState { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter @@ -243,7 +262,12 @@ impl AppState { Arc::clone(&verification_task_deletion_fence), ), ); - let access_credentials = Arc::new(InMemoryAccessCredentialStore::new()); + let access_credentials = Arc::new( + InMemoryAccessCredentialStore::with_verification_task_repository_and_deletion_fence( + verification_tasks.clone(), + Arc::clone(&verification_task_deletion_fence), + ), + ); let creator_authority_store = InMemoryCreatorAuthorityStore::new(); let creator_authorities = Arc::new(creator_authority_store.clone()); let creator_authority_manager = Arc::new(LegacyCookieCreatorAuthorityManager::new( @@ -261,7 +285,8 @@ impl AppState { let private_runtime = PrivateRuntimeAdapters { content_lock_ownership: Arc::new(InMemoryContentLockOwnershipRepository::new()), content_lock_deletions: Arc::new( - InMemoryContentLockDeletionRepository::with_verification_task_fence( + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + Arc::clone(&access_credentials), verification_task_deletion_fence, ), ), @@ -304,7 +329,12 @@ impl AppState { Arc::clone(&verification_task_deletion_fence), ), ); - let access_credentials = Arc::new(InMemoryAccessCredentialStore::new()); + let access_credentials = Arc::new( + InMemoryAccessCredentialStore::with_verification_task_repository_and_deletion_fence( + verification_tasks.clone(), + Arc::clone(&verification_task_deletion_fence), + ), + ); let creator_authority_store = InMemoryCreatorAuthorityStore::new(); let creator_authorities = Arc::new(creator_authority_store.clone()); let creator_authority_manager = Arc::new(LegacyCookieCreatorAuthorityManager::new( @@ -321,7 +351,8 @@ impl AppState { let private_runtime = PrivateRuntimeAdapters { content_lock_ownership: Arc::new(InMemoryContentLockOwnershipRepository::new()), content_lock_deletions: Arc::new( - InMemoryContentLockDeletionRepository::with_verification_task_fence( + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + Arc::clone(&access_credentials), verification_task_deletion_fence, ), ), @@ -364,7 +395,12 @@ impl AppState { Arc::clone(&verification_task_deletion_fence), ), ); - let access_credentials = Arc::new(InMemoryAccessCredentialStore::new()); + let access_credentials = Arc::new( + InMemoryAccessCredentialStore::with_verification_task_repository_and_deletion_fence( + verification_tasks.clone(), + Arc::clone(&verification_task_deletion_fence), + ), + ); let creator_authority_store = InMemoryCreatorAuthorityStore::new(); let creator_authorities = Arc::new(creator_authority_store.clone()); let creator_authority_manager = Arc::new(LegacyCookieCreatorAuthorityManager::new( @@ -398,7 +434,8 @@ impl AppState { let private_runtime = PrivateRuntimeAdapters { content_lock_ownership: Arc::new(InMemoryContentLockOwnershipRepository::new()), content_lock_deletions: Arc::new( - InMemoryContentLockDeletionRepository::with_verification_task_fence( + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + Arc::clone(&access_credentials), verification_task_deletion_fence, ), ), @@ -426,11 +463,16 @@ impl AppState { config: LockServerRuntimeConfig, pool: PgPool, creator_authority_cipher: CreatorAuthoritySecretCipher, + final_credential_cipher: FinalCredentialCipher, ) -> Self { let verification_tasks = Arc::new(PostgresVerificationTaskRepository::new(pool.clone())); let verification_task_claimer = Arc::new(PostgresVerificationTaskClaimer::new(pool.clone())); - let access_credentials = Arc::new(PostgresAccessCredentialStore::new(pool.clone())); + let access_credentials = + Arc::new(PostgresAccessCredentialStore::with_final_credential_cipher( + pool.clone(), + final_credential_cipher, + )); let creator_authority_store = PostgresCreatorAuthorityStore::new_encrypted(pool.clone(), creator_authority_cipher); let creator_authorities = Arc::new(creator_authority_store.clone()); @@ -485,7 +527,7 @@ impl AppState { pub fn new_with_postgres_runtime_and_creator_repositories( config: LockServerRuntimeConfig, pool: PgPool, - creator_authority_cipher: CreatorAuthoritySecretCipher, + ciphers: RuntimeSecretCiphers, content_locks: Arc, guarded_resources: Arc, lock_service_pointers: Arc, @@ -494,9 +536,13 @@ impl AppState { let verification_tasks = Arc::new(PostgresVerificationTaskRepository::new(pool.clone())); let verification_task_claimer = Arc::new(PostgresVerificationTaskClaimer::new(pool.clone())); - let access_credentials = Arc::new(PostgresAccessCredentialStore::new(pool.clone())); + let access_credentials = + Arc::new(PostgresAccessCredentialStore::with_final_credential_cipher( + pool.clone(), + ciphers.final_credential, + )); let creator_authority_store = - PostgresCreatorAuthorityStore::new_encrypted(pool.clone(), creator_authority_cipher); + PostgresCreatorAuthorityStore::new_encrypted(pool.clone(), ciphers.creator_authority); let creator_authorities = Arc::new(creator_authority_store.clone()); let pubky_http_client = build_pubky_http_client(&config.pubky); let creator_authority_manager: Arc = @@ -759,6 +805,24 @@ impl AppState { self } + #[cfg(test)] + pub fn with_access_credentials( + mut self, + access_credentials: Arc, + ) -> Self { + self.access_credentials = access_credentials; + self + } + + #[cfg(test)] + pub fn with_guarded_resources( + mut self, + guarded_resources: Arc, + ) -> Self { + self.guarded_resources = guarded_resources; + self + } + #[cfg(any(test, feature = "test-support"))] pub fn with_legacy_creator_connect_flow_client( mut self, diff --git a/locks-server/src/app_state/test_support.rs b/locks-server/src/app_state/test_support.rs index 612d987..47c57fe 100644 --- a/locks-server/src/app_state/test_support.rs +++ b/locks-server/src/app_state/test_support.rs @@ -23,6 +23,7 @@ use locks_service::application::ports::{ CreatorConnectFlowIdGenerator, FrontendSessionCodeGenerator, FrontendSessionTokenGenerator, VerificationTaskIdGenerator, }; +use locks_service::infrastructure::final_credentials::FinalCredentialCipher; use locks_service::infrastructure::postgres::CreatorAuthoritySecretCipher; #[test] @@ -41,8 +42,12 @@ async fn postgres_state_uses_postgres_for_private_runtime_adapters() { .connect_lazy("postgres://locks:locks@localhost/locks_test") .unwrap(); - let state = - AppState::new_with_postgres_runtime(test_config(), pool, test_creator_authority_cipher()); + let state = AppState::new_with_postgres_runtime( + test_config(), + pool, + test_creator_authority_cipher(), + test_final_credential_cipher(), + ); assert_eq!( state.private_runtime_storage_kind(), @@ -56,8 +61,12 @@ async fn postgres_state_wires_legacy_connect_flow_runtime_state() { .connect_lazy("postgres://locks:locks@localhost/locks_test") .unwrap(); - let state = - AppState::new_with_postgres_runtime(test_config(), pool, test_creator_authority_cipher()); + let state = AppState::new_with_postgres_runtime( + test_config(), + pool, + test_creator_authority_cipher(), + test_final_credential_cipher(), + ); assert!(Arc::strong_count(state.creator_connect_flows()) >= 1); assert!(Arc::strong_count(state.frontend_session_codes()) >= 1); @@ -74,7 +83,12 @@ async fn postgres_state_uses_acquisition_gate_to_wire_legacy_connect_client() { let mut config = test_config(); config.creator_authority_acquisition.enabled = true; - let state = AppState::new_with_postgres_runtime(config, pool, test_creator_authority_cipher()); + let state = AppState::new_with_postgres_runtime( + config, + pool, + test_creator_authority_cipher(), + test_final_credential_cipher(), + ); let result = state .legacy_creator_connect_flow_client() @@ -125,8 +139,12 @@ async fn persisted_state_keeps_postgres_pool_for_readiness() { .connect_lazy("postgres://locks:locks@localhost/locks_test") .unwrap(); - let state = - AppState::new_with_postgres_runtime(test_config(), pool, test_creator_authority_cipher()); + let state = AppState::new_with_postgres_runtime( + test_config(), + pool, + test_creator_authority_cipher(), + test_final_credential_cipher(), + ); assert!(state.postgres_pool().is_some()); } @@ -137,8 +155,12 @@ async fn persisted_state_composes_pubky_homeserver_creator_repositories() { .connect_lazy("postgres://locks:locks@localhost/locks_test") .unwrap(); - let state = - AppState::new_with_postgres_runtime(test_config(), pool, test_creator_authority_cipher()); + let state = AppState::new_with_postgres_runtime( + test_config(), + pool, + test_creator_authority_cipher(), + test_final_credential_cipher(), + ); assert!(Arc::strong_count(state.content_locks()) >= 1); assert!(Arc::strong_count(state.guarded_resources()) >= 1); @@ -187,6 +209,7 @@ async fn postgres_state_has_rate_limiter_configured_from_runtime_config() { test_config_with_rate_limit(true, 1, 60), pool, test_creator_authority_cipher(), + test_final_credential_cipher(), ); let key = rate_limit_key(); let now = datetime!(2026-06-03 12:00:00 UTC); @@ -314,6 +337,10 @@ fn disabled_runtime_rate_limiter_in_state_always_allows() { } } +fn test_final_credential_cipher() -> FinalCredentialCipher { + FinalCredentialCipher::new([8; 32]) +} + fn test_creator_authority_cipher() -> CreatorAuthoritySecretCipher { CreatorAuthoritySecretCipher::new([7; 32]) } @@ -350,6 +377,7 @@ fn test_config() -> LockServerRuntimeConfig { pkdns: crate::config::PkdnsConfig::default(), rate_limits: RateLimitsConfig::default(), content_locks: ContentLocksConfig::default(), + deletion: crate::config::DeletionConfig::default(), paykit: None, } } diff --git a/locks-server/src/config/defaults.rs b/locks-server/src/config/defaults.rs index fd030ec..82d3250 100644 --- a/locks-server/src/config/defaults.rs +++ b/locks-server/src/config/defaults.rs @@ -2,4 +2,9 @@ pub(super) const DEFAULT_SERVICE_HOME: &str = ".pubky-lock"; pub(super) const DEFAULT_CONFIG_FILE: &str = "config.toml"; pub(super) const DEFAULT_SECRET_FILE: &str = "secret.sess"; pub(super) const PUBLIC_KEY_PLACEHOLDER: &str = ""; -pub(super) const DEFAULT_CREATOR_AUTHORITY_KEY_ENV: &str = "PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY"; +pub(super) const DEFAULT_RUNTIME_MASTER_KEY_ENV: &str = "PUBKY_LOCK_RUNTIME_MASTER_KEY"; +pub(super) const DEFAULT_DELETION_RETRY_MAX_ATTEMPTS: u32 = 10; +pub(super) const DEFAULT_DELETION_RETRY_INITIAL_BACKOFF_SECONDS: u64 = 1; +pub(super) const DEFAULT_DELETION_RETRY_MAX_BACKOFF_SECONDS: u64 = 300; +pub(super) const DEFAULT_FINAL_CREDENTIAL_ISSUANCE_WINDOW_SECONDS: u64 = 900; +pub(super) const DEFAULT_FINAL_READ_WINDOW_SECONDS: u64 = 900; diff --git a/locks-server/src/config/examples.rs b/locks-server/src/config/examples.rs index f27082c..abb1d08 100644 --- a/locks-server/src/config/examples.rs +++ b/locks-server/src/config/examples.rs @@ -49,7 +49,14 @@ frontend_session_code_ttl_seconds = 120 allowed_return_origins = ["http://localhost:3000"] [secrets] -creator_authority_key_env = "PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY" +runtime_master_key_env = "PUBKY_LOCK_RUNTIME_MASTER_KEY" + +[deletion] +retry_max_attempts = 10 +retry_initial_backoff_seconds = 1 +retry_max_backoff_seconds = 300 +final_credential_issuance_window_seconds = 900 +final_read_window_seconds = 900 [logging] level = "info" @@ -495,6 +502,139 @@ fn allows_wildcard_return_origin_outside_production() { ); } +#[test] +fn accepts_closed_deletion_defaults_and_runtime_master_key_contract() { + let temp_dir = tempdir().unwrap(); + let secret_path = temp_dir.path().join("secret.sess"); + let public_key = test_identity(&secret_path); + let config_path = temp_dir.path().join("config.toml"); + std::fs::write( + &config_path, + minimal_config(&secret_path, &public_key, "development"), + ) + .unwrap(); + + let config = load_existing_config_from_path(&config_path).unwrap(); + assert_eq!( + config.secrets.runtime_master_key_env, + "PUBKY_LOCK_RUNTIME_MASTER_KEY" + ); + assert_eq!(config.deletion.retry_max_attempts, 10); + assert_eq!(config.deletion.retry_initial_backoff_seconds, 1); + assert_eq!(config.deletion.retry_max_backoff_seconds, 300); + assert_eq!( + config.deletion.final_credential_issuance_window_seconds, + 900 + ); + assert_eq!(config.deletion.final_read_window_seconds, 900); +} + +#[test] +fn rejects_retired_creator_authority_key_env() { + let temp_dir = tempdir().unwrap(); + let secret_path = temp_dir.path().join("secret.sess"); + let public_key = test_identity(&secret_path); + let config_path = temp_dir.path().join("config.toml"); + let config = minimal_config(&secret_path, &public_key, "development").replace( + "runtime_master_key_env = \"PUBKY_LOCK_RUNTIME_MASTER_KEY\"", + "creator_authority_key_env = \"PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY\"", + ); + std::fs::write(&config_path, config).unwrap(); + + let error = load_existing_config_from_path(&config_path).unwrap_err(); + assert!(matches!(error, ConfigError::ParseConfig { .. })); + assert!(error.to_string().contains("creator_authority_key_env")); +} + +#[test] +fn rejects_zero_or_inverted_deletion_retry_contract() { + let temp_dir = tempdir().unwrap(); + let secret_path = temp_dir.path().join("secret.sess"); + let public_key = test_identity(&secret_path); + + for (name, from, to, expected) in [ + ( + "zero-attempts", + "retry_max_attempts = 10", + "retry_max_attempts = 0", + ConfigError::InvalidDeletionRetry, + ), + ( + "zero-initial", + "retry_initial_backoff_seconds = 1", + "retry_initial_backoff_seconds = 0", + ConfigError::InvalidDeletionRetry, + ), + ( + "inverted", + "retry_initial_backoff_seconds = 1", + "retry_initial_backoff_seconds = 301", + ConfigError::InvalidDeletionRetryBackoffOrder, + ), + ] { + let config_path = temp_dir.path().join(format!("{name}.toml")); + let config = minimal_config(&secret_path, &public_key, "development").replace(from, to); + std::fs::write(&config_path, config).unwrap(); + + let error = load_existing_config_from_path(&config_path).unwrap_err(); + assert_eq!( + std::mem::discriminant(&error), + std::mem::discriminant(&expected) + ); + } +} + +#[test] +fn accepts_maximum_deletion_windows_and_rejects_out_of_range_values() { + let temp_dir = tempdir().unwrap(); + let secret_path = temp_dir.path().join("secret.sess"); + let public_key = test_identity(&secret_path); + let base = minimal_config(&secret_path, &public_key, "development"); + + let max_path = temp_dir.path().join("max.toml"); + let max = base + .replace( + "final_credential_issuance_window_seconds = 900", + "final_credential_issuance_window_seconds = 3600", + ) + .replace( + "final_read_window_seconds = 900", + "final_read_window_seconds = 3600", + ); + std::fs::write(&max_path, max).unwrap(); + assert!(load_existing_config_from_path(&max_path).is_ok()); + + for (name, from, to) in [ + ( + "zero-issuance", + "final_credential_issuance_window_seconds = 900", + "final_credential_issuance_window_seconds = 0", + ), + ( + "long-issuance", + "final_credential_issuance_window_seconds = 900", + "final_credential_issuance_window_seconds = 3601", + ), + ( + "zero-read", + "final_read_window_seconds = 900", + "final_read_window_seconds = 0", + ), + ( + "long-read", + "final_read_window_seconds = 900", + "final_read_window_seconds = 3601", + ), + ] { + let config_path = temp_dir.path().join(format!("{name}.toml")); + std::fs::write(&config_path, base.replace(from, to)).unwrap(); + assert!(matches!( + load_existing_config_from_path(&config_path).unwrap_err(), + ConfigError::InvalidDeletionCredentialWindow + )); + } +} + fn test_identity(secret_path: &std::path::Path) -> LockServerPubky { let keypair = pubky_common::crypto::Keypair::from_secret(&[9; 32]); let public_key = LockServerPubky::from_str(&keypair.public_key().to_string()).unwrap(); @@ -547,7 +687,14 @@ frontend_session_code_ttl_seconds = 120 allowed_return_origins = [] [secrets] -creator_authority_key_env = "PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY" +runtime_master_key_env = "PUBKY_LOCK_RUNTIME_MASTER_KEY" + +[deletion] +retry_max_attempts = 10 +retry_initial_backoff_seconds = 1 +retry_max_backoff_seconds = 300 +final_credential_issuance_window_seconds = 900 +final_read_window_seconds = 900 [logging] level = "info" diff --git a/locks-server/src/config/loading.rs b/locks-server/src/config/loading.rs index 414c67f..8b1164a 100644 --- a/locks-server/src/config/loading.rs +++ b/locks-server/src/config/loading.rs @@ -4,9 +4,9 @@ use super::defaults::{DEFAULT_CONFIG_FILE, DEFAULT_SECRET_FILE, DEFAULT_SERVICE_ use super::raw::RawConfig; use super::schema::{ ConfigError, ConfigPathResolution, ContentLocksConfig, CreatorAuthorityAcquisitionConfig, - DatabaseConfig, LockServerCredentialsConfig, LockServerRuntimeConfig, LoggingConfig, - PaykitConfig, PkdnsConfig, PubkyConfig, RateLimitsConfig, RuntimeConfig, RuntimeEnvironment, - SecretsConfig, WorkerConfig, + DatabaseConfig, DeletionConfig, LockServerCredentialsConfig, LockServerRuntimeConfig, + LoggingConfig, PaykitConfig, PkdnsConfig, PubkyConfig, RateLimitsConfig, RuntimeConfig, + RuntimeEnvironment, SecretsConfig, WorkerConfig, }; use super::secrets::{LockServerIdentityProvider, parse_lock_server_keypair_seed}; @@ -148,6 +148,7 @@ fn initialize_default_config( pkdns: PkdnsConfig::default(), rate_limits: RateLimitsConfig::default(), content_locks: ContentLocksConfig::default(), + deletion: DeletionConfig::default(), paykit: Some(PaykitConfig { server_url: "http://127.0.0.1:3001".to_owned(), minimum_confirmations: 0, @@ -190,7 +191,14 @@ frontend_session_code_ttl_seconds = {} # One-time callback code lifetime. Keep s allowed_return_origins = [] # Origins allowed to receive auth callback codes, e.g. ["https://pubky.app"]. Empty rejects all /connect return_to values; ["*"] is dev-only and unsafe for staging/prod. [secrets] -creator_authority_key_env = "{}" # Environment variable containing a 32-byte base64url key for encrypting creator authority at rest. Rotating requires data migration. +runtime_master_key_env = "{}" # Environment variable containing the 32-byte base64url runtime master key. Domain-separated keys encrypt creator authority and final credentials. Rotating requires data migration. + +[deletion] +retry_max_attempts = {} # Maximum attempts per deletion phase before stable retry_exhausted failure. +retry_initial_backoff_seconds = {} # Initial durable retry delay; must be positive and no greater than the maximum. +retry_max_backoff_seconds = {} # Maximum durable retry delay in seconds. +final_credential_issuance_window_seconds = {} # Bounded final-credential issuance window; must be 1..=3600. +final_read_window_seconds = {} # Bounded one-read-per-resource window; must be 1..=3600. [logging] level = "{}" # Tracing level/filter, e.g. error, warn, info, debug, trace, or EnvFilter syntax. Higher verbosity may expose operational detail in logs. @@ -252,7 +260,12 @@ max_total_resource_bytes = {} # Maximum combined bytes across resources in one c config .creator_authority_acquisition .frontend_session_code_ttl_seconds, - config.secrets.creator_authority_key_env, + config.secrets.runtime_master_key_env, + config.deletion.retry_max_attempts, + config.deletion.retry_initial_backoff_seconds, + config.deletion.retry_max_backoff_seconds, + config.deletion.final_credential_issuance_window_seconds, + config.deletion.final_read_window_seconds, config.logging.level, config.pkdns.public_ip, config.pkdns.public_pubky_tls_port.unwrap_or(6287), diff --git a/locks-server/src/config/mod.rs b/locks-server/src/config/mod.rs index 138d1c5..3231e22 100644 --- a/locks-server/src/config/mod.rs +++ b/locks-server/src/config/mod.rs @@ -10,9 +10,9 @@ mod validation; pub use loading::{load_existing_config_from_path, load_or_initialize_config, resolve_config_path}; pub use schema::{ ConfigError, ConfigPathResolution, ContentLocksConfig, CreatorAuthorityAcquisitionConfig, - CreatorAuthorityAcquisitionMethod, DatabaseConfig, LegacyConnectAcquisitionConfig, - LockServerCredentialsConfig, LockServerRuntimeConfig, LoggingConfig, - PAYKIT_CONNECT_TIMEOUT_SECONDS, PAYKIT_REQUEST_TIMEOUT_SECONDS, PaykitConfig, + CreatorAuthorityAcquisitionMethod, DatabaseConfig, DeletionConfig, + LegacyConnectAcquisitionConfig, LockServerCredentialsConfig, LockServerRuntimeConfig, + LoggingConfig, PAYKIT_CONNECT_TIMEOUT_SECONDS, PAYKIT_REQUEST_TIMEOUT_SECONDS, PaykitConfig, PaykitConnectionStateLookupRateLimitConfig, PkdnsConfig, PubkyConfig, PubkyNetwork, PubkyResolution, RateLimitsConfig, RuntimeConfig, RuntimeEnvironment, SecretsConfig, VerificationSubmissionRateLimitConfig, WorkerConfig, diff --git a/locks-server/src/config/raw.rs b/locks-server/src/config/raw.rs index d3c0a80..355c3a6 100644 --- a/locks-server/src/config/raw.rs +++ b/locks-server/src/config/raw.rs @@ -7,14 +7,19 @@ use serde::Deserialize; use tracing_subscriber::EnvFilter; use url::Url; -use super::defaults::{DEFAULT_CREATOR_AUTHORITY_KEY_ENV, PUBLIC_KEY_PLACEHOLDER}; +use super::defaults::{ + DEFAULT_DELETION_RETRY_INITIAL_BACKOFF_SECONDS, DEFAULT_DELETION_RETRY_MAX_ATTEMPTS, + DEFAULT_DELETION_RETRY_MAX_BACKOFF_SECONDS, DEFAULT_FINAL_CREDENTIAL_ISSUANCE_WINDOW_SECONDS, + DEFAULT_FINAL_READ_WINDOW_SECONDS, DEFAULT_RUNTIME_MASTER_KEY_ENV, PUBLIC_KEY_PLACEHOLDER, +}; use super::schema::{ ConfigError, ContentLocksConfig, CreatorAuthorityAcquisitionConfig, - CreatorAuthorityAcquisitionMethod, DatabaseConfig, LegacyConnectAcquisitionConfig, - LockServerCredentialsConfig, LockServerRuntimeConfig, LoggingConfig, - PAYKIT_REQUEST_TIMEOUT_SECONDS, PaykitConfig, PaykitConnectionStateLookupRateLimitConfig, - PkdnsConfig, PubkyConfig, PubkyNetwork, PubkyResolution, RateLimitsConfig, RuntimeConfig, - RuntimeEnvironment, SecretsConfig, VerificationSubmissionRateLimitConfig, WorkerConfig, + CreatorAuthorityAcquisitionMethod, DatabaseConfig, DeletionConfig, + LegacyConnectAcquisitionConfig, LockServerCredentialsConfig, LockServerRuntimeConfig, + LoggingConfig, MAX_DELETION_CREDENTIAL_WINDOW_SECONDS, PAYKIT_REQUEST_TIMEOUT_SECONDS, + PaykitConfig, PaykitConnectionStateLookupRateLimitConfig, PkdnsConfig, PubkyConfig, + PubkyNetwork, PubkyResolution, RateLimitsConfig, RuntimeConfig, RuntimeEnvironment, + SecretsConfig, VerificationSubmissionRateLimitConfig, WorkerConfig, }; #[derive(Debug, Deserialize)] @@ -40,6 +45,8 @@ pub(super) struct RawConfig { #[serde(default)] content_locks: RawContentLocksConfig, #[serde(default)] + deletion: RawDeletionConfig, + #[serde(default)] paykit: Option, } @@ -367,20 +374,64 @@ fn validate_allowed_return_origin(value: String) -> Result #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct RawSecretsConfig { - #[serde(default = "default_creator_authority_key_env")] - creator_authority_key_env: String, + #[serde(default = "default_runtime_master_key_env")] + runtime_master_key_env: String, } impl Default for RawSecretsConfig { fn default() -> Self { Self { - creator_authority_key_env: default_creator_authority_key_env(), + runtime_master_key_env: default_runtime_master_key_env(), } } } -fn default_creator_authority_key_env() -> String { - DEFAULT_CREATOR_AUTHORITY_KEY_ENV.to_owned() +fn default_runtime_master_key_env() -> String { + DEFAULT_RUNTIME_MASTER_KEY_ENV.to_owned() +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RawDeletionConfig { + #[serde(default = "default_deletion_retry_max_attempts")] + retry_max_attempts: u32, + #[serde(default = "default_deletion_retry_initial_backoff_seconds")] + retry_initial_backoff_seconds: u64, + #[serde(default = "default_deletion_retry_max_backoff_seconds")] + retry_max_backoff_seconds: u64, + #[serde(default = "default_final_credential_issuance_window_seconds")] + final_credential_issuance_window_seconds: u64, + #[serde(default = "default_final_read_window_seconds")] + final_read_window_seconds: u64, +} + +impl Default for RawDeletionConfig { + fn default() -> Self { + Self { + retry_max_attempts: default_deletion_retry_max_attempts(), + retry_initial_backoff_seconds: default_deletion_retry_initial_backoff_seconds(), + retry_max_backoff_seconds: default_deletion_retry_max_backoff_seconds(), + final_credential_issuance_window_seconds: + default_final_credential_issuance_window_seconds(), + final_read_window_seconds: default_final_read_window_seconds(), + } + } +} + +fn default_deletion_retry_max_attempts() -> u32 { + DEFAULT_DELETION_RETRY_MAX_ATTEMPTS +} +fn default_deletion_retry_initial_backoff_seconds() -> u64 { + DEFAULT_DELETION_RETRY_INITIAL_BACKOFF_SECONDS +} +fn default_deletion_retry_max_backoff_seconds() -> u64 { + DEFAULT_DELETION_RETRY_MAX_BACKOFF_SECONDS +} +fn default_final_credential_issuance_window_seconds() -> u64 { + DEFAULT_FINAL_CREDENTIAL_ISSUANCE_WINDOW_SECONDS +} +fn default_final_read_window_seconds() -> u64 { + DEFAULT_FINAL_READ_WINDOW_SECONDS } #[derive(Debug, Deserialize)] @@ -628,11 +679,40 @@ impl RawCreatorAuthorityAcquisitionConfig { impl RawSecretsConfig { fn into_secrets_config(self) -> Result { - if self.creator_authority_key_env.trim().is_empty() { - return Err(ConfigError::InvalidCreatorAuthorityKeyEnv); + if self.runtime_master_key_env.trim().is_empty() { + return Err(ConfigError::InvalidRuntimeMasterKeyEnv); } Ok(SecretsConfig { - creator_authority_key_env: self.creator_authority_key_env, + runtime_master_key_env: self.runtime_master_key_env, + }) + } +} + +impl RawDeletionConfig { + fn into_deletion_config(self) -> Result { + if self.retry_max_attempts == 0 + || self.retry_initial_backoff_seconds == 0 + || self.retry_max_backoff_seconds == 0 + { + return Err(ConfigError::InvalidDeletionRetry); + } + if self.retry_initial_backoff_seconds > self.retry_max_backoff_seconds { + return Err(ConfigError::InvalidDeletionRetryBackoffOrder); + } + if self.final_credential_issuance_window_seconds == 0 + || self.final_credential_issuance_window_seconds + > MAX_DELETION_CREDENTIAL_WINDOW_SECONDS + || self.final_read_window_seconds == 0 + || self.final_read_window_seconds > MAX_DELETION_CREDENTIAL_WINDOW_SECONDS + { + return Err(ConfigError::InvalidDeletionCredentialWindow); + } + Ok(DeletionConfig { + retry_max_attempts: self.retry_max_attempts, + retry_initial_backoff_seconds: self.retry_initial_backoff_seconds, + retry_max_backoff_seconds: self.retry_max_backoff_seconds, + final_credential_issuance_window_seconds: self.final_credential_issuance_window_seconds, + final_read_window_seconds: self.final_read_window_seconds, }) } } @@ -725,6 +805,7 @@ impl RawConfig { let pkdns = self.pkdns.into_pkdns_config()?; let rate_limits = self.rate_limits.into_rate_limits_config()?; let content_locks = self.content_locks.into_content_locks_config()?; + let deletion = self.deletion.into_deletion_config()?; let paykit = self .paykit .map(RawPaykitConfig::into_paykit_config) @@ -768,6 +849,7 @@ impl RawConfig { pkdns, rate_limits, content_locks, + deletion, paykit, }) } diff --git a/locks-server/src/config/schema.rs b/locks-server/src/config/schema.rs index 5b1c331..01ad934 100644 --- a/locks-server/src/config/schema.rs +++ b/locks-server/src/config/schema.rs @@ -5,7 +5,11 @@ use locks_core::ids::LockServerPubky; use serde::Deserialize; use thiserror::Error; -use super::defaults::DEFAULT_CREATOR_AUTHORITY_KEY_ENV; +use super::defaults::{ + DEFAULT_DELETION_RETRY_INITIAL_BACKOFF_SECONDS, DEFAULT_DELETION_RETRY_MAX_ATTEMPTS, + DEFAULT_DELETION_RETRY_MAX_BACKOFF_SECONDS, DEFAULT_FINAL_CREDENTIAL_ISSUANCE_WINDOW_SECONDS, + DEFAULT_FINAL_READ_WINDOW_SECONDS, DEFAULT_RUNTIME_MASTER_KEY_ENV, +}; pub const PAYKIT_CONNECT_TIMEOUT_SECONDS: u64 = 5; pub const PAYKIT_REQUEST_TIMEOUT_SECONDS: u64 = 20; @@ -24,9 +28,34 @@ pub struct LockServerRuntimeConfig { pub pkdns: PkdnsConfig, pub rate_limits: RateLimitsConfig, pub content_locks: ContentLocksConfig, + pub deletion: DeletionConfig, pub paykit: Option, } +pub const MAX_DELETION_CREDENTIAL_WINDOW_SECONDS: u64 = 3600; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeletionConfig { + pub retry_max_attempts: u32, + pub retry_initial_backoff_seconds: u64, + pub retry_max_backoff_seconds: u64, + pub final_credential_issuance_window_seconds: u64, + pub final_read_window_seconds: u64, +} + +impl Default for DeletionConfig { + fn default() -> Self { + Self { + retry_max_attempts: DEFAULT_DELETION_RETRY_MAX_ATTEMPTS, + retry_initial_backoff_seconds: DEFAULT_DELETION_RETRY_INITIAL_BACKOFF_SECONDS, + retry_max_backoff_seconds: DEFAULT_DELETION_RETRY_MAX_BACKOFF_SECONDS, + final_credential_issuance_window_seconds: + DEFAULT_FINAL_CREDENTIAL_ISSUANCE_WINDOW_SECONDS, + final_read_window_seconds: DEFAULT_FINAL_READ_WINDOW_SECONDS, + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct PaykitConfig { pub server_url: String, @@ -105,13 +134,13 @@ impl Default for PkdnsConfig { #[derive(Debug, Clone, PartialEq, Eq)] pub struct SecretsConfig { - pub creator_authority_key_env: String, + pub runtime_master_key_env: String, } impl Default for SecretsConfig { fn default() -> Self { Self { - creator_authority_key_env: DEFAULT_CREATOR_AUTHORITY_KEY_ENV.to_owned(), + runtime_master_key_env: DEFAULT_RUNTIME_MASTER_KEY_ENV.to_owned(), } } } @@ -358,8 +387,16 @@ pub enum ConfigError { InvalidContentLocksTotalResourceBytes, #[error("invalid logging.level filter: {0}")] InvalidLoggingLevel(String), - #[error("secrets.creator_authority_key_env must not be empty")] - InvalidCreatorAuthorityKeyEnv, + #[error("secrets.runtime_master_key_env must not be empty")] + InvalidRuntimeMasterKeyEnv, + #[error("deletion retry values must be greater than zero")] + InvalidDeletionRetry, + #[error( + "deletion.retry_initial_backoff_seconds must not exceed deletion.retry_max_backoff_seconds" + )] + InvalidDeletionRetryBackoffOrder, + #[error("deletion credential windows must be between 1 and 3600 seconds")] + InvalidDeletionCredentialWindow, #[error( "creator_authority_acquisition.allowed_return_origins must contain http(s) origins without path, query, or fragment: {0}" )] diff --git a/locks-server/src/pkdns.rs b/locks-server/src/pkdns.rs index 47e332b..2c0f62a 100644 --- a/locks-server/src/pkdns.rs +++ b/locks-server/src/pkdns.rs @@ -366,6 +366,7 @@ mod tests { pkdns: PkdnsConfig::default(), rate_limits: RateLimitsConfig::default(), content_locks: ContentLocksConfig::default(), + deletion: crate::config::DeletionConfig::default(), paykit: None, } } diff --git a/locks-server/src/storage.rs b/locks-server/src/storage.rs index 94c963e..bdea208 100644 --- a/locks-server/src/storage.rs +++ b/locks-server/src/storage.rs @@ -1,6 +1,8 @@ +use locks_service::infrastructure::final_credentials::FinalCredentialCipher; use locks_service::infrastructure::postgres::{ CreatorAuthoritySecretCipher, PostgresError, run_migrations, }; +use locks_service::infrastructure::runtime_master_key::RuntimeMasterKey; use sqlx::postgres::PgPoolOptions; use crate::app_state::AppState; @@ -13,10 +15,10 @@ pub enum RuntimeStorageError { Connect(#[from] sqlx::Error), #[error("failed to run postgres runtime migrations: {0}")] Migrate(#[from] PostgresError), - #[error("creator authority encryption key env var is not set: {0}")] - MissingCreatorAuthorityEncryptionKeyEnv(String), - #[error("invalid creator authority encryption key in env var: {0}")] - InvalidCreatorAuthorityEncryptionKey(String), + #[error("runtime master key env var is not set: {0}")] + MissingRuntimeMasterKeyEnv(String), + #[error("invalid runtime master key in env var: {0}")] + InvalidRuntimeMasterKey(String), } /// Builds application state for the production-shaped runtime. @@ -30,7 +32,8 @@ pub enum RuntimeStorageError { pub async fn build_runtime_state( config: LockServerRuntimeConfig, ) -> Result { - let creator_authority_cipher = creator_authority_cipher_from_env(&config.secrets)?; + let (creator_authority_cipher, final_credential_cipher) = + runtime_ciphers_from_env(&config.secrets)?; let pool = connect_database(&config.database).await?; if config.database.run_migrations_on_startup { run_migrations(&pool).await?; @@ -40,22 +43,30 @@ pub async fn build_runtime_state( config, pool, creator_authority_cipher, + final_credential_cipher, )) } +#[cfg(test)] fn creator_authority_cipher_from_env( config: &SecretsConfig, ) -> Result { - let key = std::env::var(&config.creator_authority_key_env).map_err(|_| { - RuntimeStorageError::MissingCreatorAuthorityEncryptionKeyEnv( - config.creator_authority_key_env.clone(), - ) + runtime_ciphers_from_env(config).map(|(creator, _)| creator) +} + +fn runtime_ciphers_from_env( + config: &SecretsConfig, +) -> Result<(CreatorAuthoritySecretCipher, FinalCredentialCipher), RuntimeStorageError> { + let key = std::env::var(&config.runtime_master_key_env).map_err(|_| { + RuntimeStorageError::MissingRuntimeMasterKeyEnv(config.runtime_master_key_env.clone()) })?; - CreatorAuthoritySecretCipher::from_base64url_key(&key).map_err(|_| { - RuntimeStorageError::InvalidCreatorAuthorityEncryptionKey( - config.creator_authority_key_env.clone(), - ) - }) + let master_key = RuntimeMasterKey::from_base64url(&key).map_err(|_| { + RuntimeStorageError::InvalidRuntimeMasterKey(config.runtime_master_key_env.clone()) + })?; + Ok(( + CreatorAuthoritySecretCipher::new(master_key.creator_authority_key()), + FinalCredentialCipher::new(master_key.final_credential_key()), + )) } async fn connect_database(config: &DatabaseConfig) -> Result { @@ -81,7 +92,7 @@ mod tests { std::env::set_var(&env_name, URL_SAFE_NO_PAD.encode([7u8; 32])); } let config = SecretsConfig { - creator_authority_key_env: env_name.clone(), + runtime_master_key_env: env_name.clone(), }; let cipher = creator_authority_cipher_from_env(&config).unwrap(); @@ -99,14 +110,14 @@ mod tests { std::env::remove_var(&env_name); } let config = SecretsConfig { - creator_authority_key_env: env_name.clone(), + runtime_master_key_env: env_name.clone(), }; let error = creator_authority_cipher_from_env(&config).unwrap_err(); assert_eq!( error.to_string(), - format!("creator authority encryption key env var is not set: {env_name}") + format!("runtime master key env var is not set: {env_name}") ); } @@ -117,14 +128,14 @@ mod tests { std::env::set_var(&env_name, "not-a-valid-key"); } let config = SecretsConfig { - creator_authority_key_env: env_name.clone(), + runtime_master_key_env: env_name.clone(), }; let error = creator_authority_cipher_from_env(&config).unwrap_err(); assert!(matches!( error, - RuntimeStorageError::InvalidCreatorAuthorityEncryptionKey(ref name) if name == &env_name + RuntimeStorageError::InvalidRuntimeMasterKey(ref name) if name == &env_name )); let debug = format!("{error:?}"); assert!(!debug.contains("not-a-valid-key")); @@ -141,21 +152,21 @@ mod tests { } let mut config = TestServerApp::default_in_memory_config(); config.secrets = SecretsConfig { - creator_authority_key_env: env_name.clone(), + runtime_master_key_env: env_name.clone(), }; let error = build_runtime_state(config).await.unwrap_err(); assert!(matches!( error, - RuntimeStorageError::MissingCreatorAuthorityEncryptionKeyEnv(ref name) if name == &env_name + RuntimeStorageError::MissingRuntimeMasterKeyEnv(ref name) if name == &env_name )); assert!(!error.to_string().contains("postgres://")); } fn unique_env_name(suffix: &str) -> String { format!( - "LOCKS_TEST_CREATOR_AUTH_KEY_{}_{}", + "LOCKS_TEST_RUNTIME_MASTER_KEY_{}_{}", suffix, uuid::Uuid::new_v4().simple() ) diff --git a/locks-server/src/testing.rs b/locks-server/src/testing.rs index c4280b3..dda3729 100644 --- a/locks-server/src/testing.rs +++ b/locks-server/src/testing.rs @@ -100,6 +100,7 @@ impl TestServerApp { pkdns: crate::config::PkdnsConfig::default(), rate_limits: RateLimitsConfig::default(), content_locks: ContentLocksConfig::default(), + deletion: crate::config::DeletionConfig::default(), paykit: None, } } diff --git a/locks-server/src/worker.rs b/locks-server/src/worker.rs index a85d8aa..35dfac3 100644 --- a/locks-server/src/worker.rs +++ b/locks-server/src/worker.rs @@ -644,6 +644,7 @@ mod tests { pkdns: crate::config::PkdnsConfig::default(), rate_limits: RateLimitsConfig::default(), content_locks: ContentLocksConfig::default(), + deletion: crate::config::DeletionConfig::default(), paykit: None, } } diff --git a/locks-service/migrations/0016_content_lock_access_drains.sql b/locks-service/migrations/0016_content_lock_access_drains.sql new file mode 100644 index 0000000..18c4a40 --- /dev/null +++ b/locks-service/migrations/0016_content_lock_access_drains.sql @@ -0,0 +1,112 @@ +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM content_lock_deletion_jobs + WHERE state IN ('queued', 'running', 'failed') + ) THEN + RAISE EXCEPTION USING + MESSAGE = 'migration 0016 cannot classify pre-existing resumable deletion jobs; drain or explicitly reset pre-0016 deletion jobs before retrying', + HINT = 'see docs/RUNTIME.md for the required drain/reset procedure'; + END IF; +END +$$; + +ALTER TABLE content_lock_deletion_jobs + ADD COLUMN final_issuance_started_at TIMESTAMPTZ, + ADD COLUMN final_credential_issuance_deadline TIMESTAMPTZ, + ADD COLUMN final_read_deadline TIMESTAMPTZ, + ADD CONSTRAINT content_lock_deletion_jobs_final_window_shape CHECK ( + (final_issuance_started_at IS NULL + AND final_credential_issuance_deadline IS NULL + AND final_read_deadline IS NULL) + OR + (final_issuance_started_at IS NOT NULL + AND final_credential_issuance_deadline IS NOT NULL + AND final_read_deadline IS NOT NULL + AND final_issuance_started_at < final_credential_issuance_deadline + AND final_credential_issuance_deadline < final_read_deadline) + ); + +ALTER TABLE content_lock_deletion_task_snapshot + ADD COLUMN had_active_credential_at_cutoff BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN final_credential_eligible_at TIMESTAMPTZ, + ADD COLUMN final_credential_issued_at TIMESTAMPTZ, + ADD CONSTRAINT content_lock_deletion_task_snapshot_final_eligibility_valid CHECK ( + final_credential_eligible_at IS NULL + OR ( + had_active_credential_at_cutoff = FALSE + AND paykit_admission_required = TRUE + AND resolved_status = 'completed' + ) + ), + ADD CONSTRAINT content_lock_deletion_task_snapshot_final_issuance_valid CHECK ( + final_credential_issued_at IS NULL + OR final_credential_eligible_at IS NOT NULL + ); + +ALTER TABLE access_credentials + ADD COLUMN deletion_job_id UUID + REFERENCES content_lock_deletion_jobs(job_id) ON DELETE CASCADE; + +CREATE INDEX access_credentials_deletion_active_idx + ON access_credentials (deletion_job_id, expires_at) + WHERE deletion_job_id IS NOT NULL; + +CREATE TABLE content_lock_access_drain_credentials ( + credential_id UUID PRIMARY KEY, + deletion_job_id UUID NOT NULL + REFERENCES content_lock_deletion_jobs(job_id) ON DELETE CASCADE, + lookup_key BYTEA NOT NULL UNIQUE + REFERENCES access_credentials(lookup_key) ON DELETE CASCADE, + creator TEXT NOT NULL, + bundle_id TEXT NOT NULL, + credential_kind TEXT NOT NULL, + encrypted_bearer TEXT, + issued_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + CONSTRAINT content_lock_access_drain_credentials_kind_valid CHECK ( + credential_kind IN ('ordinary', 'final') + ), + CONSTRAINT content_lock_access_drain_credentials_envelope_valid CHECK ( + (credential_kind = 'ordinary' AND encrypted_bearer IS NULL) + OR + (credential_kind = 'final' + AND encrypted_bearer IS NOT NULL + AND encrypted_bearer LIKE 'v1.xchacha20poly1305:%') + ), + CONSTRAINT content_lock_access_drain_credentials_expiry_valid CHECK ( + issued_at < expires_at + ) +); + +CREATE UNIQUE INDEX content_lock_access_drain_one_final_per_bundle_idx + ON content_lock_access_drain_credentials (deletion_job_id, creator, bundle_id) + WHERE credential_kind = 'final'; + +CREATE INDEX content_lock_access_drain_credentials_job_expiry_idx + ON content_lock_access_drain_credentials (deletion_job_id, expires_at); + +CREATE TABLE content_lock_access_drain_reads ( + credential_id UUID NOT NULL + REFERENCES content_lock_access_drain_credentials(credential_id) ON DELETE CASCADE, + guarded_path TEXT NOT NULL, + claim_token UUID, + claim_expires_at TIMESTAMPTZ, + consumed_at TIMESTAMPTZ, + CONSTRAINT content_lock_access_drain_reads_pkey + PRIMARY KEY (credential_id, guarded_path), + CONSTRAINT content_lock_access_drain_reads_claim_shape CHECK ( + (claim_token IS NULL AND claim_expires_at IS NULL) + OR + (claim_token IS NOT NULL AND claim_expires_at IS NOT NULL) + ), + CONSTRAINT content_lock_access_drain_reads_consumed_shape CHECK ( + consumed_at IS NULL + OR (claim_token IS NULL AND claim_expires_at IS NULL) + ) +); + +CREATE INDEX content_lock_access_drain_reads_claim_idx + ON content_lock_access_drain_reads (claim_expires_at) + WHERE consumed_at IS NULL AND claim_token IS NOT NULL; diff --git a/locks-service/src/application/errors.rs b/locks-service/src/application/errors.rs index 75a775d..5937ed2 100644 --- a/locks-service/src/application/errors.rs +++ b/locks-service/src/application/errors.rs @@ -91,6 +91,12 @@ pub enum ApplicationError { /// Human-readable credential generation failure detail. message: String, }, + /// Final deletion credential envelope could not be encrypted or decrypted. + #[error("final credential secret error: {message}")] + FinalCredentialSecret { + /// Stable secret-free failure detail. + message: String, + }, /// Creator-granted homeserver authority is missing, expired, revoked, or unusable. #[error("creator authority unavailable")] CreatorAuthorityUnavailable, diff --git a/locks-service/src/application/models/access.rs b/locks-service/src/application/models/access.rs index 454c9bb..6caa741 100644 --- a/locks-service/src/application/models/access.rs +++ b/locks-service/src/application/models/access.rs @@ -1,7 +1,11 @@ use std::fmt; -use locks_core::ids::{BundleId, CreatorPubky}; +use locks_core::{ + ids::{BundleId, CreatorPubky}, + lock_policy::GuardedResource, +}; use time::OffsetDateTime; +use uuid::Uuid; use crate::application::errors::ApplicationError; @@ -36,6 +40,60 @@ impl fmt::Debug for AccessCredential { } } +/// Versioned encrypted bearer envelope persisted for exact final-credential replay. +#[derive(Clone, PartialEq, Eq)] +pub struct EncryptedFinalCredential(String); + +impl EncryptedFinalCredential { + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for EncryptedFinalCredential { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("EncryptedFinalCredential") + .field(&"") + .finish() + } +} + +/// Immutable identity bound into final-credential AEAD associated data. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FinalCredentialContext { + pub deletion_job_id: Uuid, + pub creator: CreatorPubky, + pub bundle_id: BundleId, +} + +/// A deletion credential returned only after its encrypted bearer is durable. +#[derive(Clone, PartialEq, Eq)] +pub struct IssuedDeletionCredential { + pub credential: AccessCredential, + pub expires_at: OffsetDateTime, +} + +impl fmt::Debug for IssuedDeletionCredential { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("IssuedDeletionCredential") + .field("credential", &"") + .field("expires_at", &self.expires_at) + .finish() + } +} + +/// Frozen-manifest authorization prepared before guarded-resource I/O. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeletionReadAuthorization { + pub claim_token: Option, + pub creator: CreatorPubky, + pub resource: GuardedResource, +} + /// Non-bearer lookup key derived from an access credential. /// /// Stores use this BLAKE3 digest instead of raw bearer credential strings. diff --git a/locks-service/src/application/ports/access.rs b/locks-service/src/application/ports/access.rs index 07539f1..f5ad1e5 100644 --- a/locks-service/src/application/ports/access.rs +++ b/locks-service/src/application/ports/access.rs @@ -1,8 +1,12 @@ use async_trait::async_trait; +use locks_core::ids::{BundleId, CreatorPubky, LockId}; +use time::OffsetDateTime; +use uuid::Uuid; use crate::application::errors::ApplicationError; use crate::application::models::{ - AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, + AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, DeletionReadAuthorization, + IssuedDeletionCredential, }; /// Store for issued opaque access credentials. @@ -13,6 +17,7 @@ pub trait AccessCredentialStore: Send + Sync { /// Returns `DuplicateRecord` if the credential lookup key already exists. async fn insert_access_credential( &self, + lock_id: &LockId, lookup_key: AccessCredentialLookupKey, record: AccessCredentialRecord, ) -> Result<(), ApplicationError>; @@ -32,6 +37,77 @@ pub trait AccessCredentialStore: Send + Sync { &self, lookup_key: &AccessCredentialLookupKey, ) -> Result<(), ApplicationError>; + + async fn initialize_final_access_windows( + &self, + _deletion_job_id: Uuid, + _worker_id: &str, + _claim_token: Uuid, + _now: OffsetDateTime, + _issuance_deadline: OffsetDateTime, + _read_deadline: OffsetDateTime, + ) -> Result { + Ok(false) + } + + async fn issue_or_replay_final_credential( + &self, + _creator: &CreatorPubky, + _bundle_id: &BundleId, + _now: OffsetDateTime, + _candidate: AccessCredential, + ) -> Result, ApplicationError> { + Ok(None) + } + + /// Reports whether this deletion Bundle may issue or replay its final credential now. + async fn final_credential_available( + &self, + _creator: &CreatorPubky, + _bundle_id: &BundleId, + _now: OffsetDateTime, + ) -> Result { + Ok(false) + } + + async fn prepare_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + _path: &str, + _now: OffsetDateTime, + _claim_expires_at: OffsetDateTime, + ) -> Result, ApplicationError> { + Ok(None) + } + + /// Reports whether a credential was enrolled in deletion, regardless of + /// whether deletion access is currently usable. + async fn deletion_credential_enrolled( + &self, + _lookup_key: &AccessCredentialLookupKey, + ) -> Result { + Ok(false) + } + + async fn release_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + _path: &str, + _claim_token: Uuid, + _now: OffsetDateTime, + ) -> Result { + Ok(false) + } + + async fn consume_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + _path: &str, + _claim_token: Uuid, + _now: OffsetDateTime, + ) -> Result { + Ok(false) + } } /// Generator for opaque access credentials. diff --git a/locks-service/src/application/use_cases/complete_verification_task.rs b/locks-service/src/application/use_cases/complete_verification_task.rs index bf4d7d9..252d3c5 100644 --- a/locks-service/src/application/use_cases/complete_verification_task.rs +++ b/locks-service/src/application/use_cases/complete_verification_task.rs @@ -415,6 +415,7 @@ fn viewer_safe_failure_message(error: &ApplicationError) -> &'static str { | ApplicationError::RateLimited | ApplicationError::UnsupportedCredentialTtl { .. } | ApplicationError::CredentialGeneration { .. } + | ApplicationError::FinalCredentialSecret { .. } | ApplicationError::CreatorAuthorityUnavailable | ApplicationError::CreatorAuthoritySecret { .. } | ApplicationError::InvalidCreatorAuthorityAuthKind { .. } diff --git a/locks-service/src/application/use_cases/credential_flow_tests.rs b/locks-service/src/application/use_cases/credential_flow_tests.rs index a2142da..7ecffc6 100644 --- a/locks-service/src/application/use_cases/credential_flow_tests.rs +++ b/locks-service/src/application/use_cases/credential_flow_tests.rs @@ -7,7 +7,7 @@ use time::OffsetDateTime; use time::macros::datetime; use locks_core::ids::{ - BundleId, ContentLockPath, CreatorPubky, GuardedResourceHash, LockServerPubky, + BundleId, ContentLockPath, CreatorPubky, GuardedResourceHash, LockId, LockServerPubky, PubkyLockResource, TaskId, }; use locks_core::lock_policy::{ @@ -583,6 +583,7 @@ impl FakeAccessCredentialStore { impl AccessCredentialStore for FakeAccessCredentialStore { async fn insert_access_credential( &self, + _lock_id: &LockId, lookup_key: AccessCredentialLookupKey, record: AccessCredentialRecord, ) -> Result<(), ApplicationError> { diff --git a/locks-service/src/application/use_cases/entitlement_check.rs b/locks-service/src/application/use_cases/entitlement_check.rs index 1cb3eed..6657cb6 100644 --- a/locks-service/src/application/use_cases/entitlement_check.rs +++ b/locks-service/src/application/use_cases/entitlement_check.rs @@ -1,4 +1,4 @@ -use locks_core::ids::{BundleId, ContentLockPath, CreatorPubky}; +use locks_core::ids::{BundleId, ContentLockPath, CreatorPubky, LockId}; use locks_core::lock_policy::ContentLock; use locks_core::verification::VerifiedProofBundle; @@ -10,6 +10,8 @@ use crate::application::ports::{ContentLockRepository, EntitlementRepository}; pub(super) struct ValidEntitlement { /// Current hash-verified content lock referenced by the entitlement. pub content_lock: ContentLock, + /// Canonical Lock ID verified against the entitlement path. + pub lock_id: LockId, } /// Loads and validates current entitlement state for credential issuance/validation. @@ -25,7 +27,7 @@ pub(super) async fn load_valid_entitlement( .ok_or(ApplicationError::EntitlementNotFound)?; let content_lock = load_current_content_lock(content_locks, &verified_proof_bundle).await?; - verify_content_lock_identity( + let lock_id = verify_content_lock_identity( &content_lock, verified_proof_bundle .pubky_lock_resource @@ -36,7 +38,10 @@ pub(super) async fn load_valid_entitlement( return Err(ApplicationError::EntitlementNotSatisfied); } - Ok(ValidEntitlement { content_lock }) + Ok(ValidEntitlement { + content_lock, + lock_id, + }) } async fn load_current_content_lock( @@ -57,7 +62,7 @@ async fn load_current_content_lock( pub(super) fn verify_content_lock_identity( content_lock: &ContentLock, content_lock_path: &ContentLockPath, -) -> Result<(), ApplicationError> { +) -> Result { let actual = content_lock .lock_id() @@ -67,7 +72,7 @@ pub(super) fn verify_content_lock_identity( let expected = content_lock_path.lock_id().clone(); if actual == expected { - Ok(()) + Ok(actual) } else { Err(ApplicationError::ContentLockHashMismatch { expected, actual }) } diff --git a/locks-service/src/application/use_cases/issue_access_credential.rs b/locks-service/src/application/use_cases/issue_access_credential.rs index 1d4e0b7..e921bf8 100644 --- a/locks-service/src/application/use_cases/issue_access_credential.rs +++ b/locks-service/src/application/use_cases/issue_access_credential.rs @@ -65,6 +65,32 @@ impl<'a> IssueAccessCredentialUseCase<'a> { &self, request: IssueAccessCredentialRequest, ) -> Result { + let now = self.clock.now(); + if self + .credential_store + .final_credential_available(&request.creator, &request.bundle_id, now) + .await? + { + let candidate = self + .credential_generator + .generate_access_credential() + .await?; + if let Some(final_credential) = self + .credential_store + .issue_or_replay_final_credential( + &request.creator, + &request.bundle_id, + now, + candidate, + ) + .await? + { + return Ok(IssuedAccessCredential { + credential: final_credential.credential, + expires_at: final_credential.expires_at, + }); + } + } let valid_entitlement = load_valid_entitlement( self.entitlements, self.content_locks, @@ -79,7 +105,7 @@ impl<'a> IssueAccessCredentialUseCase<'a> { .access_policy .requested_credential_ttl_seconds, )?; - let expires_at = self.clock.now() + Duration::seconds(requested_ttl_seconds as i64); + let expires_at = now + Duration::seconds(requested_ttl_seconds as i64); let credential = self .credential_generator .generate_access_credential() @@ -88,6 +114,7 @@ impl<'a> IssueAccessCredentialUseCase<'a> { self.credential_store .insert_access_credential( + &valid_entitlement.lock_id, lookup_key, AccessCredentialRecord { creator: request.creator, diff --git a/locks-service/src/application/use_cases/proxy_read_guarded_resource.rs b/locks-service/src/application/use_cases/proxy_read_guarded_resource.rs index f9b7388..90eaa4b 100644 --- a/locks-service/src/application/use_cases/proxy_read_guarded_resource.rs +++ b/locks-service/src/application/use_cases/proxy_read_guarded_resource.rs @@ -1,5 +1,5 @@ use crate::application::errors::ApplicationError; -use crate::application::models::AccessCredential; +use crate::application::models::{AccessCredential, AccessCredentialLookupKey}; use crate::application::ports::{ AccessCredentialStore, Clock, ContentLockRepository, EntitlementRepository, GuardedResourceRepository, @@ -30,6 +30,14 @@ pub struct ProxiedGuardedResource { pub hash: GuardedResourceHash, /// Guarded resource bytes. pub bytes: Vec, + deletion_claim: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct DeletionReadClaim { + lookup_key: AccessCredentialLookupKey, + path: String, + claim_token: uuid::Uuid, } /// Validates an access credential and returns the currently guarded resource bytes. @@ -64,6 +72,69 @@ impl<'a> ProxyReadGuardedResourceUseCase<'a> { &self, request: ProxyReadGuardedResourceRequest, ) -> Result { + let now = self.clock.now(); + let lookup_key = AccessCredentialLookupKey::derive(&request.credential); + if let Some(authorization) = self + .credential_store + .prepare_deletion_read( + &lookup_key, + &request.path, + now, + now + time::Duration::seconds(30), + ) + .await? + { + let claim_token = authorization.claim_token; + let guarded_resource = authorization.resource; + let prepared_response = async { + let guarded_record = self + .guarded_resources + .get_current_guarded_resource(&authorization.creator, &guarded_resource.path) + .await? + .ok_or(ApplicationError::GuardedResourceUnavailable)?; + if guarded_record.hash != guarded_resource.hash + || guarded_record.content_type != guarded_resource.content_type + || guarded_record.size != guarded_resource.size + { + return Err(ApplicationError::GuardedResourceUnavailable); + } + Ok(ProxiedGuardedResource { + path: guarded_resource.path, + content_type: guarded_record.content_type, + hash: guarded_resource.hash, + bytes: guarded_record.bytes, + deletion_claim: claim_token.map(|claim_token| DeletionReadClaim { + lookup_key: lookup_key.clone(), + path: request.path.clone(), + claim_token, + }), + }) + } + .await; + match prepared_response { + Ok(response) => return Ok(response), + Err(error) => { + if let Some(claim_token) = claim_token { + self.credential_store + .release_deletion_read( + &lookup_key, + &request.path, + claim_token, + self.clock.now(), + ) + .await?; + } + return Err(error); + } + } + } + if self + .credential_store + .deletion_credential_enrolled(&lookup_key) + .await? + { + return Err(ApplicationError::GuardedResourceUnavailable); + } let validation = ValidateAccessCredentialUseCase::new( self.credential_store, self.entitlements, @@ -105,18 +176,65 @@ impl<'a> ProxyReadGuardedResourceUseCase<'a> { content_type: guarded_record.content_type, hash: guarded_resource.hash, bytes: guarded_record.bytes, + deletion_claim: None, }) } + + /// Permanently consumes the exact final-read claim after a complete HTTP 200 + /// response has been constructed. A lost claim prevents response return. + pub async fn consume_prepared_deletion_read( + &self, + response: &ProxiedGuardedResource, + ) -> Result<(), ApplicationError> { + let Some(claim) = &response.deletion_claim else { + return Ok(()); + }; + if !self + .credential_store + .consume_deletion_read( + &claim.lookup_key, + &claim.path, + claim.claim_token, + self.clock.now(), + ) + .await? + { + return Err(ApplicationError::InvalidAccessCredential); + } + Ok(()) + } + + /// Releases the exact final-read claim when HTTP response construction fails. + pub async fn release_prepared_deletion_read( + &self, + response: &ProxiedGuardedResource, + ) -> Result<(), ApplicationError> { + let Some(claim) = &response.deletion_claim else { + return Ok(()); + }; + self.credential_store + .release_deletion_read( + &claim.lookup_key, + &claim.path, + claim.claim_token, + self.clock.now(), + ) + .await?; + Ok(()) + } } #[cfg(test)] mod tests { use std::collections::BTreeMap; use std::str::FromStr; + use std::sync::atomic::{AtomicUsize, Ordering}; + use async_trait::async_trait; use serde_json::json; use time::OffsetDateTime; use time::macros::datetime; + use uuid::Uuid; use locks_core::ids::{ BundleId, CreatorPubky, GuardedResourceHash, LockServerPubky, PubkyLockResource, @@ -133,7 +251,8 @@ mod tests { use super::{ProxyReadGuardedResourceRequest, ProxyReadGuardedResourceUseCase}; use crate::application::errors::ApplicationError; use crate::application::models::{ - AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, GuardedResourceRecord, + AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, + DeletionReadAuthorization, GuardedResourceRecord, }; use crate::application::ports::{ AccessCredentialStore, Clock, ContentLockRepository, EntitlementRepository, @@ -257,6 +376,207 @@ mod tests { assert_eq!(result, Err(ApplicationError::GuardedResourceUnavailable)); } + #[tokio::test] + async fn deletion_read_stays_claimed_until_response_boundary_consumes_it() { + let fixture = Fixture::seed().await; + let credentials = DeletionReadStore::new(); + let empty_public_locks = InMemoryContentLockRepository::new(); + let use_case = ProxyReadGuardedResourceUseCase::new( + &credentials, + &fixture.entitlements, + &empty_public_locks, + &fixture.guarded_resources, + &fixture.clock, + ); + + let response = use_case + .execute(ProxyReadGuardedResourceRequest { + credential: fixture.credential, + path: "/priv/locks.app/content/resource.txt".to_owned(), + }) + .await + .unwrap(); + + assert_eq!(response.bytes, b"guarded bytes".to_vec()); + assert_eq!(credentials.consumes.load(Ordering::SeqCst), 0); + use_case + .consume_prepared_deletion_read(&response) + .await + .unwrap(); + assert_eq!(credentials.consumes.load(Ordering::SeqCst), 1); + assert_eq!(credentials.releases.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn deletion_read_can_be_released_when_response_construction_fails() { + let fixture = Fixture::seed().await; + let credentials = DeletionReadStore::new(); + let use_case = ProxyReadGuardedResourceUseCase::new( + &credentials, + &fixture.entitlements, + &fixture.content_locks, + &fixture.guarded_resources, + &fixture.clock, + ); + let response = use_case + .execute(ProxyReadGuardedResourceRequest { + credential: fixture.credential, + path: "/priv/locks.app/content/resource.txt".to_owned(), + }) + .await + .unwrap(); + + use_case + .release_prepared_deletion_read(&response) + .await + .unwrap(); + + assert_eq!(credentials.releases.load(Ordering::SeqCst), 1); + assert_eq!(credentials.consumes.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn deletion_read_releases_claim_when_upstream_resource_is_unavailable() { + let fixture = Fixture::seed_without_guarded_resource().await; + let credentials = DeletionReadStore::new(); + let use_case = ProxyReadGuardedResourceUseCase::new( + &credentials, + &fixture.entitlements, + &fixture.content_locks, + &fixture.guarded_resources, + &fixture.clock, + ); + + let result = use_case + .execute(ProxyReadGuardedResourceRequest { + credential: fixture.credential, + path: "/priv/locks.app/content/resource.txt".to_owned(), + }) + .await; + + assert_eq!(result, Err(ApplicationError::GuardedResourceUnavailable)); + assert_eq!(credentials.releases.load(Ordering::SeqCst), 1); + assert_eq!(credentials.consumes.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn deletion_credential_denied_path_does_not_fall_through_to_public_lock() { + let fixture = Fixture::seed().await; + let credentials = DeletionReadStore::new(); + let empty_public_locks = InMemoryContentLockRepository::new(); + let use_case = ProxyReadGuardedResourceUseCase::new( + &credentials, + &fixture.entitlements, + &empty_public_locks, + &fixture.guarded_resources, + &fixture.clock, + ); + + let result = use_case + .execute(ProxyReadGuardedResourceRequest { + credential: fixture.credential, + path: "/priv/locks.app/content/not-frozen.txt".to_owned(), + }) + .await; + + assert_eq!(result, Err(ApplicationError::GuardedResourceUnavailable)); + assert_eq!(credentials.releases.load(Ordering::SeqCst), 0); + assert_eq!(credentials.consumes.load(Ordering::SeqCst), 0); + } + + struct DeletionReadStore { + ordinary: InMemoryAccessCredentialStore, + claim_token: Uuid, + releases: AtomicUsize, + consumes: AtomicUsize, + } + + impl DeletionReadStore { + fn new() -> Self { + Self { + ordinary: InMemoryAccessCredentialStore::new(), + claim_token: Uuid::new_v4(), + releases: AtomicUsize::new(0), + consumes: AtomicUsize::new(0), + } + } + } + + #[async_trait] + impl AccessCredentialStore for DeletionReadStore { + async fn insert_access_credential( + &self, + lock_id: &locks_core::ids::LockId, + lookup_key: AccessCredentialLookupKey, + record: AccessCredentialRecord, + ) -> Result<(), ApplicationError> { + self.ordinary + .insert_access_credential(lock_id, lookup_key, record) + .await + } + + async fn get_access_credential( + &self, + lookup_key: &AccessCredentialLookupKey, + ) -> Result, ApplicationError> { + self.ordinary.get_access_credential(lookup_key).await + } + + async fn delete_access_credential( + &self, + lookup_key: &AccessCredentialLookupKey, + ) -> Result<(), ApplicationError> { + self.ordinary.delete_access_credential(lookup_key).await + } + + async fn prepare_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + path: &str, + _now: OffsetDateTime, + _claim_expires_at: OffsetDateTime, + ) -> Result, ApplicationError> { + Ok((path == "/priv/locks.app/content/resource.txt").then(|| { + DeletionReadAuthorization { + claim_token: Some(self.claim_token), + creator: creator(), + resource: content_lock_fixture().primary_resource.unwrap(), + } + })) + } + + async fn deletion_credential_enrolled( + &self, + _lookup_key: &AccessCredentialLookupKey, + ) -> Result { + Ok(true) + } + + async fn release_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + _path: &str, + claim_token: Uuid, + _now: OffsetDateTime, + ) -> Result { + assert_eq!(claim_token, self.claim_token); + self.releases.fetch_add(1, Ordering::SeqCst); + Ok(true) + } + + async fn consume_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + _path: &str, + claim_token: Uuid, + _now: OffsetDateTime, + ) -> Result { + assert_eq!(claim_token, self.claim_token); + self.consumes.fetch_add(1, Ordering::SeqCst); + Ok(true) + } + } + struct Fixture { credentials: InMemoryAccessCredentialStore, entitlements: InMemoryEntitlementRepository, @@ -313,6 +633,7 @@ mod tests { .unwrap(); credentials .insert_access_credential( + &content_lock.lock_id().unwrap(), AccessCredentialLookupKey::derive(&credential), AccessCredentialRecord { creator: creator(), diff --git a/locks-service/src/infrastructure/final_credentials.rs b/locks-service/src/infrastructure/final_credentials.rs new file mode 100644 index 0000000..2692db6 --- /dev/null +++ b/locks-service/src/infrastructure/final_credentials.rs @@ -0,0 +1,234 @@ +use std::fmt; + +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use chacha20poly1305::aead::{Aead, KeyInit, Payload}; +use chacha20poly1305::{Key, XChaCha20Poly1305, XNonce}; +use rand::RngCore; +use rand::rngs::OsRng; + +use crate::application::errors::ApplicationError; +use crate::application::models::{ + AccessCredential, EncryptedFinalCredential, FinalCredentialContext, +}; + +const ENVELOPE_PREFIX: &str = "v1.xchacha20poly1305:"; +const AAD_DOMAIN: &[u8] = b"pubky-locks-final-credential-aad"; + +#[derive(Clone)] +pub struct FinalCredentialCipher { + key: [u8; 32], +} + +impl fmt::Debug for FinalCredentialCipher { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("FinalCredentialCipher") + .field(&"") + .finish() + } +} + +impl FinalCredentialCipher { + pub fn new(key: [u8; 32]) -> Self { + Self { key } + } + + pub fn encrypt( + &self, + context: &FinalCredentialContext, + credential: &AccessCredential, + ) -> Result { + let cipher = XChaCha20Poly1305::new(Key::from_slice(&self.key)); + let mut nonce_bytes = [0u8; 24]; + OsRng.fill_bytes(&mut nonce_bytes); + let ciphertext = cipher + .encrypt( + XNonce::from_slice(&nonce_bytes), + Payload { + msg: credential.as_str().as_bytes(), + aad: &associated_data(context), + }, + ) + .map_err(|_| encrypt_error())?; + Ok(EncryptedFinalCredential::new(format!( + "{ENVELOPE_PREFIX}{}:{}", + URL_SAFE_NO_PAD.encode(nonce_bytes), + URL_SAFE_NO_PAD.encode(ciphertext) + ))) + } + + pub fn decrypt( + &self, + context: &FinalCredentialContext, + envelope: &EncryptedFinalCredential, + ) -> Result { + let rest = envelope + .as_str() + .strip_prefix(ENVELOPE_PREFIX) + .ok_or_else(decrypt_error)?; + let (nonce, ciphertext) = rest.split_once(':').ok_or_else(decrypt_error)?; + let nonce: [u8; 24] = URL_SAFE_NO_PAD + .decode(nonce) + .map_err(|_| decrypt_error())? + .try_into() + .map_err(|_| decrypt_error())?; + let ciphertext = URL_SAFE_NO_PAD + .decode(ciphertext) + .map_err(|_| decrypt_error())?; + let cipher = XChaCha20Poly1305::new(Key::from_slice(&self.key)); + let plaintext = cipher + .decrypt( + XNonce::from_slice(&nonce), + Payload { + msg: &ciphertext, + aad: &associated_data(context), + }, + ) + .map_err(|_| decrypt_error())?; + String::from_utf8(plaintext) + .map(AccessCredential::new) + .map_err(|_| decrypt_error()) + } +} + +fn associated_data(context: &FinalCredentialContext) -> Vec { + let creator = context.creator.to_string(); + let bundle_id = context.bundle_id.to_string(); + let mut aad = Vec::with_capacity(AAD_DOMAIN.len() + creator.len() + bundle_id.len() + 32); + append_field(&mut aad, AAD_DOMAIN); + append_field(&mut aad, &[1]); + append_field(&mut aad, context.deletion_job_id.as_bytes()); + append_field(&mut aad, creator.as_bytes()); + append_field(&mut aad, bundle_id.as_bytes()); + aad +} + +fn append_field(output: &mut Vec, field: &[u8]) { + let length = u32::try_from(field.len()).expect("credential AAD fields fit in u32"); + output.extend_from_slice(&length.to_be_bytes()); + output.extend_from_slice(field); +} + +fn encrypt_error() -> ApplicationError { + ApplicationError::FinalCredentialSecret { + message: "failed to encrypt final credential".to_owned(), + } +} + +fn decrypt_error() -> ApplicationError { + ApplicationError::FinalCredentialSecret { + message: "invalid final credential envelope".to_owned(), + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use locks_core::ids::{BundleId, CreatorPubky}; + use uuid::Uuid; + + use super::*; + + #[test] + fn encrypted_final_credential_round_trips_under_exact_context() { + let cipher = FinalCredentialCipher::new([7; 32]); + let context = context(); + let credential = AccessCredential::new("secret-final-bearer"); + + let envelope = cipher.encrypt(&context, &credential).unwrap(); + let decrypted = cipher.decrypt(&context, &envelope).unwrap(); + + assert_eq!(decrypted, credential); + assert!(!envelope.as_str().contains(credential.as_str())); + assert!(!format!("{envelope:?}").contains(credential.as_str())); + assert!(!format!("{cipher:?}").contains('7')); + } + + #[test] + fn wrong_key_or_any_context_change_fails_closed() { + let cipher = FinalCredentialCipher::new([7; 32]); + let context = context(); + let envelope = cipher + .encrypt(&context, &AccessCredential::new("secret-final-bearer")) + .unwrap(); + + assert!( + FinalCredentialCipher::new([8; 32]) + .decrypt(&context, &envelope) + .is_err() + ); + for (field, changed) in [ + ( + "job", + FinalCredentialContext { + deletion_job_id: Uuid::new_v4(), + ..context.clone() + }, + ), + ( + "creator", + FinalCredentialContext { + creator: CreatorPubky::from_str( + &pubky_common::crypto::Keypair::from_secret(&[2; 32]) + .public_key() + .to_string(), + ) + .unwrap(), + ..context.clone() + }, + ), + ( + "bundle", + FinalCredentialContext { + bundle_id: BundleId::from_str("000G40R40M30E209185GR38E1V").unwrap(), + ..context.clone() + }, + ), + ] { + assert_ne!(changed, context, "{field} mutation must change context"); + assert!( + cipher.decrypt(&changed, &envelope).is_err(), + "altered {field} authenticated successfully" + ); + } + } + + #[test] + fn wrong_version_and_corrupt_envelopes_fail_without_secret_output() { + let cipher = FinalCredentialCipher::new([7; 32]); + let context = context(); + let bearer = "secret-final-bearer"; + let valid = cipher + .encrypt(&context, &AccessCredential::new(bearer)) + .unwrap(); + let corrupt = [ + EncryptedFinalCredential::new(valid.as_str().replacen("v1.", "v2.", 1)), + EncryptedFinalCredential::new("v1.xchacha20poly1305:not-base64:not-base64"), + EncryptedFinalCredential::new("v1.xchacha20poly1305:"), + ]; + + for envelope in corrupt { + let error = cipher.decrypt(&context, &envelope).unwrap_err(); + assert_eq!( + error, + ApplicationError::FinalCredentialSecret { + message: "invalid final credential envelope".to_owned() + } + ); + assert!(!format!("{error:?}").contains(bearer)); + assert!(!error.to_string().contains(envelope.as_str())); + } + } + + fn context() -> FinalCredentialContext { + FinalCredentialContext { + deletion_job_id: Uuid::from_u128(1), + creator: CreatorPubky::from_str( + "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy", + ) + .unwrap(), + bundle_id: BundleId::from_str("000G40R40M30E209185GR38E1W").unwrap(), + } + } +} diff --git a/locks-service/src/infrastructure/memory/access_credentials.rs b/locks-service/src/infrastructure/memory/access_credentials.rs index c03d9cd..fc0c3e2 100644 --- a/locks-service/src/infrastructure/memory/access_credentials.rs +++ b/locks-service/src/infrastructure/memory/access_credentials.rs @@ -1,39 +1,540 @@ -use std::collections::HashMap; +use std::{ + collections::{HashMap, HashSet}, + fmt, + sync::Arc, +}; use async_trait::async_trait; +use locks_core::{ + ids::{BundleId, CreatorPubky, LockId, TaskId}, + lock_policy::ContentLock, +}; +use rand::{RngCore, rngs::OsRng}; +use time::{Duration, OffsetDateTime}; use tokio::sync::RwLock; +use uuid::Uuid; -use crate::application::errors::ApplicationError; -use crate::application::models::{AccessCredentialLookupKey, AccessCredentialRecord}; -use crate::application::ports::AccessCredentialStore; +use crate::application::{ + errors::ApplicationError, + models::{ + AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, + ContentLockDeletionJob, ContentLockDeletionPhase, ContentLockDeletionState, + DeletionReadAuthorization, EncryptedFinalCredential, FinalCredentialContext, + IssuedDeletionCredential, VerificationTaskStatus, + }, + ports::{AccessCredentialStore, VerificationTaskRepository}, +}; +use crate::infrastructure::{ + final_credentials::FinalCredentialCipher, + memory::verification_task_deletion_fence::InMemoryVerificationTaskDeletionFence, +}; + +type JobKey = (CreatorPubky, LockId); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DrainCredentialKind { + Ordinary, + Final, +} + +#[derive(Debug, Clone)] +struct StoredCredential { + record: AccessCredentialRecord, + lock_id: LockId, + deletion: Option<(Uuid, DrainCredentialKind)>, +} + +#[derive(Debug, Clone)] +struct DeletionAccessState { + creator: CreatorPubky, + lock_id: LockId, + frozen_content_lock: ContentLock, + state: ContentLockDeletionState, + phase: ContentLockDeletionPhase, + force_requested: bool, + claimed_by: Option, + claim_token: Option, + claim_expires_at: Option, + issuance_deadline: Option, + read_deadline: Option, + payment_aggregate: Option, + bundle_snapshots: HashMap, +} + +#[derive(Debug, Clone, Copy)] +struct DeletionPaymentAggregate { + completed: bool, + accepted_count: u64, +} + +#[derive(Debug, Clone, Copy)] +struct DeletionBundleSnapshot { + task_id: TaskId, + paykit_admission_required: bool, + had_active_credential_at_cutoff: bool, + status_at_cutoff: VerificationTaskStatus, + resolved_status: Option, + resolved_at: Option, + final_credential_eligible_at: Option, + final_credential_issued: bool, +} + +impl DeletionBundleSnapshot { + fn permits_final_credential(self) -> bool { + self.paykit_admission_required + && !self.had_active_credential_at_cutoff + && self.resolved_status.unwrap_or(self.status_at_cutoff) + == VerificationTaskStatus::Completed + && self.final_credential_eligible_at.is_some() + } +} + +#[derive(Debug, Clone)] +struct FinalCredentialRecord { + lookup_key: AccessCredentialLookupKey, + encrypted_bearer: EncryptedFinalCredential, + expires_at: OffsetDateTime, + reads: HashMap, +} + +#[derive(Debug, Clone, Default)] +struct FinalReadState { + claim_token: Option, + claim_expires_at: Option, + consumed_at: Option, +} -/// In-memory access credential store keyed by non-secret lookup key. #[derive(Debug, Default)] +struct StoreState { + records: HashMap, + deletions: HashMap, + deletion_jobs_by_key: HashMap, + blocked_keys: HashSet, + final_credentials: HashMap<(Uuid, BundleId), FinalCredentialRecord>, +} + +/// In-memory access credential store with deletion-drain parity. pub struct InMemoryAccessCredentialStore { - records: RwLock>, + state: RwLock, + final_credential_cipher: FinalCredentialCipher, + verification_tasks: Option>, + verification_task_deletion_fence: Option>, +} + +impl fmt::Debug for InMemoryAccessCredentialStore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InMemoryAccessCredentialStore") + .field("state", &self.state) + .field("final_credential_cipher", &self.final_credential_cipher) + .field( + "verification_tasks", + &self.verification_tasks.as_ref().map(|_| ""), + ) + .field( + "verification_task_deletion_fence", + &self + .verification_task_deletion_fence + .as_ref() + .map(|_| ""), + ) + .finish() + } +} + +impl Default for InMemoryAccessCredentialStore { + fn default() -> Self { + Self::new() + } } impl InMemoryAccessCredentialStore { - /// Creates an empty store. + /// Creates an empty standalone store. Final credentials are unavailable until + /// verification dependencies are supplied. pub fn new() -> Self { - Self::default() + Self::build(None, None) + } + + pub fn with_verification_task_repository_and_deletion_fence( + verification_tasks: Arc, + verification_task_deletion_fence: Arc, + ) -> Self { + Self::build( + Some(verification_tasks), + Some(verification_task_deletion_fence), + ) + } + + fn build( + verification_tasks: Option>, + verification_task_deletion_fence: Option>, + ) -> Self { + let mut key = [0_u8; 32]; + OsRng.fill_bytes(&mut key); + Self { + state: RwLock::new(StoreState::default()), + final_credential_cipher: FinalCredentialCipher::new(key), + verification_tasks, + verification_task_deletion_fence, + } + } + + pub(crate) async fn register_deletion( + &self, + job: &ContentLockDeletionJob, + snapshot_bundles: &HashMap, + ) -> Result<(), ApplicationError> { + let key = (job.creator.clone(), job.lock_id.clone()); + let mut state = self.state.write().await; + if state.blocked_keys.contains(&key) || state.deletion_jobs_by_key.contains_key(&key) { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + + let bundle_snapshots = snapshot_bundles + .iter() + .map( + |(bundle_id, (task_id, paykit_admission_required, status_at_cutoff))| { + let had_active_credential_at_cutoff = state.records.values().any(|stored| { + stored.record.creator == job.creator + && stored.lock_id == job.lock_id + && stored.record.bundle_id == *bundle_id + && stored.record.expires_at > job.deletion_started_at + }); + ( + bundle_id.clone(), + DeletionBundleSnapshot { + task_id: *task_id, + paykit_admission_required: *paykit_admission_required, + had_active_credential_at_cutoff, + status_at_cutoff: *status_at_cutoff, + resolved_status: matches!( + status_at_cutoff, + VerificationTaskStatus::Completed + | VerificationTaskStatus::Failed + | VerificationTaskStatus::Expired + ) + .then_some(*status_at_cutoff), + resolved_at: matches!( + status_at_cutoff, + VerificationTaskStatus::Completed + | VerificationTaskStatus::Failed + | VerificationTaskStatus::Expired + ) + .then_some(job.deletion_started_at), + final_credential_eligible_at: (*paykit_admission_required + && *status_at_cutoff == VerificationTaskStatus::Completed + && !had_active_credential_at_cutoff) + .then_some(job.deletion_started_at), + final_credential_issued: false, + }, + ) + }, + ) + .collect(); + state.deletions.insert( + job.job_id, + DeletionAccessState { + creator: job.creator.clone(), + lock_id: job.lock_id.clone(), + frozen_content_lock: job.frozen_content_lock.clone(), + state: job.state, + phase: job.phase, + force_requested: job.force_requested_at.is_some(), + claimed_by: None, + claim_token: None, + claim_expires_at: None, + issuance_deadline: None, + read_deadline: None, + payment_aggregate: None, + bundle_snapshots, + }, + ); + state.deletion_jobs_by_key.insert(key.clone(), job.job_id); + state.blocked_keys.insert(key); + + for stored in state.records.values_mut() { + if stored.record.creator == job.creator + && stored.lock_id == job.lock_id + && snapshot_bundles.contains_key(&stored.record.bundle_id) + && stored.record.expires_at > job.deletion_started_at + { + stored.deletion = Some((job.job_id, DrainCredentialKind::Ordinary)); + } + } + Ok(()) + } + + pub(crate) async fn synchronize_job( + &self, + job: &ContentLockDeletionJob, + claimed_by: Option<&str>, + claim_token: Option, + claim_expires_at: Option, + ) { + let mut state = self.state.write().await; + if let Some(deletion) = state.deletions.get_mut(&job.job_id) { + deletion.state = job.state; + deletion.phase = job.phase; + deletion.force_requested = job.force_requested_at.is_some(); + deletion.claimed_by = claimed_by.map(str::to_owned); + deletion.claim_token = claim_token; + deletion.claim_expires_at = claim_expires_at; + } + if job.force_requested_at.is_some() + || matches!( + job.state, + ContentLockDeletionState::Completed | ContentLockDeletionState::Failed + ) + { + disable_job_access(&mut state, job.job_id); + } else if job.phase == ContentLockDeletionPhase::DeleteContent { + disable_final_access(&mut state, job.job_id); + } + } + + pub(crate) async fn block_key_and_disable_job( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + job_id: Option, + ) { + let mut state = self.state.write().await; + state + .blocked_keys + .insert((creator.clone(), lock_id.clone())); + if let Some(job_id) = job_id { + disable_job_access(&mut state, job_id); + } + } + + /// Records a terminal payment result only while deletion owns a live drain claim. + pub async fn resolve_deletion_payment( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + task_id: &TaskId, + status: VerificationTaskStatus, + ) -> Result { + if !matches!( + status, + VerificationTaskStatus::Completed | VerificationTaskStatus::Expired + ) { + return Err(ApplicationError::InvalidVerificationTaskState { + message: "payment drain transition must be completed or expired".to_owned(), + }); + } + let mut state = self.state.write().await; + let Some(deletion) = state.deletions.get_mut(&deletion_job_id) else { + return Ok(false); + }; + let owns_live_claim = deletion.state == ContentLockDeletionState::Running + && deletion.phase == ContentLockDeletionPhase::DrainPayments + && !deletion.force_requested + && deletion.claimed_by.as_deref() == Some(worker_id) + && deletion.claim_token == Some(claim_token) + && deletion + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at >= now); + if !owns_live_claim { + return Ok(false); + } + let Some(snapshot) = deletion + .bundle_snapshots + .values_mut() + .find(|snapshot| snapshot.task_id == *task_id) + else { + return Ok(false); + }; + if !snapshot.paykit_admission_required || snapshot.resolved_status.is_some() { + return Ok(false); + } + snapshot.resolved_status = Some(status); + snapshot.resolved_at = Some(now); + snapshot.final_credential_eligible_at = (status == VerificationTaskStatus::Completed + && !snapshot.had_active_credential_at_cutoff) + .then_some(now); + Ok(true) + } + + /// Marks the deletion-owned payment aggregate terminal only under the exact live drain claim. + pub async fn complete_deletion_payment_aggregate( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + ) -> Result { + let mut state = self.state.write().await; + let Some(deletion) = state.deletions.get_mut(&deletion_job_id) else { + return Ok(false); + }; + if !owns_live_payment_drain_claim(deletion, worker_id, claim_token, now) { + return Ok(false); + } + deletion.payment_aggregate = Some(DeletionPaymentAggregate { + completed: true, + accepted_count: 0, + }); + Ok(true) + } + + pub(crate) async fn check_phase_advance( + &self, + deletion_job_id: Uuid, + current_phase: ContentLockDeletionPhase, + next_phase: ContentLockDeletionPhase, + now: OffsetDateTime, + ) -> Result<(), ApplicationError> { + let state = self.state.read().await; + let Some(deletion) = state.deletions.get(&deletion_job_id) else { + return Ok(()); + }; + if current_phase == ContentLockDeletionPhase::DrainPayments + && next_phase == ContentLockDeletionPhase::DrainExistingCredentials + { + if deletion + .bundle_snapshots + .values() + .any(|snapshot| snapshot.resolved_status.is_none()) + { + return Err(invalid_deletion_state( + "every frozen deletion obligation must be terminal before credential draining", + )); + } + if !payment_aggregate_completed(deletion) { + return Err(invalid_deletion_state( + "payment drain aggregate must be durably completed before credential draining", + )); + } + } + if current_phase == ContentLockDeletionPhase::DrainExistingCredentials + && next_phase == ContentLockDeletionPhase::IssueFinalCredentials + && state.records.values().any(|stored| { + stored.deletion == Some((deletion_job_id, DrainCredentialKind::Ordinary)) + && stored.record.expires_at > now + }) + { + return Err(invalid_deletion_state( + "existing credentials must reach their original expiry before final issuance", + )); + } + if current_phase == ContentLockDeletionPhase::IssueFinalCredentials + && next_phase == ContentLockDeletionPhase::DrainFinalReads + && deletion.bundle_snapshots.values().any(|snapshot| { + snapshot.permits_final_credential() && !snapshot.final_credential_issued + }) + { + return Err(invalid_deletion_state( + "final credential issuance must complete before final-read draining", + )); + } + if current_phase == ContentLockDeletionPhase::DrainFinalReads + && next_phase == ContentLockDeletionPhase::DeleteContent + && has_live_access_obligation(&state, deletion_job_id, now) + { + return Err(invalid_deletion_state( + "credential expiry and final-read obligations must drain before destructive deletion", + )); + } + Ok(()) + } + + pub(crate) async fn check_successful_finish( + &self, + deletion_job_id: Uuid, + phase: ContentLockDeletionPhase, + now: OffsetDateTime, + ) -> Result<(), ApplicationError> { + let state = self.state.read().await; + let Some(deletion) = state.deletions.get(&deletion_job_id) else { + return Ok(()); + }; + if phase != ContentLockDeletionPhase::PurgeOperationalState { + return Err(invalid_deletion_state( + "successful completion requires the final operational-cleanup phase", + )); + } + if deletion + .bundle_snapshots + .values() + .any(|snapshot| snapshot.resolved_status.is_none()) + { + return Err(invalid_deletion_state( + "every frozen deletion obligation must be terminal before credential draining", + )); + } + if !payment_aggregate_completed(deletion) { + return Err(invalid_deletion_state( + "payment drain aggregate must be durably completed before credential draining", + )); + } + if has_live_access_obligation(&state, deletion_job_id, now) + || deletion.bundle_snapshots.values().any(|snapshot| { + snapshot.permits_final_credential() && !snapshot.final_credential_issued + }) + { + return Err(invalid_deletion_state( + "successful completion cannot bypass deletion access obligations", + )); + } + Ok(()) } } +fn owns_live_payment_drain_claim( + deletion: &DeletionAccessState, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, +) -> bool { + deletion.state == ContentLockDeletionState::Running + && deletion.phase == ContentLockDeletionPhase::DrainPayments + && !deletion.force_requested + && deletion.claimed_by.as_deref() == Some(worker_id) + && deletion.claim_token == Some(claim_token) + && deletion + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at >= now) +} + +fn payment_aggregate_completed(deletion: &DeletionAccessState) -> bool { + deletion + .payment_aggregate + .is_some_and(|aggregate| aggregate.completed && aggregate.accepted_count == 0) +} + #[async_trait] impl AccessCredentialStore for InMemoryAccessCredentialStore { async fn insert_access_credential( &self, + lock_id: &LockId, lookup_key: AccessCredentialLookupKey, record: AccessCredentialRecord, ) -> Result<(), ApplicationError> { - let mut records = self.records.write().await; - if records.contains_key(&lookup_key) { + let _admission = if let Some(fence) = &self.verification_task_deletion_fence { + Some(fence.acquire_lock_admission(&record.creator, lock_id).await) + } else { + None + }; + let mut state = self.state.write().await; + let key = (record.creator.clone(), lock_id.clone()); + if state.blocked_keys.contains(&key) || state.deletion_jobs_by_key.contains_key(&key) { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + if state.records.contains_key(&lookup_key) { return Err(ApplicationError::DuplicateRecord { record: "access_credential", }); } - records.insert(lookup_key, record); + state.records.insert( + lookup_key, + StoredCredential { + record, + lock_id: lock_id.clone(), + deletion: None, + }, + ); Ok(()) } @@ -41,16 +542,444 @@ impl AccessCredentialStore for InMemoryAccessCredentialStore { &self, lookup_key: &AccessCredentialLookupKey, ) -> Result, ApplicationError> { - Ok(self.records.read().await.get(lookup_key).cloned()) + Ok(self + .state + .read() + .await + .records + .get(lookup_key) + .map(|stored| stored.record.clone())) } async fn delete_access_credential( &self, lookup_key: &AccessCredentialLookupKey, ) -> Result<(), ApplicationError> { - self.records.write().await.remove(lookup_key); + let mut state = self.state.write().await; + state.records.remove(lookup_key); + state + .final_credentials + .retain(|_, credential| &credential.lookup_key != lookup_key); Ok(()) } + + async fn initialize_final_access_windows( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + issuance_deadline: OffsetDateTime, + read_deadline: OffsetDateTime, + ) -> Result { + if issuance_deadline <= now || read_deadline <= issuance_deadline { + return Err(ApplicationError::Storage { + message: "invalid final access window ordering".to_owned(), + }); + } + let mut state = self.state.write().await; + let Some(deletion) = state.deletions.get_mut(&deletion_job_id) else { + return Ok(false); + }; + let owns_live_claim = deletion.state == ContentLockDeletionState::Running + && deletion.phase == ContentLockDeletionPhase::IssueFinalCredentials + && !deletion.force_requested + && deletion.claimed_by.as_deref() == Some(worker_id) + && deletion.claim_token == Some(claim_token) + && deletion + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at > now); + if !owns_live_claim { + return Ok(false); + } + deletion.issuance_deadline.get_or_insert(issuance_deadline); + deletion.read_deadline.get_or_insert(read_deadline); + Ok(true) + } + + async fn issue_or_replay_final_credential( + &self, + creator: &CreatorPubky, + bundle_id: &BundleId, + now: OffsetDateTime, + candidate: AccessCredential, + ) -> Result, ApplicationError> { + let mut state = self.state.write().await; + let Some((deletion_job_id, _)) = state.deletions.iter().find(|(_, deletion)| { + deletion.creator == *creator && deletion.bundle_snapshots.contains_key(bundle_id) + }) else { + return Ok(None); + }; + let deletion_job_id = *deletion_job_id; + let Some(deletion) = state.deletions.get(&deletion_job_id) else { + return Ok(None); + }; + if deletion.creator != *creator + || deletion.force_requested + || !matches!( + deletion.state, + ContentLockDeletionState::Queued | ContentLockDeletionState::Running + ) + || !matches!( + deletion.phase, + ContentLockDeletionPhase::IssueFinalCredentials + | ContentLockDeletionPhase::DrainFinalReads + ) + || deletion + .read_deadline + .is_none_or(|deadline| deadline <= now) + { + return Ok(None); + } + + let context = FinalCredentialContext { + deletion_job_id, + creator: creator.clone(), + bundle_id: bundle_id.clone(), + }; + if let Some(existing) = state + .final_credentials + .get(&(deletion_job_id, bundle_id.clone())) + { + let credential = self + .final_credential_cipher + .decrypt(&context, &existing.encrypted_bearer)?; + return Ok(Some(IssuedDeletionCredential { + credential, + expires_at: existing.expires_at, + })); + } + + if deletion.phase != ContentLockDeletionPhase::IssueFinalCredentials + || deletion + .issuance_deadline + .is_none_or(|deadline| now >= deadline) + || !deletion + .bundle_snapshots + .get(bundle_id) + .copied() + .is_some_and(DeletionBundleSnapshot::permits_final_credential) + { + return Ok(None); + } + + let expires_at = deletion + .read_deadline + .expect("final issuance requires an initialized read deadline"); + let frozen_content_lock = deletion.frozen_content_lock.clone(); + let lock_id = deletion.lock_id.clone(); + let encrypted_bearer = self.final_credential_cipher.encrypt(&context, &candidate)?; + let lookup_key = AccessCredentialLookupKey::derive(&candidate); + if state.records.contains_key(&lookup_key) { + return Err(ApplicationError::DuplicateRecord { + record: "access_credential", + }); + } + let mut reads = HashMap::new(); + if let Some(resource) = frozen_content_lock.primary_resource { + reads.insert(resource.path, FinalReadState::default()); + } + for path in frozen_content_lock.secondary_resources.keys() { + reads.insert(path.clone(), FinalReadState::default()); + } + state.records.insert( + lookup_key.clone(), + StoredCredential { + record: AccessCredentialRecord { + creator: creator.clone(), + bundle_id: bundle_id.clone(), + expires_at, + }, + lock_id, + deletion: Some((deletion_job_id, DrainCredentialKind::Final)), + }, + ); + state.final_credentials.insert( + (deletion_job_id, bundle_id.clone()), + FinalCredentialRecord { + lookup_key, + encrypted_bearer, + expires_at, + reads, + }, + ); + state + .deletions + .get_mut(&deletion_job_id) + .and_then(|deletion| deletion.bundle_snapshots.get_mut(bundle_id)) + .expect("issued final credential must retain its immutable snapshot") + .final_credential_issued = true; + Ok(Some(IssuedDeletionCredential { + credential: candidate, + expires_at, + })) + } + + async fn final_credential_available( + &self, + creator: &CreatorPubky, + bundle_id: &BundleId, + now: OffsetDateTime, + ) -> Result { + let state = self.state.read().await; + let Some((deletion_job_id, deletion)) = state.deletions.iter().find(|(_, deletion)| { + deletion.creator == *creator && deletion.bundle_snapshots.contains_key(bundle_id) + }) else { + return Ok(false); + }; + let lifecycle_allows_access = !deletion.force_requested + && matches!( + deletion.state, + ContentLockDeletionState::Queued | ContentLockDeletionState::Running + ) + && matches!( + deletion.phase, + ContentLockDeletionPhase::IssueFinalCredentials + | ContentLockDeletionPhase::DrainFinalReads + ) + && deletion + .read_deadline + .is_some_and(|deadline| deadline > now); + if !lifecycle_allows_access { + return Ok(false); + } + if state + .final_credentials + .contains_key(&(*deletion_job_id, bundle_id.clone())) + { + return Ok(true); + } + Ok( + deletion.phase == ContentLockDeletionPhase::IssueFinalCredentials + && deletion + .issuance_deadline + .is_some_and(|deadline| now < deadline) + && deletion + .bundle_snapshots + .get(bundle_id) + .copied() + .is_some_and(DeletionBundleSnapshot::permits_final_credential), + ) + } + + async fn prepare_deletion_read( + &self, + lookup_key: &AccessCredentialLookupKey, + path: &str, + now: OffsetDateTime, + claim_expires_at: OffsetDateTime, + ) -> Result, ApplicationError> { + let mut state = self.state.write().await; + let Some(stored) = state.records.get(lookup_key).cloned() else { + return Ok(None); + }; + let Some((deletion_job_id, kind)) = stored.deletion else { + return Ok(None); + }; + let Some(deletion) = state.deletions.get(&deletion_job_id) else { + return Ok(None); + }; + if stored.record.expires_at <= now + || deletion.force_requested + || !matches!( + deletion.state, + ContentLockDeletionState::Queued | ContentLockDeletionState::Running + ) + || !matches!( + deletion.phase, + ContentLockDeletionPhase::Withdraw + | ContentLockDeletionPhase::StartPaymentDrain + | ContentLockDeletionPhase::DrainPayments + | ContentLockDeletionPhase::DrainExistingCredentials + | ContentLockDeletionPhase::IssueFinalCredentials + | ContentLockDeletionPhase::DrainFinalReads + ) + { + return Ok(None); + } + let Some(resource) = deletion.frozen_content_lock.resource_for_path(path) else { + return Ok(None); + }; + let creator = deletion.creator.clone(); + if kind == DrainCredentialKind::Ordinary { + return Ok(Some(DeletionReadAuthorization { + claim_token: None, + creator, + resource, + })); + } + if !matches!( + deletion.phase, + ContentLockDeletionPhase::IssueFinalCredentials + | ContentLockDeletionPhase::DrainFinalReads + ) || deletion + .read_deadline + .is_none_or(|deadline| deadline <= now) + { + return Ok(None); + } + let read_deadline = deletion + .read_deadline + .expect("final credential requires a read deadline"); + let Some(final_credential) = state + .final_credentials + .get_mut(&(deletion_job_id, stored.record.bundle_id.clone())) + else { + return Ok(None); + }; + let Some(read) = final_credential.reads.get_mut(path) else { + return Ok(None); + }; + if read.consumed_at.is_some() + || (read.claim_token.is_some() + && read + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at > now)) + { + return Ok(None); + } + let bounded_expiry = claim_expires_at + .min(now + Duration::seconds(30)) + .min(final_credential.expires_at) + .min(read_deadline); + if bounded_expiry <= now { + return Ok(None); + } + let claim_token = Uuid::new_v4(); + read.claim_token = Some(claim_token); + read.claim_expires_at = Some(bounded_expiry); + Ok(Some(DeletionReadAuthorization { + claim_token: Some(claim_token), + creator, + resource, + })) + } + + async fn deletion_credential_enrolled( + &self, + lookup_key: &AccessCredentialLookupKey, + ) -> Result { + let state = self.state.read().await; + Ok(state + .records + .get(lookup_key) + .is_some_and(|stored| stored.deletion.is_some())) + } + + async fn release_deletion_read( + &self, + lookup_key: &AccessCredentialLookupKey, + path: &str, + claim_token: Uuid, + _now: OffsetDateTime, + ) -> Result { + let mut state = self.state.write().await; + let Some((deletion_job_id, bundle_id)) = final_credential_identity(&state, lookup_key) + else { + return Ok(false); + }; + let Some(read) = state + .final_credentials + .get_mut(&(deletion_job_id, bundle_id)) + .and_then(|credential| credential.reads.get_mut(path)) + else { + return Ok(false); + }; + if read.consumed_at.is_some() || read.claim_token != Some(claim_token) { + return Ok(false); + } + read.claim_token = None; + read.claim_expires_at = None; + Ok(true) + } + + async fn consume_deletion_read( + &self, + lookup_key: &AccessCredentialLookupKey, + path: &str, + claim_token: Uuid, + now: OffsetDateTime, + ) -> Result { + let mut state = self.state.write().await; + let Some((deletion_job_id, bundle_id)) = final_credential_identity(&state, lookup_key) + else { + return Ok(false); + }; + let Some(read) = state + .final_credentials + .get_mut(&(deletion_job_id, bundle_id)) + .and_then(|credential| credential.reads.get_mut(path)) + else { + return Ok(false); + }; + if read.consumed_at.is_some() + || read.claim_token != Some(claim_token) + || read + .claim_expires_at + .is_none_or(|claim_expires_at| claim_expires_at <= now) + { + return Ok(false); + } + read.claim_token = None; + read.claim_expires_at = None; + read.consumed_at = Some(now); + Ok(true) + } +} + +fn has_live_access_obligation( + state: &StoreState, + deletion_job_id: Uuid, + now: OffsetDateTime, +) -> bool { + state.records.values().any(|stored| { + stored.deletion == Some((deletion_job_id, DrainCredentialKind::Ordinary)) + && stored.record.expires_at > now + }) || state + .final_credentials + .iter() + .any(|((job_id, _), credential)| { + *job_id == deletion_job_id + && credential.expires_at > now + && credential + .reads + .values() + .any(|read| read.consumed_at.is_none()) + }) +} + +fn invalid_deletion_state(message: &str) -> ApplicationError { + ApplicationError::InvalidContentLockDeletionState { + message: message.to_owned(), + } +} + +fn final_credential_identity( + state: &StoreState, + lookup_key: &AccessCredentialLookupKey, +) -> Option<(Uuid, BundleId)> { + let stored = state.records.get(lookup_key)?; + let (job_id, kind) = stored.deletion?; + (kind == DrainCredentialKind::Final).then(|| (job_id, stored.record.bundle_id.clone())) +} + +fn disable_final_access(state: &mut StoreState, job_id: Uuid) { + revoke_job_read_claims(state, job_id); +} + +fn disable_job_access(state: &mut StoreState, job_id: Uuid) { + revoke_job_read_claims(state, job_id); +} + +fn revoke_job_read_claims(state: &mut StoreState, job_id: Uuid) { + for ((deletion_job_id, _), credential) in &mut state.final_credentials { + if *deletion_job_id == job_id { + for read in credential.reads.values_mut() { + read.claim_token = None; + read.claim_expires_at = None; + } + } + } } #[cfg(test)] @@ -59,10 +988,9 @@ mod tests { use time::macros::datetime; - use locks_core::ids::{BundleId, CreatorPubky}; + use locks_core::ids::{BundleId, CreatorPubky, LockId}; use super::*; - use crate::application::models::AccessCredential; #[tokio::test] async fn insert_rejects_duplicate_read_miss_is_none_delete_is_ensure_absent() { @@ -70,13 +998,15 @@ mod tests { let credential = AccessCredential::new("raw-bearer-credential"); let lookup_key = AccessCredentialLookupKey::derive(&credential); let record = record(); + let lock_id = + LockId::from_str("000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG").unwrap(); assert_eq!( store.get_access_credential(&lookup_key).await.unwrap(), None ); store - .insert_access_credential(lookup_key.clone(), record.clone()) + .insert_access_credential(&lock_id, lookup_key.clone(), record.clone()) .await .unwrap(); assert_eq!( @@ -85,7 +1015,7 @@ mod tests { ); assert_eq!( store - .insert_access_credential(lookup_key.clone(), record) + .insert_access_credential(&lock_id, lookup_key.clone(), record) .await, Err(ApplicationError::DuplicateRecord { record: "access_credential", diff --git a/locks-service/src/infrastructure/memory/content_lock_deletions.rs b/locks-service/src/infrastructure/memory/content_lock_deletions.rs index 4494d62..b11508b 100644 --- a/locks-service/src/infrastructure/memory/content_lock_deletions.rs +++ b/locks-service/src/infrastructure/memory/content_lock_deletions.rs @@ -17,7 +17,10 @@ use crate::application::{ }, ports::ContentLockDeletionRepository, }; -use crate::infrastructure::memory::verification_task_deletion_fence::InMemoryVerificationTaskDeletionFence; +use crate::infrastructure::memory::{ + access_credentials::InMemoryAccessCredentialStore, + verification_task_deletion_fence::InMemoryVerificationTaskDeletionFence, +}; type JobKey = (CreatorPubky, LockId); @@ -36,11 +39,15 @@ pub struct InMemoryContentLockDeletionRepository { force_receipts: RwLock>, publication_intents: RwLock>, verification_task_fence: Arc, + access_credentials: Arc, } impl Default for InMemoryContentLockDeletionRepository { fn default() -> Self { - Self::with_verification_task_fence(Arc::new(InMemoryVerificationTaskDeletionFence::new())) + Self::with_access_credentials_and_verification_task_fence( + Arc::new(InMemoryAccessCredentialStore::new()), + Arc::new(InMemoryVerificationTaskDeletionFence::new()), + ) } } @@ -51,12 +58,23 @@ impl InMemoryContentLockDeletionRepository { pub fn with_verification_task_fence( verification_task_fence: Arc, + ) -> Self { + Self::with_access_credentials_and_verification_task_fence( + Arc::new(InMemoryAccessCredentialStore::new()), + verification_task_fence, + ) + } + + pub fn with_access_credentials_and_verification_task_fence( + access_credentials: Arc, + verification_task_fence: Arc, ) -> Self { Self { jobs: RwLock::new(HashMap::new()), force_receipts: RwLock::new(HashSet::new()), publication_intents: RwLock::new(HashMap::new()), verification_task_fence, + access_credentials, } } } @@ -127,10 +145,15 @@ impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { .contains_key(&(creator.clone(), lock_id.clone()))) } - async fn insert_job(&self, job: ContentLockDeletionJob) -> Result<(), ApplicationError> { + async fn insert_job(&self, mut job: ContentLockDeletionJob) -> Result<(), ApplicationError> { job.validate_frozen_identity()?; job.validate_state(false)?; let key = (job.creator.clone(), job.lock_id.clone()); + let _admission = self + .verification_task_fence + .acquire_lock_admission(&job.creator, &job.lock_id) + .await; + job.deletion_started_at = self.verification_task_fence.authoritative_cutoff(); let mut verification_tasks = self.verification_task_fence.records.write().await; let intents = self.publication_intents.read().await; let mut jobs = self.jobs.write().await; @@ -149,6 +172,17 @@ impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { (task.creator == job.creator && task.lock_id == job.lock_id).then_some(*task_id) }) .collect::>(); + let snapshot_bundles = matching_task_ids + .iter() + .filter_map(|task_id| { + verification_tasks.get(task_id).map(|task| { + ( + task.bundle_id.clone(), + (*task_id, task.paykit_admission_required, task.status), + ) + }) + }) + .collect::>(); if matching_task_ids.iter().any(|task_id| { verification_tasks .get(task_id) @@ -156,6 +190,9 @@ impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { }) { return Err(ApplicationError::ContentLockDeletionInProgress); } + self.access_credentials + .register_deletion(&job, &snapshot_bundles) + .await?; for task_id in matching_task_ids { if let Some(task) = verification_tasks.get_mut(&task_id) { task.deletion_job_id = Some(job.job_id); @@ -217,6 +254,14 @@ impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { stored.claimed_by = Some(worker_id.to_owned()); stored.claim_token = Some(claim_token); stored.claim_expires_at = Some(claim_expires_at); + self.access_credentials + .synchronize_job( + &stored.job, + stored.claimed_by.as_deref(), + stored.claim_token, + stored.claim_expires_at, + ) + .await; Ok(Some(ClaimedContentLockDeletionJob { job: stored.job.clone(), claim_token, @@ -241,6 +286,9 @@ impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { stored.job.state = ContentLockDeletionState::Queued; stored.job.next_attempt_at = Some(next_attempt_at); clear_claim(stored); + self.access_credentials + .synchronize_job(&stored.job, None, None, None) + .await; Ok(Some(stored.job.clone())) } @@ -264,12 +312,18 @@ impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { message: "deletion phase must advance to its immediate successor".to_owned(), }); } + self.access_credentials + .check_phase_advance(job_id, stored.job.phase, next_phase, now) + .await?; stored.job.phase = next_phase; stored.job.state = ContentLockDeletionState::Queued; stored.job.attempt_count = 0; stored.job.next_attempt_at = None; stored.job.failure_code = None; clear_claim(stored); + self.access_credentials + .synchronize_job(&stored.job, None, None, None) + .await; Ok(Some(stored.job.clone())) } @@ -288,6 +342,11 @@ impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { else { return Ok(None); }; + if failure_code.is_none() { + self.access_credentials + .check_successful_finish(job_id, stored.job.phase, now) + .await?; + } stored.job.state = if failure_code.is_some() { ContentLockDeletionState::Failed } else { @@ -296,6 +355,9 @@ impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { stored.job.failure_code = failure_code; stored.job.next_attempt_at = None; clear_claim(stored); + self.access_credentials + .synchronize_job(&stored.job, None, None, None) + .await; Ok(Some(stored.job.clone())) } @@ -320,6 +382,9 @@ impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { stored.job.failure_code = None; clear_claim(stored); } + self.access_credentials + .synchronize_job(&stored.job, None, None, None) + .await; Ok(Some(stored.job.clone())) } @@ -360,13 +425,22 @@ impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { stored.job.state = ContentLockDeletionState::Queued; stored.job.next_attempt_at = None; clear_claim(stored); + self.access_credentials + .synchronize_job(&stored.job, None, None, None) + .await; return Ok(PrepareForceDeletionResult::Active(stored.job.clone())); } let job = stored.job.clone(); + self.access_credentials + .block_key_and_disable_job(creator, lock_id, Some(job.job_id)) + .await; jobs.remove(&key); receipts.insert(key); return Ok(PrepareForceDeletionResult::Synchronous(Some(job))); } + self.access_credentials + .block_key_and_disable_job(creator, lock_id, None) + .await; receipts.insert(key); Ok(PrepareForceDeletionResult::Synchronous(None)) } diff --git a/locks-service/src/infrastructure/memory/verification_task_deletion_fence.rs b/locks-service/src/infrastructure/memory/verification_task_deletion_fence.rs index 993cbc9..dd1ca8b 100644 --- a/locks-service/src/infrastructure/memory/verification_task_deletion_fence.rs +++ b/locks-service/src/infrastructure/memory/verification_task_deletion_fence.rs @@ -1,23 +1,60 @@ -use std::collections::HashMap; +use std::{collections::HashMap, fmt, sync::Arc}; -use locks_core::ids::{CreatorPubky, LockId, TaskId}; -use tokio::sync::RwLock; +use locks_core::ids::{BundleId, CreatorPubky, LockId, TaskId}; +use locks_core::lock_policy::VerifierType; +use time::OffsetDateTime; +use tokio::sync::{Mutex, OwnedMutexGuard, RwLock}; use uuid::Uuid; -use crate::application::models::VerificationTaskRecord; +use crate::application::{ + models::{VerificationTaskRecord, VerificationTaskStatus}, + ports::Clock, +}; + +type LockKey = (CreatorPubky, LockId); + +#[derive(Debug)] +struct SystemClock; + +impl Clock for SystemClock { + fn now(&self) -> OffsetDateTime { + OffsetDateTime::now_utc() + } +} #[derive(Debug, Clone)] pub(crate) struct InMemoryVerificationTaskFenceRecord { pub(crate) creator: CreatorPubky, pub(crate) lock_id: LockId, + pub(crate) bundle_id: BundleId, + pub(crate) paykit_admission_required: bool, + pub(crate) status: VerificationTaskStatus, pub(crate) entitlement_publication_claim_token: Option, pub(crate) deletion_job_id: Option, } -/// Shared in-memory serialization state for verification publication and deletion admission. -#[derive(Debug, Default)] +/// Shared in-memory serialization state for all admission decisions for one content lock. pub struct InMemoryVerificationTaskDeletionFence { pub(crate) records: RwLock>, + lock_admissions: Mutex>>>, + clock: Arc, +} + +impl fmt::Debug for InMemoryVerificationTaskDeletionFence { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("InMemoryVerificationTaskDeletionFence") + .field("records", &self.records) + .field("lock_admissions", &self.lock_admissions) + .field("clock", &"") + .finish() + } +} + +impl Default for InMemoryVerificationTaskDeletionFence { + fn default() -> Self { + Self::with_clock(Arc::new(SystemClock)) + } } impl InMemoryVerificationTaskDeletionFence { @@ -25,6 +62,36 @@ impl InMemoryVerificationTaskDeletionFence { Self::default() } + /// Creates a canonical in-memory admission fence with an injected cutoff clock. + pub fn with_clock(clock: Arc) -> Self { + Self { + records: RwLock::new(HashMap::new()), + lock_admissions: Mutex::new(HashMap::new()), + clock, + } + } + + pub(crate) async fn acquire_lock_admission( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> OwnedMutexGuard<()> { + let key = (creator.clone(), lock_id.clone()); + let admission = { + let mut admissions = self.lock_admissions.lock().await; + Arc::clone( + admissions + .entry(key) + .or_insert_with(|| Arc::new(Mutex::new(()))), + ) + }; + admission.lock_owned().await + } + + pub(crate) fn authoritative_cutoff(&self) -> OffsetDateTime { + self.clock.now() + } + pub(crate) fn from_tasks(tasks: &[VerificationTaskRecord]) -> Self { Self { records: RwLock::new( @@ -40,6 +107,15 @@ impl InMemoryVerificationTaskDeletionFence { .pubky_lock_resource .lock_id() .clone(), + bundle_id: task.submitted_proof_bundle.bundle_id.clone(), + paykit_admission_required: task + .submitted_proof_bundle + .proofs + .iter() + .any(|proof| { + proof.verifier_type == VerifierType::PaykitPayment + }), + status: task.status, entitlement_publication_claim_token: None, deletion_job_id: None, }, @@ -47,6 +123,8 @@ impl InMemoryVerificationTaskDeletionFence { }) .collect(), ), + lock_admissions: Mutex::new(HashMap::new()), + clock: Arc::new(SystemClock), } } } diff --git a/locks-service/src/infrastructure/memory/verification_tasks.rs b/locks-service/src/infrastructure/memory/verification_tasks.rs index 4413e2f..694873c 100644 --- a/locks-service/src/infrastructure/memory/verification_tasks.rs +++ b/locks-service/src/infrastructure/memory/verification_tasks.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use tokio::sync::RwLock; use locks_core::ids::{BundleId, CreatorPubky, TaskId}; +use locks_core::lock_policy::VerifierType; use crate::application::errors::ApplicationError; use crate::application::models::VerificationTaskRecord; @@ -67,6 +68,13 @@ impl VerificationTaskRepository for InMemoryVerificationTaskRepository { .pubky_lock_resource .lock_id() .clone(), + bundle_id: task.submitted_proof_bundle.bundle_id.clone(), + paykit_admission_required: task + .submitted_proof_bundle + .proofs + .iter() + .any(|proof| proof.verifier_type == VerifierType::PaykitPayment), + status: task.status, entitlement_publication_claim_token: None, deletion_job_id: None, }, @@ -79,12 +87,16 @@ impl VerificationTaskRepository for InMemoryVerificationTaskRepository { &self, task: VerificationTaskRecord, ) -> Result<(), ApplicationError> { + let mut fence_records = self.deletion_fence.records.write().await; let mut records = self.records.write().await; if !records.contains_key(&task.task_id) { return Err(ApplicationError::MissingRecord { record: "verification_task", }); } + if let Some(fence_record) = fence_records.get_mut(&task.task_id) { + fence_record.status = task.status; + } records.insert(task.task_id, task); Ok(()) } diff --git a/locks-service/src/infrastructure/mod.rs b/locks-service/src/infrastructure/mod.rs index d0b718f..2acc2b3 100644 --- a/locks-service/src/infrastructure/mod.rs +++ b/locks-service/src/infrastructure/mod.rs @@ -1,4 +1,6 @@ +pub mod final_credentials; pub mod memory; pub mod postgres; pub mod pubky; +pub mod runtime_master_key; pub mod verifiers; diff --git a/locks-service/src/infrastructure/postgres/access_credentials.rs b/locks-service/src/infrastructure/postgres/access_credentials.rs index ab17cbe..b7790f0 100644 --- a/locks-service/src/infrastructure/postgres/access_credentials.rs +++ b/locks-service/src/infrastructure/postgres/access_credentials.rs @@ -1,24 +1,49 @@ use std::str::FromStr; use async_trait::async_trait; -use sqlx::{PgPool, Row}; +use sqlx::{PgPool, Postgres, Row, Transaction}; +use time::OffsetDateTime; +use uuid::Uuid; -use locks_core::ids::{BundleId, CreatorPubky}; +use locks_core::{ + ids::{BundleId, CreatorPubky, LockId}, + lock_policy::{ContentLock, GuardedResource}, +}; use crate::application::errors::ApplicationError; -use crate::application::models::{AccessCredentialLookupKey, AccessCredentialRecord}; +use crate::application::models::{ + AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, DeletionReadAuthorization, + EncryptedFinalCredential, FinalCredentialContext, IssuedDeletionCredential, +}; use crate::application::ports::AccessCredentialStore; +use crate::infrastructure::final_credentials::FinalCredentialCipher; + +use super::proof_admission::lock_proof_admission; /// Postgres-backed store for issued access credential lookup records. #[derive(Debug, Clone)] pub struct PostgresAccessCredentialStore { pool: PgPool, + final_credential_cipher: Option, } impl PostgresAccessCredentialStore { /// Creates a store backed by the provided migrated Postgres pool. pub fn new(pool: PgPool) -> Self { - Self { pool } + Self { + pool, + final_credential_cipher: None, + } + } + + pub fn with_final_credential_cipher( + pool: PgPool, + final_credential_cipher: FinalCredentialCipher, + ) -> Self { + Self { + pool, + final_credential_cipher: Some(final_credential_cipher), + } } } @@ -26,9 +51,28 @@ impl PostgresAccessCredentialStore { impl AccessCredentialStore for PostgresAccessCredentialStore { async fn insert_access_credential( &self, + lock_id: &LockId, lookup_key: AccessCredentialLookupKey, record: AccessCredentialRecord, ) -> Result<(), ApplicationError> { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, &record.creator, lock_id).await?; + let deletion_exists: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 + FROM content_lock_deletion_jobs + WHERE creator = $1 AND lock_id = $2 + )", + ) + .bind(record.creator.to_string()) + .bind(lock_id.to_string()) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if deletion_exists { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + let result = sqlx::query( "INSERT INTO access_credentials (lookup_key, creator, bundle_id, expires_at) VALUES ($1, $2, $3, $4) @@ -38,7 +82,7 @@ impl AccessCredentialStore for PostgresAccessCredentialStore { .bind(record.creator.to_string()) .bind(record.bundle_id.to_string()) .bind(record.expires_at) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(storage_error)?; @@ -48,6 +92,7 @@ impl AccessCredentialStore for PostgresAccessCredentialStore { }); } + transaction.commit().await.map_err(storage_error)?; Ok(()) } @@ -79,6 +124,568 @@ impl AccessCredentialStore for PostgresAccessCredentialStore { .map_err(storage_error)?; Ok(()) } + + async fn initialize_final_access_windows( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + issuance_deadline: OffsetDateTime, + read_deadline: OffsetDateTime, + ) -> Result { + if issuance_deadline <= now || read_deadline <= issuance_deadline { + return Err(ApplicationError::Storage { + message: "invalid final access window ordering".to_owned(), + }); + } + let updated = sqlx::query( + "UPDATE content_lock_deletion_jobs + SET final_issuance_started_at = COALESCE(final_issuance_started_at, $4), + final_credential_issuance_deadline = + COALESCE(final_credential_issuance_deadline, $5), + final_read_deadline = COALESCE(final_read_deadline, $6) + WHERE job_id = $1 AND claimed_by = $2 AND claim_token = $3 + AND claim_expires_at > $4 AND state = 'running' + AND force_requested_at IS NULL AND phase = 'issue_final_credentials'", + ) + .bind(deletion_job_id) + .bind(worker_id) + .bind(claim_token) + .bind(now) + .bind(issuance_deadline) + .bind(read_deadline) + .execute(&self.pool) + .await + .map_err(storage_error)?; + Ok(updated.rows_affected() == 1) + } + + async fn issue_or_replay_final_credential( + &self, + creator: &CreatorPubky, + bundle_id: &BundleId, + now: OffsetDateTime, + candidate: AccessCredential, + ) -> Result, ApplicationError> { + let cipher = match &self.final_credential_cipher { + Some(cipher) => cipher, + None => return Ok(None), + }; + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let job = sqlx::query( + "SELECT job.job_id, job.phase, job.final_credential_issuance_deadline, + job.final_read_deadline, job.frozen_content_lock + FROM content_lock_deletion_jobs AS job + WHERE job.creator = $1 + AND job.state IN ('queued', 'running') + AND job.force_requested_at IS NULL + AND job.phase IN ('issue_final_credentials', 'drain_final_reads') + AND job.final_read_deadline > $3 + AND EXISTS ( + SELECT 1 + FROM content_lock_deletion_task_snapshot AS snapshot + WHERE snapshot.deletion_job_id = job.job_id + AND snapshot.bundle_id = $2 + AND snapshot.resolved_status = 'completed' + AND snapshot.final_credential_eligible_at IS NOT NULL + ) + FOR UPDATE OF job", + ) + .bind(creator.to_string()) + .bind(bundle_id.to_string()) + .bind(now) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + let Some(job) = job else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let deletion_job_id: Uuid = job.try_get("job_id").map_err(storage_error)?; + let phase: String = job.try_get("phase").map_err(storage_error)?; + let issuance_deadline: OffsetDateTime = job + .try_get("final_credential_issuance_deadline") + .map_err(storage_error)?; + let expires_at: OffsetDateTime = + job.try_get("final_read_deadline").map_err(storage_error)?; + let snapshot_exists = sqlx::query_scalar::<_, bool>( + "SELECT TRUE + FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 AND bundle_id = $2 + AND resolved_status = 'completed' + AND final_credential_eligible_at IS NOT NULL + FOR UPDATE", + ) + .bind(deletion_job_id) + .bind(bundle_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)? + .unwrap_or(false); + if !snapshot_exists { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let existing_encrypted: Option = sqlx::query_scalar( + "SELECT encrypted_bearer + FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND creator = $2 AND bundle_id = $3 + AND credential_kind = 'final' + FOR UPDATE", + ) + .bind(deletion_job_id) + .bind(creator.to_string()) + .bind(bundle_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)? + .flatten(); + let context = FinalCredentialContext { + deletion_job_id, + creator: creator.clone(), + bundle_id: bundle_id.clone(), + }; + if let Some(encrypted) = existing_encrypted { + let credential = cipher.decrypt(&context, &EncryptedFinalCredential::new(encrypted))?; + transaction.commit().await.map_err(storage_error)?; + return Ok(Some(IssuedDeletionCredential { + credential, + expires_at, + })); + } + if phase != "issue_final_credentials" || now >= issuance_deadline { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let frozen: serde_json::Value = + job.try_get("frozen_content_lock").map_err(storage_error)?; + let frozen: ContentLock = + serde_json::from_value(frozen).map_err(|error| ApplicationError::Storage { + message: format!("invalid frozen content lock stored in Postgres: {error}"), + })?; + let encrypted = cipher.encrypt(&context, &candidate)?; + let lookup_key = AccessCredentialLookupKey::derive(&candidate); + let credential_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO access_credentials ( + lookup_key, creator, bundle_id, expires_at, deletion_job_id + ) VALUES ($1, $2, $3, $4, $5)", + ) + .bind(lookup_key.as_bytes().as_slice()) + .bind(creator.to_string()) + .bind(bundle_id.to_string()) + .bind(expires_at) + .bind(deletion_job_id) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + sqlx::query( + "INSERT INTO content_lock_access_drain_credentials ( + credential_id, deletion_job_id, lookup_key, creator, bundle_id, + credential_kind, issued_at, expires_at, encrypted_bearer + ) VALUES ($1, $2, $3, $4, $5, 'final', $6, $7, $8)", + ) + .bind(credential_id) + .bind(deletion_job_id) + .bind(lookup_key.as_bytes().as_slice()) + .bind(creator.to_string()) + .bind(bundle_id.to_string()) + .bind(now) + .bind(expires_at) + .bind(encrypted.as_str()) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + let mut resources: Vec = frozen.primary_resource.into_iter().collect(); + resources.extend( + frozen + .secondary_resources + .into_iter() + .map(|(path, resource)| { + GuardedResource::new(path, resource.hash, resource.content_type, resource.size) + .expect("persisted frozen manifest was validated at deletion admission") + }), + ); + for resource in resources { + sqlx::query( + "INSERT INTO content_lock_access_drain_reads (credential_id, guarded_path) + VALUES ($1, $2)", + ) + .bind(credential_id) + .bind(resource.path) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + } + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET final_credential_issued_at = $3 + WHERE deletion_job_id = $1 AND bundle_id = $2 + AND final_credential_issued_at IS NULL", + ) + .bind(deletion_job_id) + .bind(bundle_id.to_string()) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(Some(IssuedDeletionCredential { + credential: candidate, + expires_at, + })) + } + + async fn final_credential_available( + &self, + creator: &CreatorPubky, + bundle_id: &BundleId, + now: OffsetDateTime, + ) -> Result { + if self.final_credential_cipher.is_none() { + return Ok(false); + } + sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 + FROM content_lock_deletion_jobs AS job + JOIN content_lock_deletion_task_snapshot AS snapshot + ON snapshot.deletion_job_id = job.job_id + WHERE job.creator = $1 AND snapshot.bundle_id = $2 + AND job.state IN ('queued', 'running') + AND job.force_requested_at IS NULL + AND job.phase IN ('issue_final_credentials', 'drain_final_reads') + AND snapshot.resolved_status = 'completed' + AND snapshot.final_credential_eligible_at IS NOT NULL + AND job.final_read_deadline > $3 + AND ( + EXISTS ( + SELECT 1 + FROM content_lock_access_drain_credentials AS credential + WHERE credential.deletion_job_id = job.job_id + AND credential.creator = $1 + AND credential.bundle_id = $2 + AND credential.credential_kind = 'final' + ) + OR ( + job.phase = 'issue_final_credentials' + AND job.final_credential_issuance_deadline > $3 + ) + ) + )", + ) + .bind(creator.to_string()) + .bind(bundle_id.to_string()) + .bind(now) + .fetch_one(&self.pool) + .await + .map_err(storage_error) + } + + async fn prepare_deletion_read( + &self, + lookup_key: &AccessCredentialLookupKey, + path: &str, + now: OffsetDateTime, + _claim_expires_at: OffsetDateTime, + ) -> Result, ApplicationError> { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let Some(deletion_job_id) = lookup_deletion_job_id(&mut transaction, lookup_key).await? + else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let Some(job) = lock_active_drain_job(&mut transaction, deletion_job_id).await? else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let credential = sqlx::query( + "SELECT credential_id, credential_kind, creator, expires_at + FROM content_lock_access_drain_credentials + WHERE lookup_key = $1 AND deletion_job_id = $2 + FOR UPDATE", + ) + .bind(lookup_key.as_bytes().as_slice()) + .bind(deletion_job_id) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + let Some(credential) = credential else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let kind: String = credential + .try_get("credential_kind") + .map_err(storage_error)?; + let credential_expiry: OffsetDateTime = + credential.try_get("expires_at").map_err(storage_error)?; + if credential_expiry <= now || !phase_allows_credential_access(&job.phase, &kind) { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let frozen: ContentLock = + serde_json::from_value(job.frozen_content_lock).map_err(|error| { + ApplicationError::Storage { + message: format!("invalid frozen content lock stored in Postgres: {error}"), + } + })?; + let Some(resource) = frozen.resource_for_path(path) else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let creator = CreatorPubky::from_str( + &credential + .try_get::("creator") + .map_err(storage_error)?, + ) + .map_err(|error| ApplicationError::Storage { + message: format!("invalid drain credential creator stored in Postgres: {error}"), + })?; + if kind == "ordinary" { + transaction.commit().await.map_err(storage_error)?; + return Ok(Some(DeletionReadAuthorization { + claim_token: None, + creator, + resource, + })); + } + let credential_id: Uuid = credential.try_get("credential_id").map_err(storage_error)?; + let read = sqlx::query( + "SELECT claim_token, claim_expires_at, consumed_at + FROM content_lock_access_drain_reads + WHERE credential_id = $1 AND guarded_path = $2 + FOR UPDATE", + ) + .bind(credential_id) + .bind(path) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + let Some(read) = read else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + if read + .try_get::, _>("consumed_at") + .map_err(storage_error)? + .is_some() + { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let existing_claim: Option = read.try_get("claim_token").map_err(storage_error)?; + let existing_expiry: Option = + read.try_get("claim_expires_at").map_err(storage_error)?; + if existing_claim.is_some() && existing_expiry.is_some_and(|expiry| expiry > now) { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let Some(read_deadline) = job.final_read_deadline else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let bounded_expiry = (now + time::Duration::seconds(30)) + .min(credential_expiry) + .min(read_deadline); + if read_deadline <= now || bounded_expiry <= now { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let claim_token = Uuid::new_v4(); + let updated = sqlx::query( + "UPDATE content_lock_access_drain_reads + SET claim_token = $3, claim_expires_at = $4 + WHERE credential_id = $1 AND guarded_path = $2 + AND consumed_at IS NULL + AND (claim_token IS NULL OR claim_expires_at <= $5)", + ) + .bind(credential_id) + .bind(path) + .bind(claim_token) + .bind(bounded_expiry) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + if updated.rows_affected() != 1 { + transaction.rollback().await.map_err(storage_error)?; + return Ok(None); + } + transaction.commit().await.map_err(storage_error)?; + Ok(Some(DeletionReadAuthorization { + claim_token: Some(claim_token), + creator, + resource, + })) + } + + async fn deletion_credential_enrolled( + &self, + lookup_key: &AccessCredentialLookupKey, + ) -> Result { + sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 + FROM content_lock_access_drain_credentials + WHERE lookup_key = $1 + )", + ) + .bind(lookup_key.as_bytes().as_slice()) + .fetch_one(&self.pool) + .await + .map_err(storage_error) + } + + async fn release_deletion_read( + &self, + lookup_key: &AccessCredentialLookupKey, + path: &str, + claim_token: Uuid, + _now: OffsetDateTime, + ) -> Result { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let Some(deletion_job_id) = lookup_deletion_job_id(&mut transaction, lookup_key).await? + else { + transaction.commit().await.map_err(storage_error)?; + return Ok(false); + }; + let Some(job) = lock_active_drain_job(&mut transaction, deletion_job_id).await? else { + transaction.commit().await.map_err(storage_error)?; + return Ok(false); + }; + if !phase_allows_credential_access(&job.phase, "final") { + transaction.commit().await.map_err(storage_error)?; + return Ok(false); + } + let updated = sqlx::query( + "UPDATE content_lock_access_drain_reads AS read + SET claim_token = NULL, claim_expires_at = NULL + FROM content_lock_access_drain_credentials AS credential + WHERE read.credential_id = credential.credential_id + AND credential.deletion_job_id = $4 + AND credential.lookup_key = $1 AND read.guarded_path = $2 + AND read.claim_token = $3 AND read.consumed_at IS NULL", + ) + .bind(lookup_key.as_bytes().as_slice()) + .bind(path) + .bind(claim_token) + .bind(deletion_job_id) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(updated.rows_affected() == 1) + } + + async fn consume_deletion_read( + &self, + lookup_key: &AccessCredentialLookupKey, + path: &str, + claim_token: Uuid, + now: OffsetDateTime, + ) -> Result { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let Some(deletion_job_id) = lookup_deletion_job_id(&mut transaction, lookup_key).await? + else { + transaction.commit().await.map_err(storage_error)?; + return Ok(false); + }; + let Some(job) = lock_active_drain_job(&mut transaction, deletion_job_id).await? else { + transaction.commit().await.map_err(storage_error)?; + return Ok(false); + }; + if !phase_allows_credential_access(&job.phase, "final") + || job + .final_read_deadline + .is_none_or(|deadline| deadline <= now) + { + transaction.commit().await.map_err(storage_error)?; + return Ok(false); + } + let updated = sqlx::query( + "UPDATE content_lock_access_drain_reads AS read + SET claim_token = NULL, claim_expires_at = NULL, consumed_at = $4 + FROM content_lock_access_drain_credentials AS credential + WHERE read.credential_id = credential.credential_id + AND credential.deletion_job_id = $5 + AND credential.lookup_key = $1 AND read.guarded_path = $2 + AND read.claim_token = $3 AND read.claim_expires_at > $4 + AND read.consumed_at IS NULL", + ) + .bind(lookup_key.as_bytes().as_slice()) + .bind(path) + .bind(claim_token) + .bind(now) + .bind(deletion_job_id) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(updated.rows_affected() == 1) + } +} + +struct LockedDrainJob { + phase: String, + final_read_deadline: Option, + frozen_content_lock: serde_json::Value, +} + +async fn lookup_deletion_job_id( + transaction: &mut Transaction<'_, Postgres>, + lookup_key: &AccessCredentialLookupKey, +) -> Result, ApplicationError> { + sqlx::query_scalar( + "SELECT deletion_job_id + FROM content_lock_access_drain_credentials + WHERE lookup_key = $1", + ) + .bind(lookup_key.as_bytes().as_slice()) + .fetch_optional(&mut **transaction) + .await + .map_err(storage_error) +} + +async fn lock_active_drain_job( + transaction: &mut Transaction<'_, Postgres>, + deletion_job_id: Uuid, +) -> Result, ApplicationError> { + let row = sqlx::query( + "SELECT phase, final_read_deadline, frozen_content_lock + FROM content_lock_deletion_jobs + WHERE job_id = $1 AND state IN ('queued', 'running') + AND force_requested_at IS NULL + FOR UPDATE", + ) + .bind(deletion_job_id) + .fetch_optional(&mut **transaction) + .await + .map_err(storage_error)?; + row.map(|row| { + Ok(LockedDrainJob { + phase: row.try_get("phase").map_err(storage_error)?, + final_read_deadline: row.try_get("final_read_deadline").map_err(storage_error)?, + frozen_content_lock: row.try_get("frozen_content_lock").map_err(storage_error)?, + }) + }) + .transpose() +} + +fn phase_allows_credential_access(phase: &str, credential_kind: &str) -> bool { + match credential_kind { + "ordinary" => matches!( + phase, + "withdraw" + | "start_payment_drain" + | "drain_payments" + | "drain_existing_credentials" + | "issue_final_credentials" + | "drain_final_reads" + ), + "final" => matches!(phase, "issue_final_credentials" | "drain_final_reads"), + _ => false, + } } fn row_to_record(row: sqlx::postgres::PgRow) -> Result { @@ -115,7 +722,7 @@ mod tests { use sqlx::Row; use time::macros::datetime; - use locks_core::ids::{BundleId, CreatorPubky}; + use locks_core::ids::{BundleId, CreatorPubky, LockId}; use super::PostgresAccessCredentialStore; use crate::application::errors::ApplicationError; @@ -132,6 +739,8 @@ mod tests { let credential = AccessCredential::new("raw-bearer-credential"); let lookup_key = AccessCredentialLookupKey::derive(&credential); let record = access_credential_record(); + let lock_id = + LockId::from_str("000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG").unwrap(); assert_eq!( store.get_access_credential(&lookup_key).await.unwrap(), @@ -139,7 +748,7 @@ mod tests { ); store - .insert_access_credential(lookup_key.clone(), record.clone()) + .insert_access_credential(&lock_id, lookup_key.clone(), record.clone()) .await .unwrap(); assert_eq!( @@ -148,7 +757,7 @@ mod tests { ); assert_eq!( store - .insert_access_credential(lookup_key.clone(), record) + .insert_access_credential(&lock_id, lookup_key.clone(), record) .await, Err(ApplicationError::DuplicateRecord { record: "access_credential", @@ -175,9 +784,11 @@ mod tests { let credential = AccessCredential::new(raw_credential); let lookup_key = AccessCredentialLookupKey::derive(&credential); let record = access_credential_record(); + let lock_id = + LockId::from_str("000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG").unwrap(); original_store - .insert_access_credential(lookup_key.clone(), record.clone()) + .insert_access_credential(&lock_id, lookup_key.clone(), record.clone()) .await .unwrap(); @@ -194,6 +805,43 @@ mod tests { database.cleanup().await; } + #[tokio::test] + async fn committed_deletion_rejects_ordinary_credential_without_inserting() { + let database = TestDatabase::create().await; + let store = PostgresAccessCredentialStore::new(database.pool().clone()); + let lock_id = + LockId::from_str("000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG").unwrap(); + let record = access_credential_record(); + let lookup_key = AccessCredentialLookupKey::derive(&AccessCredential::new("rejected")); + sqlx::query( + "INSERT INTO content_lock_deletion_jobs ( + job_id, creator, lock_id, deletion_started_at, frozen_content_lock + ) VALUES ($1, $2, $3, $4, $5)", + ) + .bind(uuid::Uuid::new_v4()) + .bind(record.creator.to_string()) + .bind(lock_id.to_string()) + .bind(datetime!(2026-05-29 12:00:00 UTC)) + .bind(serde_json::json!({"version": "1"})) + .execute(database.pool()) + .await + .unwrap(); + + assert_eq!( + store + .insert_access_credential(&lock_id, lookup_key, record) + .await, + Err(ApplicationError::ContentLockDeletionInProgress) + ); + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM access_credentials") + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(count, 0); + + database.cleanup().await; + } + async fn assert_stored_lookup_key_is_exact( pool: &sqlx::PgPool, lookup_key: &AccessCredentialLookupKey, diff --git a/locks-service/src/infrastructure/postgres/content_lock_deletions.rs b/locks-service/src/infrastructure/postgres/content_lock_deletions.rs index 0a364a5..d0312ce 100644 --- a/locks-service/src/infrastructure/postgres/content_lock_deletions.rs +++ b/locks-service/src/infrastructure/postgres/content_lock_deletions.rs @@ -5,7 +5,7 @@ use locks_core::{ ids::{CreatorPubky, LockId}, lock_policy::ContentLock, }; -use sqlx::{FromRow, PgPool, Postgres, Transaction}; +use sqlx::{FromRow, PgPool, Postgres, Row, Transaction}; use time::OffsetDateTime; use uuid::Uuid; @@ -124,6 +124,10 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { ); let mut transaction = self.pool.begin().await.map_err(storage_error)?; lock_proof_admission(&mut transaction, &job.creator, &job.lock_id).await?; + let admission_cutoff: OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; let deletion_cutoff_exists = sqlx::query_scalar::<_, bool>( "SELECT EXISTS (SELECT 1 FROM content_lock_force_deletion_receipts WHERE creator = $1 AND lock_id = $2) @@ -175,7 +179,7 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { .bind(job.creator.to_string()) .bind(job.lock_id.to_string()) .bind(frozen) - .bind(job.deletion_started_at) + .bind(admission_cutoff) .bind(state_to_database(job.state)) .bind(phase_to_database(job.phase)) .bind(i64::from(job.attempt_count)) @@ -220,6 +224,99 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { .execute(&mut *transaction) .await .map_err(storage_error)?; + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot AS snapshot + SET had_active_credential_at_cutoff = EXISTS ( + SELECT 1 FROM access_credentials AS credential + WHERE credential.creator = snapshot.creator + AND credential.bundle_id = snapshot.bundle_id + AND credential.expires_at > $2 + ) + WHERE snapshot.deletion_job_id = $1", + ) + .bind(job.job_id) + .bind(admission_cutoff) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET resolved_status = status_at_cutoff, + resolved_at = $2, + final_credential_eligible_at = CASE + WHEN status_at_cutoff = 'completed' + AND paykit_admission_required + AND NOT had_active_credential_at_cutoff + THEN $2 + ELSE NULL + END + WHERE deletion_job_id = $1 + AND status_at_cutoff IN ('completed', 'failed', 'expired')", + ) + .bind(job.job_id) + .bind(admission_cutoff) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + sqlx::query( + "UPDATE access_credentials AS credential + SET deletion_job_id = $1 + FROM content_lock_deletion_task_snapshot AS snapshot + WHERE snapshot.deletion_job_id = $1 + AND credential.creator = snapshot.creator + AND credential.bundle_id = snapshot.bundle_id + AND credential.expires_at > $2", + ) + .bind(job.job_id) + .bind(admission_cutoff) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + let attached_credentials = sqlx::query( + "SELECT lookup_key, creator, bundle_id, expires_at + FROM access_credentials + WHERE deletion_job_id = $1 + ORDER BY lookup_key", + ) + .bind(job.job_id) + .fetch_all(&mut *transaction) + .await + .map_err(storage_error)?; + for credential in attached_credentials { + let credential_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO content_lock_access_drain_credentials ( + credential_id, deletion_job_id, lookup_key, creator, bundle_id, + credential_kind, issued_at, expires_at + ) VALUES ($1, $2, $3, $4, $5, 'ordinary', $6, $7)", + ) + .bind(credential_id) + .bind(job.job_id) + .bind( + credential + .try_get::, _>("lookup_key") + .map_err(storage_error)?, + ) + .bind( + credential + .try_get::("creator") + .map_err(storage_error)?, + ) + .bind( + credential + .try_get::("bundle_id") + .map_err(storage_error)?, + ) + .bind(admission_cutoff) + .bind( + credential + .try_get::("expires_at") + .map_err(storage_error)?, + ) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + } sqlx::query( "UPDATE verification_tasks AS task SET status = 'pending', started_at = NULL, claimed_by = NULL, @@ -233,7 +330,7 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { AND task.status IN ('pending', 'in_progress')", ) .bind(job.job_id) - .bind(job.deletion_started_at) + .bind(admission_cutoff) .execute(&mut *transaction) .await .map_err(storage_error)?; @@ -341,6 +438,20 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { message: "deletion phase must advance to its immediate successor".to_owned(), }); } + check_access_obligations_for_phase( + &mut transaction, + job_id, + current.phase, + next_phase, + now, + ) + .await?; + if current.phase == ContentLockDeletionPhase::DrainPayments + && next_phase == ContentLockDeletionPhase::DrainExistingCredentials + { + ensure_all_frozen_snapshots_terminal(&mut transaction, job_id).await?; + ensure_payment_drain_completed(&mut transaction, job_id).await?; + } if next_phase == ContentLockDeletionPhase::StartPaymentDrain { let has_unready_paykit_admission = sqlx::query_scalar::<_, bool>( "SELECT EXISTS ( @@ -404,6 +515,9 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { .await .map_err(storage_error)?; } + if next_phase == ContentLockDeletionPhase::DeleteContent { + revoke_read_claims(&mut transaction, job_id).await?; + } let sql = format!( "UPDATE content_lock_deletion_jobs SET phase = $2, state = 'queued', attempt_count = 0, next_attempt_at = NULL, @@ -430,6 +544,42 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { now: OffsetDateTime, failure_code: Option, ) -> Result, ApplicationError> { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let current = + load_owned_claim(&mut transaction, job_id, worker_id, claim_token, now).await?; + let Some(current) = current else { + transaction.rollback().await.map_err(storage_error)?; + return Ok(None); + }; + if failure_code.is_none() { + if current.phase != ContentLockDeletionPhase::PurgeOperationalState { + return Err(invalid_state( + "successful completion requires the final operational-state cleanup phase", + )); + } + ensure_all_frozen_snapshots_terminal(&mut transaction, job_id).await?; + ensure_payment_drain_completed(&mut transaction, job_id).await?; + ensure_no_live_access_obligations(&mut transaction, job_id, now).await?; + let issuance_incomplete: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 + FROM content_lock_deletion_task_snapshot AS snapshot + WHERE snapshot.deletion_job_id = $1 + AND snapshot.final_credential_eligible_at IS NOT NULL + AND snapshot.final_credential_issued_at IS NULL + )", + ) + .bind(job_id) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if issuance_incomplete { + return Err(invalid_state( + "successful completion cannot bypass final credential issuance", + )); + } + } + revoke_read_claims(&mut transaction, job_id).await?; let state = if failure_code.is_some() { "failed" } else { @@ -437,24 +587,21 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { }; let sql = format!( "UPDATE content_lock_deletion_jobs - SET state = $5, failure_code = $6, next_attempt_at = NULL, + SET state = $2, failure_code = $3, next_attempt_at = NULL, claimed_by = NULL, claim_token = NULL, claim_expires_at = NULL, updated_at = $4 - WHERE job_id = $1 AND state = 'running' AND claimed_by = $2 - AND claim_token = $3 AND claim_expires_at >= $4 + WHERE job_id = $1 RETURNING {ROW_COLUMNS}" ); - fetch_optional_job( - sqlx::query_as::<_, DeletionJobRow>(&sql) - .bind(job_id) - .bind(worker_id) - .bind(claim_token) - .bind(now) - .bind(state) - .bind(failure_code.map(ContentLockDeletionFailureCode::as_str)) - .fetch_optional(&self.pool) - .await - .map_err(storage_error)?, - ) + let updated = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(job_id) + .bind(state) + .bind(failure_code.map(ContentLockDeletionFailureCode::as_str)) + .bind(now) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(Some(row_to_job(updated)?)) } async fn resume_failed_job( @@ -559,6 +706,7 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { transaction.commit().await.map_err(storage_error)?; return Ok(PrepareForceDeletionResult::PublicationInProgress); } + revoke_read_claims(&mut transaction, job.job_id).await?; let sql = format!( "UPDATE content_lock_deletion_jobs SET force_requested_at = COALESCE(force_requested_at, $3), @@ -626,6 +774,190 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { } } +async fn check_access_obligations_for_phase( + transaction: &mut Transaction<'_, Postgres>, + job_id: Uuid, + current_phase: ContentLockDeletionPhase, + next_phase: ContentLockDeletionPhase, + now: OffsetDateTime, +) -> Result<(), ApplicationError> { + if current_phase == ContentLockDeletionPhase::DrainExistingCredentials + && next_phase == ContentLockDeletionPhase::IssueFinalCredentials + { + let ordinary_active: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND credential_kind = 'ordinary' + AND expires_at > $2 + )", + ) + .bind(job_id) + .bind(now) + .fetch_one(&mut **transaction) + .await + .map_err(storage_error)?; + if ordinary_active { + return Err(invalid_state( + "existing credentials must reach their original expiry before final issuance", + )); + } + } + + if current_phase == ContentLockDeletionPhase::IssueFinalCredentials + && next_phase == ContentLockDeletionPhase::DrainFinalReads + { + let has_unissued_eligible: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 + AND final_credential_eligible_at IS NOT NULL + AND final_credential_issued_at IS NULL + )", + ) + .bind(job_id) + .fetch_one(&mut **transaction) + .await + .map_err(storage_error)?; + if has_unissued_eligible { + return Err(invalid_state( + "every eligible final credential must be durably issued before final-read draining", + )); + } + } + + if current_phase == ContentLockDeletionPhase::DrainFinalReads + && next_phase == ContentLockDeletionPhase::DeleteContent + { + ensure_no_live_access_obligations(transaction, job_id, now).await?; + } + Ok(()) +} + +async fn ensure_no_live_access_obligations( + transaction: &mut Transaction<'_, Postgres>, + job_id: Uuid, + now: OffsetDateTime, +) -> Result<(), ApplicationError> { + sqlx::query( + "UPDATE content_lock_access_drain_reads AS read + SET claim_token = NULL, claim_expires_at = NULL + FROM content_lock_access_drain_credentials AS credential + WHERE read.credential_id = credential.credential_id + AND credential.deletion_job_id = $1 + AND read.claim_token IS NOT NULL + AND read.claim_expires_at <= $2", + ) + .bind(job_id) + .bind(now) + .execute(&mut **transaction) + .await + .map_err(storage_error)?; + + let has_live_obligation: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 + FROM content_lock_access_drain_credentials AS credential + WHERE credential.deletion_job_id = $1 + AND credential.credential_kind = 'ordinary' + AND credential.expires_at > $2 + ) OR EXISTS ( + SELECT 1 + FROM content_lock_access_drain_credentials AS credential + JOIN content_lock_access_drain_reads AS read + ON read.credential_id = credential.credential_id + WHERE credential.deletion_job_id = $1 + AND credential.credential_kind = 'final' + AND credential.expires_at > $2 + AND read.consumed_at IS NULL + ) OR EXISTS ( + SELECT 1 + FROM content_lock_access_drain_credentials AS credential + JOIN content_lock_access_drain_reads AS read + ON read.credential_id = credential.credential_id + WHERE credential.deletion_job_id = $1 + AND read.claim_token IS NOT NULL + AND read.claim_expires_at > $2 + )", + ) + .bind(job_id) + .bind(now) + .fetch_one(&mut **transaction) + .await + .map_err(storage_error)?; + if has_live_obligation { + return Err(invalid_state( + "credential expiry and final-read obligations must drain before destructive deletion", + )); + } + Ok(()) +} + +async fn ensure_all_frozen_snapshots_terminal( + transaction: &mut Transaction<'_, Postgres>, + job_id: Uuid, +) -> Result<(), ApplicationError> { + let resolved_statuses = sqlx::query_scalar::<_, Option>( + "SELECT resolved_status + FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 + ORDER BY verification_task_id + FOR UPDATE", + ) + .bind(job_id) + .fetch_all(&mut **transaction) + .await + .map_err(storage_error)?; + if resolved_statuses.iter().any(Option::is_none) { + return Err(invalid_state( + "every frozen deletion obligation must be terminal before credential draining", + )); + } + Ok(()) +} + +async fn ensure_payment_drain_completed( + transaction: &mut Transaction<'_, Postgres>, + job_id: Uuid, +) -> Result<(), ApplicationError> { + let aggregate: Option<(String, i64)> = sqlx::query_as( + "SELECT status, accepted_count + FROM content_lock_payment_drains + WHERE deletion_job_id = $1 + FOR UPDATE", + ) + .bind(job_id) + .fetch_optional(&mut **transaction) + .await + .map_err(storage_error)?; + if !aggregate + .is_some_and(|(status, accepted_count)| status == "completed" && accepted_count == 0) + { + return Err(invalid_state( + "payment drain aggregate must be durably completed before credential draining", + )); + } + Ok(()) +} + +async fn revoke_read_claims( + transaction: &mut Transaction<'_, Postgres>, + job_id: Uuid, +) -> Result<(), ApplicationError> { + sqlx::query( + "UPDATE content_lock_access_drain_reads AS read + SET claim_token = NULL, claim_expires_at = NULL + FROM content_lock_access_drain_credentials AS credential + WHERE read.credential_id = credential.credential_id + AND credential.deletion_job_id = $1 + AND read.claim_token IS NOT NULL", + ) + .bind(job_id) + .execute(&mut **transaction) + .await + .map_err(storage_error)?; + Ok(()) +} + async fn delete_publication_intent( pool: &PgPool, creator: &CreatorPubky, @@ -814,12 +1146,13 @@ mod tests { application::{ errors::ApplicationError, models::{ + AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, ContentLockDeletionFailureCode, ContentLockDeletionJob, ContentLockDeletionPhase, ContentLockDeletionState, PrepareForceDeletionResult, VerificationTaskRecord, VerificationTaskStatus, }, ports::{ - Clock, ContentLockDeletionRepository, EntitlementRepository, + AccessCredentialStore, Clock, ContentLockDeletionRepository, EntitlementRepository, PaymentDrainCleanupToken, PaymentDrainClient, PaymentDrainClientError, PaymentDrainRepository, PaymentDrainStatus, PaymentDrainSummary, PaymentDrainTerminalTransition, PaymentRequestState, PaymentRequestStatus, @@ -829,14 +1162,92 @@ mod tests { }, infrastructure::memory::entitlements::InMemoryEntitlementRepository, infrastructure::postgres::{ - PostgresPaymentDrainRepository, PostgresVerificationTaskClaimer, - PostgresVerificationTaskRepository, testing::TestDatabase, + PostgresAccessCredentialStore, PostgresPaymentDrainRepository, + PostgresVerificationTaskClaimer, PostgresVerificationTaskRepository, + testing::TestDatabase, }, }; const CREATOR: &str = "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy"; const NOW: time::OffsetDateTime = datetime!(2026-08-12 05:00:00 UTC); + #[tokio::test] + async fn admission_cutoff_timestamp_is_established_after_the_canonical_fence() { + let database = TestDatabase::create().await; + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let credentials = PostgresAccessCredentialStore::new(database.pool().clone()); + let lock = content_lock(); + let lock_id = lock.lock_id().unwrap(); + let bundle_id = BundleId::from_bytes([91; 16]); + let mut task = verification_task(&lock, bundle_id.clone()); + task.status = VerificationTaskStatus::Completed; + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::PaykitPayment; + tasks.insert_verification_task(task).await.unwrap(); + let lookup_key = AccessCredentialLookupKey::derive(&AccessCredential::new("pre-fence")); + let expires_at = NOW + time::Duration::hours(1); + credentials + .insert_access_credential( + &lock_id, + lookup_key.clone(), + AccessCredentialRecord { + creator: lock.creator.clone(), + bundle_id, + expires_at, + }, + ) + .await + .unwrap(); + + let mut blocker = database.pool().begin().await.unwrap(); + super::lock_proof_admission(&mut blocker, &lock.creator, &lock_id) + .await + .unwrap(); + let fence_held_at: time::OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *blocker) + .await + .unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + let inserting = tokio::spawn({ + let deletions = deletions.clone(); + let job = job.clone(); + async move { deletions.insert_job(job).await } + }); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + assert!(!inserting.is_finished()); + blocker.commit().await.unwrap(); + inserting.await.unwrap().unwrap(); + + let (cutoff, had_active, eligible_at): ( + time::OffsetDateTime, + bool, + Option, + ) = sqlx::query_as( + "SELECT job.deletion_started_at, snapshot.had_active_credential_at_cutoff, + snapshot.final_credential_eligible_at + FROM content_lock_deletion_jobs AS job + JOIN content_lock_deletion_task_snapshot AS snapshot + ON snapshot.deletion_job_id = job.job_id + WHERE job.job_id = $1", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert!(cutoff >= fence_held_at); + assert!(cutoff > expires_at); + assert!(!had_active); + assert_eq!(eligible_at, Some(cutoff)); + assert!( + !credentials + .deletion_credential_enrolled(&lookup_key) + .await + .unwrap() + ); + + database.cleanup().await; + } + #[tokio::test] async fn deletion_commit_order_is_the_authoritative_proof_admission_cutoff() { let database = TestDatabase::create().await; @@ -880,6 +1291,181 @@ mod tests { database.cleanup().await; } + #[tokio::test] + async fn credential_committed_before_deletion_is_classified_and_enrolled_at_cutoff() { + let database = TestDatabase::create().await; + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let credentials = PostgresAccessCredentialStore::new(database.pool().clone()); + let lock = content_lock(); + let guarded_path = lock.primary_resource.as_ref().unwrap().path.clone(); + let lock_id = lock.lock_id().unwrap(); + let bundle_id = BundleId::from_bytes([3; 16]); + tasks + .insert_verification_task(verification_task(&lock, bundle_id.clone())) + .await + .unwrap(); + let bearer = AccessCredential::new("cutoff-active-credential"); + let lookup_key = AccessCredentialLookupKey::derive(&bearer); + let expires_at: time::OffsetDateTime = + sqlx::query_scalar("SELECT clock_timestamp() + INTERVAL '1 hour'") + .fetch_one(database.pool()) + .await + .unwrap(); + credentials + .insert_access_credential( + &lock_id, + lookup_key.clone(), + AccessCredentialRecord { + creator: lock.creator.clone(), + bundle_id, + expires_at, + }, + ) + .await + .unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + + deletions.insert_job(job.clone()).await.unwrap(); + + let attached_job: Option = sqlx::query_scalar( + "SELECT deletion_job_id FROM access_credentials WHERE lookup_key = $1", + ) + .bind(lookup_key.as_bytes().as_slice()) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(attached_job, Some(job.job_id)); + let had_active: bool = sqlx::query_scalar( + "SELECT had_active_credential_at_cutoff + FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert!(had_active); + let enrolled_expiry: time::OffsetDateTime = sqlx::query_scalar( + "SELECT expires_at FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND lookup_key = $2 AND credential_kind = 'ordinary'", + ) + .bind(job.job_id) + .bind(lookup_key.as_bytes().as_slice()) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(enrolled_expiry, expires_at); + let final_read_allowances: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM content_lock_access_drain_reads") + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(final_read_allowances, 0); + assert!( + credentials + .deletion_credential_enrolled(&lookup_key) + .await + .unwrap() + ); + let authorization = credentials + .prepare_deletion_read( + &lookup_key, + &guarded_path, + NOW, + NOW + time::Duration::seconds(30), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(authorization.resource.path, guarded_path); + assert_eq!(authorization.claim_token, None); + assert!( + credentials + .prepare_deletion_read( + &lookup_key, + "/priv/locks.app/content/not-frozen.json", + NOW, + NOW + time::Duration::seconds(30), + ) + .await + .unwrap() + .is_none() + ); + assert!( + credentials + .deletion_credential_enrolled(&lookup_key) + .await + .unwrap() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn concurrent_deletion_and_credential_issuance_have_one_serialized_cutoff_order() { + for iteration in 1..=20_u8 { + let database = TestDatabase::create().await; + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let credentials = PostgresAccessCredentialStore::new(database.pool().clone()); + let lock = content_lock(); + let lock_id = lock.lock_id().unwrap(); + let bundle_id = BundleId::from_bytes([iteration; 16]); + tasks + .insert_verification_task(verification_task(&lock, bundle_id.clone())) + .await + .unwrap(); + let lookup_key = AccessCredentialLookupKey::derive(&AccessCredential::new(format!( + "concurrent-{iteration}" + ))); + let record = AccessCredentialRecord { + creator: lock.creator.clone(), + bundle_id, + expires_at: datetime!(2026-08-12 06:00:00 UTC), + }; + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + + let (deletion_result, credential_result) = tokio::join!( + deletions.insert_job(job.clone()), + credentials.insert_access_credential(&lock_id, lookup_key.clone(), record) + ); + deletion_result.unwrap(); + + let attached_job: Option> = sqlx::query_scalar( + "SELECT deletion_job_id FROM access_credentials WHERE lookup_key = $1", + ) + .bind(lookup_key.as_bytes().as_slice()) + .fetch_optional(database.pool()) + .await + .unwrap(); + let enrolled: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND lookup_key = $2 + )", + ) + .bind(job.job_id) + .bind(lookup_key.as_bytes().as_slice()) + .fetch_one(database.pool()) + .await + .unwrap(); + match credential_result { + Ok(()) => { + assert_eq!(attached_job, Some(Some(job.job_id))); + assert!(enrolled); + } + Err(ApplicationError::ContentLockDeletionInProgress) => { + assert_eq!(attached_job, None); + assert!(!enrolled); + } + other => panic!("unexpected concurrent credential result: {other:?}"), + } + + database.cleanup().await; + } + } + #[tokio::test] async fn concurrent_deletion_and_new_bundle_have_one_serialized_cutoff_order() { for iteration in 0..20_u8 { @@ -1238,10 +1824,15 @@ mod tests { repository.insert_job(job.clone()).await.unwrap(); let reopened = PostgresContentLockDeletionRepository::new(database.pool().clone()); - assert_eq!( - reopened.get_job(&job.creator, &job.lock_id).await.unwrap(), - Some(job.clone()) - ); + let persisted = reopened + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .unwrap(); + assert!(persisted.deletion_started_at > job.deletion_started_at); + let mut expected = job.clone(); + expected.deletion_started_at = persisted.deletion_started_at; + assert_eq!(persisted, expected); assert!(reopened.insert_job(job.clone()).await.is_err()); let mut distinct_lock = content_lock(); distinct_lock.access_policy.requested_credential_ttl_seconds = 901; @@ -1702,6 +2293,251 @@ mod tests { .unwrap() ); assert!(drains.all_obligations_terminal(job.job_id).await.unwrap()); + let final_credential_eligible_at: Option = sqlx::query_scalar( + "SELECT final_credential_eligible_at + FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 AND verification_task_id = $2", + ) + .bind(job.job_id) + .bind(task.task_id.as_uuid()) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(final_credential_eligible_at, Some(NOW)); + let issuance_deadline = NOW + time::Duration::minutes(15); + let read_deadline = issuance_deadline + time::Duration::minutes(15); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET phase = 'issue_final_credentials' + WHERE job_id = $1", + ) + .bind(job.job_id) + .execute(database.pool()) + .await + .unwrap(); + let access = PostgresAccessCredentialStore::with_final_credential_cipher( + database.pool().clone(), + crate::infrastructure::final_credentials::FinalCredentialCipher::new([8; 32]), + ); + assert!( + access + .initialize_final_access_windows( + job.job_id, + "deletion", + drain_claim.claim_token, + NOW, + issuance_deadline, + read_deadline, + ) + .await + .unwrap() + ); + assert!( + access + .initialize_final_access_windows( + job.job_id, + "deletion", + drain_claim.claim_token, + NOW + time::Duration::seconds(1), + issuance_deadline + time::Duration::hours(1), + read_deadline + time::Duration::hours(1), + ) + .await + .unwrap() + ); + let persisted_windows: ( + Option, + Option, + Option, + ) = sqlx::query_as( + "SELECT final_issuance_started_at, + final_credential_issuance_deadline, + final_read_deadline + FROM content_lock_deletion_jobs WHERE job_id = $1", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!( + persisted_windows, + (Some(NOW), Some(issuance_deadline), Some(read_deadline)) + ); + let first = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + NOW, + AccessCredential::new("first-final-bearer"), + ) + .await + .unwrap() + .unwrap(); + let replay = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + NOW + time::Duration::seconds(1), + AccessCredential::new("different-retry-candidate"), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(first.credential, replay.credential); + assert_eq!(first.expires_at, read_deadline); + assert_eq!(replay.expires_at, read_deadline); + let late_replay = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + issuance_deadline + time::Duration::seconds(1), + AccessCredential::new("candidate-after-issuance-deadline"), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(late_replay, first); + let encrypted: String = sqlx::query_scalar( + "SELECT encrypted_bearer + FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND credential_kind = 'final'", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert!(!encrypted.contains(first.credential.as_str())); + let final_rows: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND credential_kind = 'final'", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(final_rows, 1); + let read_rows: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM content_lock_access_drain_reads AS read + JOIN content_lock_access_drain_credentials AS credential + ON credential.credential_id = read.credential_id + WHERE credential.deletion_job_id = $1 AND credential.credential_kind = 'final'", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(read_rows, 1); + let final_lookup = AccessCredentialLookupKey::derive(&first.credential); + let guarded_path = "/priv/locks.app/content/post.json"; + let (first_attempt, second_attempt) = tokio::join!( + access.prepare_deletion_read( + &final_lookup, + guarded_path, + NOW, + NOW + time::Duration::seconds(30), + ), + access.prepare_deletion_read( + &final_lookup, + guarded_path, + NOW, + NOW + time::Duration::seconds(30), + ), + ); + let first_attempt = first_attempt.unwrap(); + let second_attempt = second_attempt.unwrap(); + assert_eq!( + usize::from(first_attempt.is_some()) + usize::from(second_attempt.is_some()), + 1 + ); + let first_claim = first_attempt.or(second_attempt).unwrap(); + let first_token = first_claim.claim_token.unwrap(); + assert_eq!(first_claim.resource.path, guarded_path); + assert!( + access + .prepare_deletion_read( + &final_lookup, + guarded_path, + NOW + time::Duration::seconds(1), + NOW + time::Duration::seconds(31), + ) + .await + .unwrap() + .is_none() + ); + assert!( + !access + .release_deletion_read( + &final_lookup, + guarded_path, + Uuid::new_v4(), + NOW + time::Duration::seconds(1), + ) + .await + .unwrap() + ); + assert!( + access + .release_deletion_read( + &final_lookup, + guarded_path, + first_token, + NOW + time::Duration::seconds(1), + ) + .await + .unwrap() + ); + let second_claim = access + .prepare_deletion_read( + &final_lookup, + guarded_path, + NOW + time::Duration::seconds(2), + NOW + time::Duration::seconds(32), + ) + .await + .unwrap() + .unwrap(); + let second_token = second_claim.claim_token.unwrap(); + assert_ne!(second_token, first_token); + assert!( + !access + .consume_deletion_read( + &final_lookup, + guarded_path, + first_token, + NOW + time::Duration::seconds(3), + ) + .await + .unwrap() + ); + assert!( + access + .consume_deletion_read( + &final_lookup, + guarded_path, + second_token, + NOW + time::Duration::seconds(3), + ) + .await + .unwrap() + ); + assert!( + access + .prepare_deletion_read( + &final_lookup, + guarded_path, + NOW + time::Duration::seconds(4), + NOW + time::Duration::seconds(34), + ) + .await + .unwrap() + .is_none() + ); + assert!( + access + .deletion_credential_enrolled(&final_lookup) + .await + .unwrap() + ); let retained_marker: Option = sqlx::query_scalar( "SELECT entitlement_publication_claim_token FROM verification_tasks WHERE task_id = $1", ) @@ -1917,59 +2753,872 @@ mod tests { status: Mutex, } - #[async_trait] - impl PaymentDrainClient for MutablePaymentDrainClient { - async fn start_payment_drain( - &self, - _lock_resource: &PubkyLockResource, - ) -> Result { - Ok(self.summary.clone()) - } + #[tokio::test] + async fn concurrent_final_issuers_replay_one_winner() { + let database = TestDatabase::create().await; + let (job, task, access) = eligible_final_credential_fixture(&database).await; - async fn lookup_payment_drain( - &self, - _lock_resource: &PubkyLockResource, - ) -> Result, PaymentDrainClientError> { - Ok(Some(self.summary.clone())) - } + let (first, second) = tokio::join!( + access.issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + NOW, + AccessCredential::new("concurrent-final-one"), + ), + access.issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + NOW, + AccessCredential::new("concurrent-final-two"), + ), + ); + let first = first.unwrap().unwrap(); + let second = second.unwrap().unwrap(); + assert_eq!(first.credential, second.credential); + let final_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND credential_kind = 'final'", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(final_count, 1); - async fn payment_request_status( - &self, - _creator: &CreatorPubky, - _bundle_id: &BundleId, - ) -> Result, PaymentDrainClientError> { - Ok(Some(*self.status.lock().unwrap())) - } + database.cleanup().await; } #[tokio::test] - async fn read_rejects_corrupt_frozen_manifest_identity() { + async fn force_revokes_live_final_read_claim() { + let database = TestDatabase::create().await; + let (job, task, access) = eligible_final_credential_fixture(&database).await; + let issued = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + NOW, + AccessCredential::new("final-before-force"), + ) + .await + .unwrap() + .unwrap(); + let lookup = AccessCredentialLookupKey::derive(&issued.credential); + let path = job + .frozen_content_lock + .primary_resource + .as_ref() + .unwrap() + .path + .clone(); + let prepared = access + .prepare_deletion_read(&lookup, &path, NOW, NOW + time::Duration::seconds(30)) + .await + .unwrap() + .unwrap(); + let read_token = prepared.claim_token.unwrap(); + + let force = PostgresContentLockDeletionRepository::new(database.pool().clone()); + assert!(matches!( + force + .prepare_force_deletion( + &job.creator, + &job.lock_id, + NOW + time::Duration::seconds(1) + ) + .await + .unwrap(), + PrepareForceDeletionResult::Active(_) + )); + assert!( + !access + .consume_deletion_read( + &lookup, + &path, + read_token, + NOW + time::Duration::seconds(2), + ) + .await + .unwrap() + ); + assert!( + access + .prepare_deletion_read( + &lookup, + &path, + NOW + time::Duration::seconds(2), + NOW + time::Duration::seconds(30), + ) + .await + .unwrap() + .is_none() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn final_issuance_waiting_on_snapshot_observes_force_winner() { + let database = TestDatabase::create().await; + let (job, task, access) = eligible_final_credential_fixture(&database).await; + let mut blocker = database.pool().begin().await.unwrap(); + sqlx::query( + "SELECT job_id FROM content_lock_deletion_jobs + WHERE job_id = $1 FOR UPDATE", + ) + .bind(job.job_id) + .fetch_one(&mut *blocker) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET force_requested_at = $2 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(NOW) + .execute(&mut *blocker) + .await + .unwrap(); + + let issuing_access = access.clone(); + let issuing_creator = job.creator.clone(); + let issuing_bundle = task.submitted_proof_bundle.bundle_id.clone(); + let issuing = tokio::spawn(async move { + issuing_access + .issue_or_replay_final_credential( + &issuing_creator, + &issuing_bundle, + NOW, + AccessCredential::new("must-not-escape-force"), + ) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + assert!(!issuing.is_finished()); + + blocker.commit().await.unwrap(); + assert!(issuing.await.unwrap().unwrap().is_none()); + + database.cleanup().await; + } + + #[tokio::test] + async fn phase_advancement_and_successful_finish_cannot_bypass_access_obligations() { let database = TestDatabase::create().await; + let (job, task, access) = eligible_final_credential_fixture(&database).await; let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); - let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); - repository.insert_job(job.clone()).await.unwrap(); + let claim_token = Uuid::new_v4(); sqlx::query( "UPDATE content_lock_deletion_jobs - SET frozen_content_lock = jsonb_set( - frozen_content_lock, - '{access_policy,requested_credential_ttl_seconds}', - '901'::jsonb - ) + SET state = 'running', claimed_by = 'worker', claim_token = $2, + claim_expires_at = $3 WHERE job_id = $1", ) .bind(job.job_id) + .bind(claim_token) + .bind(NOW + time::Duration::hours(1)) .execute(database.pool()) .await .unwrap(); + assert!(matches!( + repository + .advance_phase( + job.job_id, + "worker", + claim_token, + NOW, + ContentLockDeletionPhase::DrainFinalReads, + ) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + assert!(matches!( + repository + .finish(job.job_id, "worker", claim_token, NOW, None) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + + let issued = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + NOW, + AccessCredential::new("phase-obligation-final"), + ) + .await + .unwrap() + .unwrap(); assert!( repository - .get_job(&job.creator, &job.lock_id) + .advance_phase( + job.job_id, + "worker", + claim_token, + NOW, + ContentLockDeletionPhase::DrainFinalReads, + ) .await - .is_err() + .unwrap() + .is_some() ); - - database.cleanup().await; + let drain_claim = repository + .claim_next("worker", NOW, NOW + time::Duration::hours(1)) + .await + .unwrap() + .unwrap(); + assert!(matches!( + repository + .advance_phase( + job.job_id, + "worker", + drain_claim.claim_token, + NOW, + ContentLockDeletionPhase::DeleteContent, + ) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + let lookup = AccessCredentialLookupKey::derive(&issued.credential); + let path = job + .frozen_content_lock + .primary_resource + .as_ref() + .unwrap() + .path + .clone(); + let read = access + .prepare_deletion_read(&lookup, &path, NOW, NOW + time::Duration::seconds(30)) + .await + .unwrap() + .unwrap(); + assert!( + access + .consume_deletion_read(&lookup, &path, read.claim_token.unwrap(), NOW) + .await + .unwrap() + ); + assert!( + repository + .advance_phase( + job.job_id, + "worker", + drain_claim.claim_token, + NOW, + ContentLockDeletionPhase::DeleteContent, + ) + .await + .unwrap() + .is_some() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn drain_payments_phase_requires_durable_completed_aggregate() { + let database = TestDatabase::create().await; + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let lock = content_lock(); + let mut task = verification_task(&lock, BundleId::from_bytes([61; 16])); + task.status = VerificationTaskStatus::Completed; + task.started_at = Some(NOW); + task.completed_at = Some(NOW); + tasks.insert_verification_task(task).await.unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository.insert_job(job.clone()).await.unwrap(); + sqlx::query( + "INSERT INTO content_lock_payment_drains ( + deletion_job_id, status, accepted_count, terminal_count, + cancellation_enqueued_count, cleanup_token, created_at, updated_at + ) VALUES ($1, 'active', 1, 0, 0, $2, $3, $3)", + ) + .bind(job.job_id) + .bind("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + let claim_token = Uuid::new_v4(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET phase = 'drain_payments', state = 'running', claimed_by = 'worker', + claim_token = $2, claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(claim_token) + .bind(NOW + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + + assert!(matches!( + repository + .advance_phase( + job.job_id, + "worker", + claim_token, + NOW, + ContentLockDeletionPhase::DrainExistingCredentials, + ) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + + database.cleanup().await; + } + + #[tokio::test] + async fn drain_payments_phase_requires_every_frozen_snapshot_terminal() { + let database = TestDatabase::create().await; + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let lock = content_lock(); + let mut paykit = verification_task(&lock, BundleId::from_bytes([62; 16])); + paykit.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::PaykitPayment; + let local = verification_task(&lock, BundleId::from_bytes([63; 16])); + tasks + .insert_verification_task(paykit.clone()) + .await + .unwrap(); + tasks.insert_verification_task(local).await.unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository.insert_job(job.clone()).await.unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET resolved_status = 'expired', resolved_at = $3 + WHERE deletion_job_id = $1 AND verification_task_id = $2", + ) + .bind(job.job_id) + .bind(paykit.task_id.as_uuid()) + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO content_lock_payment_drains ( + deletion_job_id, status, accepted_count, terminal_count, + cancellation_enqueued_count, cleanup_token, created_at, updated_at + ) VALUES ($1, 'completed', 0, 1, 0, $2, $3, $3)", + ) + .bind(job.job_id) + .bind("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB") + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + let claim_token = Uuid::new_v4(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET phase = 'drain_payments', state = 'running', claimed_by = 'worker', + claim_token = $2, claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(claim_token) + .bind(NOW + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + + assert!(matches!( + repository + .advance_phase( + job.job_id, + "worker", + claim_token, + NOW, + ContentLockDeletionPhase::DrainExistingCredentials, + ) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + + database.cleanup().await; + } + + #[tokio::test] + async fn expired_final_read_claim_does_not_wedge_destructive_phase_and_cannot_consume() { + let database = TestDatabase::create().await; + let (job, task, access) = eligible_final_credential_fixture(&database).await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let issued = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + NOW, + AccessCredential::new("expires-with-final-read-window"), + ) + .await + .unwrap() + .unwrap(); + let lookup = AccessCredentialLookupKey::derive(&issued.credential); + let path = job + .frozen_content_lock + .primary_resource + .as_ref() + .unwrap() + .path + .clone(); + let issue_claim = Uuid::new_v4(); + let read_deadline = NOW + time::Duration::minutes(30); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET state = 'running', claimed_by = 'worker', claim_token = $2, + claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(issue_claim) + .bind(read_deadline + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + repository + .advance_phase( + job.job_id, + "worker", + issue_claim, + NOW, + ContentLockDeletionPhase::DrainFinalReads, + ) + .await + .unwrap() + .unwrap(); + let drain_claim = repository + .claim_next( + "worker", + read_deadline - time::Duration::seconds(10), + read_deadline + time::Duration::minutes(5), + ) + .await + .unwrap() + .unwrap(); + let read = access + .prepare_deletion_read( + &lookup, + &path, + read_deadline - time::Duration::seconds(10), + read_deadline + time::Duration::minutes(1), + ) + .await + .unwrap() + .unwrap(); + let stale_read_token = read.claim_token.unwrap(); + + assert!( + repository + .advance_phase( + job.job_id, + "worker", + drain_claim.claim_token, + read_deadline, + ContentLockDeletionPhase::DeleteContent, + ) + .await + .unwrap() + .is_some() + ); + assert!( + !access + .consume_deletion_read(&lookup, &path, stale_read_token, read_deadline,) + .await + .unwrap() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn unissued_eligible_snapshot_blocks_final_read_transition_after_issuance_deadline() { + let database = TestDatabase::create().await; + let (job, _task, _access) = eligible_final_credential_fixture(&database).await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let claim_token = Uuid::new_v4(); + let after_deadline = NOW + time::Duration::minutes(16); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET state = 'running', claimed_by = 'worker', claim_token = $2, + claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(claim_token) + .bind(after_deadline + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + + assert!(matches!( + repository + .advance_phase( + job.job_id, + "worker", + claim_token, + after_deadline, + ContentLockDeletionPhase::DrainFinalReads, + ) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + + database.cleanup().await; + } + + #[tokio::test] + async fn successful_finish_rechecks_paykit_and_non_paykit_frozen_obligations() { + let database = TestDatabase::create().await; + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let lock = content_lock(); + let mut paykit = verification_task(&lock, BundleId::from_bytes([64; 16])); + paykit.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::PaykitPayment; + let local = verification_task(&lock, BundleId::from_bytes([65; 16])); + tasks + .insert_verification_task(paykit.clone()) + .await + .unwrap(); + tasks.insert_verification_task(local).await.unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository.insert_job(job.clone()).await.unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET resolved_status = 'expired', resolved_at = $3 + WHERE deletion_job_id = $1 AND verification_task_id = $2", + ) + .bind(job.job_id) + .bind(paykit.task_id.as_uuid()) + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO content_lock_payment_drains ( + deletion_job_id, status, accepted_count, terminal_count, + cancellation_enqueued_count, cleanup_token, created_at, updated_at + ) VALUES ($1, 'completed', 0, 1, 0, $2, $3, $3)", + ) + .bind(job.job_id) + .bind("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC") + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + let claim_token = Uuid::new_v4(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET phase = 'purge_operational_state', state = 'running', claimed_by = 'worker', + claim_token = $2, claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(claim_token) + .bind(NOW + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + + assert!(matches!( + repository + .finish(job.job_id, "worker", claim_token, NOW, None) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET resolved_status = 'failed', resolved_at = $2 + WHERE deletion_job_id = $1 AND resolved_status IS NULL", + ) + .bind(job.job_id) + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_payment_drains + SET status = 'active', accepted_count = 1, updated_at = $2 + WHERE deletion_job_id = $1", + ) + .bind(job.job_id) + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + assert!(matches!( + repository + .finish(job.job_id, "worker", claim_token, NOW, None) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + sqlx::query( + "UPDATE content_lock_payment_drains + SET status = 'completed', accepted_count = 0, updated_at = $2 + WHERE deletion_job_id = $1", + ) + .bind(job.job_id) + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + repository + .finish(job.job_id, "worker", claim_token, NOW, None) + .await + .unwrap(); + + database.cleanup().await; + } + + #[tokio::test] + async fn successful_finish_requires_exact_final_cleanup_phase() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + let claimed = repository + .claim_next("worker", NOW, NOW + time::Duration::minutes(5)) + .await + .unwrap() + .unwrap(); + + assert!(matches!( + repository + .finish(job.job_id, "worker", claimed.claim_token, NOW, None) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + + database.cleanup().await; + } + + #[tokio::test] + async fn issuance_deadline_is_half_open_and_transition_preserves_only_exact_replay() { + let database = TestDatabase::create().await; + let (job, task, access) = eligible_final_credential_fixture(&database).await; + let issuance_deadline = NOW + time::Duration::minutes(15); + + assert!( + access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + issuance_deadline, + AccessCredential::new("must-not-insert-at-deadline"), + ) + .await + .unwrap() + .is_none() + ); + let final_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND credential_kind = 'final'", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(final_count, 0); + + let issued = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + issuance_deadline - time::Duration::seconds(1), + AccessCredential::new("persisted-before-transition"), + ) + .await + .unwrap() + .unwrap(); + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let claim_token = Uuid::new_v4(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET state = 'running', claimed_by = 'worker', claim_token = $2, + claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(claim_token) + .bind(issuance_deadline + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + repository + .advance_phase( + job.job_id, + "worker", + claim_token, + issuance_deadline, + ContentLockDeletionPhase::DrainFinalReads, + ) + .await + .unwrap() + .unwrap(); + + let replay = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + issuance_deadline + time::Duration::seconds(1), + AccessCredential::new("must-not-replace-persisted-winner"), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(replay, issued); + + database.cleanup().await; + } + + #[tokio::test] + async fn final_read_claim_lease_is_fixed_to_thirty_seconds_at_storage_boundary() { + let database = TestDatabase::create().await; + let (job, task, access) = eligible_final_credential_fixture(&database).await; + let issued = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + NOW, + AccessCredential::new("fixed-storage-lease"), + ) + .await + .unwrap() + .unwrap(); + let lookup = AccessCredentialLookupKey::derive(&issued.credential); + let path = job + .frozen_content_lock + .primary_resource + .as_ref() + .unwrap() + .path + .clone(); + + access + .prepare_deletion_read(&lookup, &path, NOW, NOW + time::Duration::seconds(1)) + .await + .unwrap() + .unwrap(); + let stored_expiry: time::OffsetDateTime = sqlx::query_scalar( + "SELECT read.claim_expires_at + FROM content_lock_access_drain_reads AS read + JOIN content_lock_access_drain_credentials AS credential + ON credential.credential_id = read.credential_id + WHERE credential.lookup_key = $1 AND read.guarded_path = $2", + ) + .bind(lookup.as_bytes().as_slice()) + .bind(&path) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(stored_expiry, NOW + time::Duration::seconds(30)); + + database.cleanup().await; + } + + #[async_trait] + impl PaymentDrainClient for MutablePaymentDrainClient { + async fn start_payment_drain( + &self, + _lock_resource: &PubkyLockResource, + ) -> Result { + Ok(self.summary.clone()) + } + + async fn lookup_payment_drain( + &self, + _lock_resource: &PubkyLockResource, + ) -> Result, PaymentDrainClientError> { + Ok(Some(self.summary.clone())) + } + + async fn payment_request_status( + &self, + _creator: &CreatorPubky, + _bundle_id: &BundleId, + ) -> Result, PaymentDrainClientError> { + Ok(Some(*self.status.lock().unwrap())) + } + } + + #[tokio::test] + async fn read_rejects_corrupt_frozen_manifest_identity() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET frozen_content_lock = jsonb_set( + frozen_content_lock, + '{access_policy,requested_credential_ttl_seconds}', + '901'::jsonb + ) + WHERE job_id = $1", + ) + .bind(job.job_id) + .execute(database.pool()) + .await + .unwrap(); + + assert!( + repository + .get_job(&job.creator, &job.lock_id) + .await + .is_err() + ); + + database.cleanup().await; + } + + async fn eligible_final_credential_fixture( + database: &TestDatabase, + ) -> ( + ContentLockDeletionJob, + VerificationTaskRecord, + PostgresAccessCredentialStore, + ) { + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let lock = content_lock(); + let mut task = verification_task(&lock, BundleId::from_bytes([42; 16])); + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::PaykitPayment; + tasks.insert_verification_task(task.clone()).await.unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + PostgresContentLockDeletionRepository::new(database.pool().clone()) + .insert_job(job.clone()) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET resolved_status = 'completed', resolved_at = $2, + final_credential_eligible_at = $2 + WHERE deletion_job_id = $1", + ) + .bind(job.job_id) + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET phase = 'issue_final_credentials', final_issuance_started_at = $2, + final_credential_issuance_deadline = $3, final_read_deadline = $4 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(NOW) + .bind(NOW + time::Duration::minutes(15)) + .bind(NOW + time::Duration::minutes(30)) + .execute(database.pool()) + .await + .unwrap(); + let access = PostgresAccessCredentialStore::with_final_credential_cipher( + database.pool().clone(), + crate::infrastructure::final_credentials::FinalCredentialCipher::new([9; 32]), + ); + (job, task, access) } fn verification_task(lock: &ContentLock, bundle_id: BundleId) -> VerificationTaskRecord { diff --git a/locks-service/src/infrastructure/postgres/migrations.rs b/locks-service/src/infrastructure/postgres/migrations.rs index 5c7e599..b7983e6 100644 --- a/locks-service/src/infrastructure/postgres/migrations.rs +++ b/locks-service/src/infrastructure/postgres/migrations.rs @@ -54,6 +54,8 @@ mod tests { assert_table_exists(&mut connection, "content_lock_force_deletion_receipts").await; assert_table_exists(&mut connection, "content_lock_publication_intents").await; assert_table_exists(&mut connection, "content_lock_deletion_task_snapshot").await; + assert_table_exists(&mut connection, "content_lock_access_drain_credentials").await; + assert_table_exists(&mut connection, "content_lock_access_drain_reads").await; assert_table_exists(&mut connection, "paykit_task_admissions").await; assert_column_exists( &mut connection, @@ -77,6 +79,37 @@ mod tests { assert_column_exists(&mut connection, "verification_tasks", "bundle_id").await; assert_column_exists(&mut connection, "verification_tasks", "next_attempt_at").await; assert_column_exists(&mut connection, "verification_tasks", "claim_token").await; + assert_column_exists( + &mut connection, + "content_lock_deletion_jobs", + "final_issuance_started_at", + ) + .await; + assert_column_exists( + &mut connection, + "content_lock_deletion_jobs", + "final_credential_issuance_deadline", + ) + .await; + assert_column_exists( + &mut connection, + "content_lock_deletion_jobs", + "final_read_deadline", + ) + .await; + assert_column_exists( + &mut connection, + "content_lock_deletion_task_snapshot", + "had_active_credential_at_cutoff", + ) + .await; + assert_column_exists( + &mut connection, + "content_lock_deletion_task_snapshot", + "final_credential_eligible_at", + ) + .await; + assert_column_exists(&mut connection, "access_credentials", "deletion_job_id").await; assert_index_exists( &mut connection, "verification_tasks", @@ -134,6 +167,74 @@ mod tests { database.cleanup().await; } + #[tokio::test] + async fn migration_0016_rejects_every_preexisting_resumable_deletion_state() { + for (state, failure_code) in [ + ("queued", None), + ("running", None), + ("failed", Some("retry_exhausted")), + ] { + let database = TestDatabase::create().await; + sqlx::raw_sql( + "DROP TABLE content_lock_access_drain_reads; + DROP TABLE content_lock_access_drain_credentials; + ALTER TABLE access_credentials DROP COLUMN deletion_job_id; + ALTER TABLE content_lock_deletion_task_snapshot + DROP CONSTRAINT content_lock_deletion_task_snapshot_final_issuance_valid, + DROP CONSTRAINT content_lock_deletion_task_snapshot_final_eligibility_valid, + DROP COLUMN final_credential_issued_at, + DROP COLUMN final_credential_eligible_at, + DROP COLUMN had_active_credential_at_cutoff; + ALTER TABLE content_lock_deletion_jobs + DROP CONSTRAINT content_lock_deletion_jobs_final_window_shape, + DROP COLUMN final_read_deadline, + DROP COLUMN final_credential_issuance_deadline, + DROP COLUMN final_issuance_started_at;", + ) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO content_lock_deletion_jobs + (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, + state, phase, failure_code, claimed_by, claim_token, claim_expires_at) + VALUES ( + $1, 'creator', 'lock', '{}'::jsonb, NOW(), + $2, 'withdraw', $3, + CASE WHEN $2 = 'running' THEN 'worker' ELSE NULL END, + CASE WHEN $2 = 'running' THEN $4::uuid ELSE NULL END, + CASE WHEN $2 = 'running' THEN NOW() + INTERVAL '1 minute' ELSE NULL END + )", + ) + .bind(uuid::Uuid::new_v4()) + .bind(state) + .bind(failure_code) + .bind(uuid::Uuid::new_v4()) + .execute(database.pool()) + .await + .unwrap(); + let migration = super::MIGRATOR + .iter() + .find(|migration| migration.version == 16) + .expect("migration 0016 exists"); + + let error = sqlx::raw_sql(migration.sql.as_ref()) + .execute(database.pool()) + .await + .expect_err( + "0016 must fail closed instead of misclassifying a resumable Task 7 job", + ); + assert!( + error + .to_string() + .contains("drain or explicitly reset pre-0016 deletion jobs"), + "unexpected migration error for {state}: {error}" + ); + + database.cleanup().await; + } + } + async fn assert_table_exists( connection: &mut sqlx::pool::PoolConnection, table_name: &str, diff --git a/locks-service/src/infrastructure/postgres/payment_drains.rs b/locks-service/src/infrastructure/postgres/payment_drains.rs index a290b7d..1649437 100644 --- a/locks-service/src/infrastructure/postgres/payment_drains.rs +++ b/locks-service/src/infrastructure/postgres/payment_drains.rs @@ -45,6 +45,22 @@ struct DrainRow { cleanup_token: String, } +#[derive(FromRow)] +struct DeletionOwnershipRow { + state: String, + phase: String, + force_requested_at: Option, + claimed_by: Option, + claim_token: Option, + claim_expires_at: Option, +} + +#[derive(FromRow)] +struct ObligationFenceRow { + paykit_admission_required: Option, + resolved_status: Option, +} + #[async_trait] impl PaymentDrainRepository for PostgresPaymentDrainRepository { async fn store_payment_drain( @@ -57,22 +73,14 @@ impl PaymentDrainRepository for PostgresPaymentDrainRepository { ) -> Result { let counts = summary_counts(summary)?; let mut transaction = self.pool.begin().await.map_err(storage_error)?; - let owns_claim: bool = sqlx::query_scalar( - "SELECT EXISTS ( - SELECT 1 FROM content_lock_deletion_jobs - WHERE job_id = $1 AND state = 'running' AND claimed_by = $2 - AND claim_token = $3 AND claim_expires_at >= $4 - AND phase = 'start_payment_drain' AND force_requested_at IS NULL - )", - ) - .bind(deletion_job_id) - .bind(worker_id) - .bind(claim_token) - .bind(now) - .fetch_one(&mut *transaction) - .await - .map_err(storage_error)?; - if !owns_claim { + let ownership = lock_deletion_ownership(&mut transaction, deletion_job_id).await?; + if !owns_live_drain_claim( + ownership.as_ref(), + worker_id, + claim_token, + now, + "start_payment_drain", + ) { transaction.rollback().await.map_err(storage_error)?; return Ok(false); } @@ -154,35 +162,41 @@ impl PaymentDrainRepository for PostgresPaymentDrainRepository { summary: &PaymentDrainSummary, ) -> Result { let counts = summary_counts(summary)?; + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let ownership = lock_deletion_ownership(&mut transaction, deletion_job_id).await?; + if !owns_live_drain_claim( + ownership.as_ref(), + worker_id, + claim_token, + now, + "drain_payments", + ) { + transaction.rollback().await.map_err(storage_error)?; + return Ok(false); + } let updated = sqlx::query( - "UPDATE content_lock_payment_drains AS drain - SET status = $5, accepted_count = $6, terminal_count = $7, updated_at = $4 - FROM content_lock_deletion_jobs AS deletion - WHERE deletion.job_id = $1 AND deletion.state = 'running' - AND deletion.claimed_by = $2 AND deletion.claim_token = $3 - AND deletion.claim_expires_at >= $4 AND deletion.phase = 'drain_payments' - AND deletion.force_requested_at IS NULL - AND drain.deletion_job_id = deletion.job_id - AND drain.cleanup_token = $8 - AND drain.cancellation_enqueued_count = $9 - AND drain.accepted_count >= $6 - AND drain.terminal_count <= $7 - AND drain.accepted_count - $6 = $7 - drain.terminal_count - AND NOT (drain.status = 'completed' AND $5 <> 'completed') - AND (($5 = 'completed' AND $6 = 0) OR ($5 = 'active' AND $6 > 0))", + "UPDATE content_lock_payment_drains + SET status = $3, accepted_count = $4, terminal_count = $5, updated_at = $2 + WHERE deletion_job_id = $1 + AND cleanup_token = $6 + AND cancellation_enqueued_count = $7 + AND accepted_count >= $4 + AND terminal_count <= $5 + AND accepted_count - $4 = $5 - terminal_count + AND NOT (status = 'completed' AND $3 <> 'completed') + AND (($3 = 'completed' AND $4 = 0) OR ($3 = 'active' AND $4 > 0))", ) .bind(deletion_job_id) - .bind(worker_id) - .bind(claim_token) .bind(now) .bind(drain_status_to_database(summary.status)) .bind(counts.0) .bind(counts.1) .bind(summary.cleanup_token.as_str()) .bind(counts.2) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; Ok(updated.rows_affected() == 1) } @@ -293,33 +307,54 @@ impl PaymentDrainRepository for PostgresPaymentDrainRepository { }); } let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let ownership = lock_deletion_ownership(&mut transaction, deletion_job_id).await?; + if !owns_live_drain_claim( + ownership.as_ref(), + worker_id, + claim_token, + now, + "drain_payments", + ) { + transaction.rollback().await.map_err(storage_error)?; + return Ok(false); + } + let snapshot = sqlx::query_as::<_, ObligationFenceRow>( + "SELECT paykit_admission_required, resolved_status + FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 AND verification_task_id = $2::uuid + FOR UPDATE", + ) + .bind(deletion_job_id) + .bind(task_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + if !matches!( + snapshot, + Some(ObligationFenceRow { + paykit_admission_required: Some(true), + resolved_status: None, + }) + ) { + transaction.rollback().await.map_err(storage_error)?; + return Ok(false); + } let updated = sqlx::query( - "UPDATE verification_tasks AS task - SET status = $6, started_at = COALESCE(started_at, $5), completed_at = $5, + "UPDATE verification_tasks + SET status = $3, started_at = COALESCE(started_at, $2), completed_at = $2, failure_message = NULL, claimed_by = NULL, claim_token = NULL, claim_expires_at = NULL, next_attempt_at = NULL, last_attempt_error = NULL, entitlement_publication_claim_token = NULL, - updated_at = $5 - FROM content_lock_deletion_jobs AS deletion, - content_lock_deletion_task_snapshot AS snapshot - WHERE deletion.job_id = $1 AND deletion.state = 'running' - AND deletion.claimed_by = $2 AND deletion.claim_token = $3 - AND deletion.claim_expires_at >= $5 AND deletion.phase = 'drain_payments' - AND deletion.force_requested_at IS NULL - AND snapshot.deletion_job_id = deletion.job_id - AND snapshot.verification_task_id = task.task_id - AND snapshot.paykit_admission_required = TRUE - AND snapshot.resolved_status IS NULL - AND task.task_id = $4::uuid - AND task.status IN ('pending', 'in_progress') - AND task.entitlement_publication_claim_token IS NOT DISTINCT FROM $7", + updated_at = $2 + WHERE task_id = $1::uuid + AND deletion_job_id = $4 + AND status IN ('pending', 'in_progress') + AND entitlement_publication_claim_token IS NOT DISTINCT FROM $5", ) - .bind(deletion_job_id) - .bind(worker_id) - .bind(claim_token) .bind(task_id.to_string()) .bind(now) .bind(status_to_database(status)) + .bind(deletion_job_id) .bind(entitlement_publication_token) .execute(&mut *transaction) .await @@ -330,7 +365,14 @@ impl PaymentDrainRepository for PostgresPaymentDrainRepository { } let resolved = sqlx::query( "UPDATE content_lock_deletion_task_snapshot - SET resolved_status = $3, resolved_at = $4 + SET resolved_status = $3, resolved_at = $4, + final_credential_eligible_at = CASE + WHEN $3 = 'completed' + AND paykit_admission_required + AND NOT had_active_credential_at_cutoff + THEN $4 + ELSE NULL + END WHERE deletion_job_id = $1 AND verification_task_id = $2::uuid AND resolved_status IS NULL", ) @@ -370,6 +412,41 @@ impl PaymentDrainRepository for PostgresPaymentDrainRepository { } } +async fn lock_deletion_ownership( + transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, + deletion_job_id: Uuid, +) -> Result, ApplicationError> { + sqlx::query_as::<_, DeletionOwnershipRow>( + "SELECT state, phase, force_requested_at, claimed_by, claim_token, claim_expires_at + FROM content_lock_deletion_jobs + WHERE job_id = $1 + FOR UPDATE", + ) + .bind(deletion_job_id) + .fetch_optional(&mut **transaction) + .await + .map_err(storage_error) +} + +fn owns_live_drain_claim( + ownership: Option<&DeletionOwnershipRow>, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + phase: &str, +) -> bool { + ownership.is_some_and(|ownership| { + ownership.state == "running" + && ownership.phase == phase + && ownership.force_requested_at.is_none() + && ownership.claimed_by.as_deref() == Some(worker_id) + && ownership.claim_token == Some(claim_token) + && ownership + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at >= now) + }) +} + fn row_to_obligation(row: ObligationRow) -> Result { Ok(PaymentDrainObligation { task_id: TaskId::from_str(&row.task_id.to_string()).map_err(storage_display)?, @@ -471,13 +548,18 @@ fn storage_display(error: impl std::fmt::Display) -> ApplicationError { #[cfg(test)] mod tests { + use std::str::FromStr; + use base64::Engine; use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use locks_core::ids::TaskId; use time::OffsetDateTime; use uuid::Uuid; + use crate::application::models::VerificationTaskStatus; use crate::application::ports::{ PaymentDrainCleanupToken, PaymentDrainRepository, PaymentDrainStatus, PaymentDrainSummary, + PaymentDrainTerminalTransition, }; use crate::infrastructure::postgres::testing::TestDatabase; @@ -632,4 +714,380 @@ mod tests { database.cleanup().await; } + + #[tokio::test] + async fn force_first_fences_stale_initial_payment_drain_store() { + let database = TestDatabase::create().await; + let repository = PostgresPaymentDrainRepository::new(database.pool().clone()); + let job_id = Uuid::new_v4(); + let stale_claim_token = Uuid::new_v4(); + let now = OffsetDateTime::now_utc(); + insert_start_drain_job(database.pool(), job_id, stale_claim_token, now).await; + + let mut force = database.pool().begin().await.unwrap(); + sqlx::query("SELECT job_id FROM content_lock_deletion_jobs WHERE job_id = $1 FOR UPDATE") + .bind(job_id) + .fetch_one(&mut *force) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET force_requested_at = $2, state = 'queued', claimed_by = NULL, + claim_token = NULL, claim_expires_at = NULL + WHERE job_id = $1", + ) + .bind(job_id) + .bind(now) + .execute(&mut *force) + .await + .unwrap(); + + let summary = PaymentDrainSummary { + status: PaymentDrainStatus::Active, + accepted_count: 1, + terminal_count: 0, + cancellation_enqueued_count: 0, + cleanup_token: PaymentDrainCleanupToken::parse(&URL_SAFE_NO_PAD.encode([10_u8; 32])) + .unwrap(), + }; + let stale = tokio::spawn(async move { + repository + .store_payment_drain(job_id, "worker", stale_claim_token, now, &summary) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!( + !stale.is_finished(), + "initial drain persistence must wait for the deletion-job ownership row" + ); + + force.commit().await.unwrap(); + assert!(!stale.await.unwrap().unwrap()); + assert_eq!( + PostgresPaymentDrainRepository::new(database.pool().clone()) + .get_payment_drain(job_id) + .await + .unwrap(), + None + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn reclaim_first_fences_stale_initial_payment_drain_store() { + let database = TestDatabase::create().await; + let repository = PostgresPaymentDrainRepository::new(database.pool().clone()); + let job_id = Uuid::new_v4(); + let stale_claim_token = Uuid::new_v4(); + let replacement_claim_token = Uuid::new_v4(); + let now = OffsetDateTime::now_utc(); + insert_start_drain_job(database.pool(), job_id, stale_claim_token, now).await; + + let mut reclaim = database.pool().begin().await.unwrap(); + sqlx::query("SELECT job_id FROM content_lock_deletion_jobs WHERE job_id = $1 FOR UPDATE") + .bind(job_id) + .fetch_one(&mut *reclaim) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET claimed_by = 'replacement-worker', claim_token = $2, + claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job_id) + .bind(replacement_claim_token) + .bind(now + time::Duration::minutes(10)) + .execute(&mut *reclaim) + .await + .unwrap(); + + let summary = PaymentDrainSummary { + status: PaymentDrainStatus::Active, + accepted_count: 1, + terminal_count: 0, + cancellation_enqueued_count: 0, + cleanup_token: PaymentDrainCleanupToken::parse(&URL_SAFE_NO_PAD.encode([11_u8; 32])) + .unwrap(), + }; + let stale = tokio::spawn(async move { + repository + .store_payment_drain(job_id, "worker", stale_claim_token, now, &summary) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!( + !stale.is_finished(), + "initial drain persistence must wait for the deletion-job ownership row" + ); + + reclaim.commit().await.unwrap(); + assert!(!stale.await.unwrap().unwrap()); + assert_eq!( + PostgresPaymentDrainRepository::new(database.pool().clone()) + .get_payment_drain(job_id) + .await + .unwrap(), + None + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn concurrent_force_winner_fences_stale_payment_drain_reconciliation() { + let database = TestDatabase::create().await; + let repository = PostgresPaymentDrainRepository::new(database.pool().clone()); + let job_id = Uuid::new_v4(); + let stale_claim_token = Uuid::new_v4(); + let now = OffsetDateTime::now_utc(); + let cleanup_token = + PaymentDrainCleanupToken::parse(&URL_SAFE_NO_PAD.encode([9_u8; 32])).unwrap(); + insert_drain_job( + database.pool(), + job_id, + stale_claim_token, + now, + cleanup_token.as_str(), + ) + .await; + + let mut force = database.pool().begin().await.unwrap(); + sqlx::query("SELECT job_id FROM content_lock_deletion_jobs WHERE job_id = $1 FOR UPDATE") + .bind(job_id) + .fetch_one(&mut *force) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET force_requested_at = $2, state = 'queued', claimed_by = NULL, + claim_token = NULL, claim_expires_at = NULL + WHERE job_id = $1", + ) + .bind(job_id) + .bind(now) + .execute(&mut *force) + .await + .unwrap(); + + let completed = PaymentDrainSummary { + status: PaymentDrainStatus::Completed, + accepted_count: 0, + terminal_count: 1, + cancellation_enqueued_count: 0, + cleanup_token: cleanup_token.clone(), + }; + let stale = tokio::spawn(async move { + repository + .reconcile_payment_drain(job_id, "worker", stale_claim_token, now, &completed) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!( + !stale.is_finished(), + "reconciliation must wait for the deletion-job ownership row" + ); + + force.commit().await.unwrap(); + assert!(!stale.await.unwrap().unwrap()); + assert_eq!( + PostgresPaymentDrainRepository::new(database.pool().clone()) + .get_payment_drain(job_id) + .await + .unwrap(), + Some(PaymentDrainSummary { + status: PaymentDrainStatus::Active, + accepted_count: 1, + terminal_count: 0, + cancellation_enqueued_count: 0, + cleanup_token, + }) + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn concurrent_reclaim_winner_fences_stale_terminal_obligation_persistence() { + let database = TestDatabase::create().await; + let repository = PostgresPaymentDrainRepository::new(database.pool().clone()); + let job_id = Uuid::new_v4(); + let task_uuid = Uuid::new_v4(); + let task_id = TaskId::from_str(&task_uuid.to_string()).unwrap(); + let stale_claim_token = Uuid::new_v4(); + let replacement_claim_token = Uuid::new_v4(); + let now = OffsetDateTime::now_utc(); + insert_terminal_obligation(database.pool(), job_id, task_uuid, stale_claim_token, now) + .await; + + let mut reclaim = database.pool().begin().await.unwrap(); + sqlx::query("SELECT job_id FROM content_lock_deletion_jobs WHERE job_id = $1 FOR UPDATE") + .bind(job_id) + .fetch_one(&mut *reclaim) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET claimed_by = 'replacement-worker', claim_token = $2, + claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job_id) + .bind(replacement_claim_token) + .bind(now + time::Duration::minutes(10)) + .execute(&mut *reclaim) + .await + .unwrap(); + + let stale = tokio::spawn(async move { + repository + .persist_terminal_obligation( + job_id, + "worker", + stale_claim_token, + now, + &task_id, + PaymentDrainTerminalTransition { + status: VerificationTaskStatus::Completed, + entitlement_publication_token: None, + }, + ) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!( + !stale.is_finished(), + "terminal persistence must wait for the deletion-job ownership row" + ); + + reclaim.commit().await.unwrap(); + assert!(!stale.await.unwrap().unwrap()); + let task_status: String = + sqlx::query_scalar("SELECT status FROM verification_tasks WHERE task_id = $1") + .bind(task_uuid) + .fetch_one(database.pool()) + .await + .unwrap(); + let resolved_status: Option = sqlx::query_scalar( + "SELECT resolved_status FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 AND verification_task_id = $2", + ) + .bind(job_id) + .bind(task_uuid) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(task_status, "pending"); + assert_eq!(resolved_status, None); + database.cleanup().await; + } + + async fn insert_start_drain_job( + pool: &sqlx::PgPool, + job_id: Uuid, + claim_token: Uuid, + now: OffsetDateTime, + ) { + sqlx::query( + "INSERT INTO content_lock_deletion_jobs + (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, + state, phase, claimed_by, claim_token, claim_expires_at) + VALUES ($1, 'creator', 'lock', '{}'::jsonb, $2, + 'running', 'start_payment_drain', 'worker', $3, $4)", + ) + .bind(job_id) + .bind(now) + .bind(claim_token) + .bind(now + time::Duration::minutes(5)) + .execute(pool) + .await + .unwrap(); + } + + async fn insert_drain_job( + pool: &sqlx::PgPool, + job_id: Uuid, + claim_token: Uuid, + now: OffsetDateTime, + cleanup_token: &str, + ) { + sqlx::query( + "INSERT INTO content_lock_deletion_jobs + (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, + state, phase, claimed_by, claim_token, claim_expires_at) + VALUES ($1, 'creator', 'lock', '{}'::jsonb, $2, + 'running', 'drain_payments', 'worker', $3, $4)", + ) + .bind(job_id) + .bind(now) + .bind(claim_token) + .bind(now + time::Duration::minutes(5)) + .execute(pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO content_lock_payment_drains + (deletion_job_id, status, accepted_count, terminal_count, + cancellation_enqueued_count, cleanup_token, created_at, updated_at) + VALUES ($1, 'active', 1, 0, 0, $2, $3, $3)", + ) + .bind(job_id) + .bind(cleanup_token) + .bind(now) + .execute(pool) + .await + .unwrap(); + } + + async fn insert_terminal_obligation( + pool: &sqlx::PgPool, + job_id: Uuid, + task_id: Uuid, + claim_token: Uuid, + now: OffsetDateTime, + ) { + sqlx::query( + "INSERT INTO content_lock_deletion_jobs + (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, + state, phase, claimed_by, claim_token, claim_expires_at) + VALUES ($1, 'creator', 'lock', '{}'::jsonb, $2, + 'running', 'drain_payments', 'worker', $3, $4)", + ) + .bind(job_id) + .bind(now) + .bind(claim_token) + .bind(now + time::Duration::minutes(5)) + .execute(pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO verification_tasks + (task_id, status, submitted_proof_bundle, submitted_at, creator, bundle_id, + deletion_job_id) + VALUES ($1, 'pending', '{}'::jsonb, $2, 'creator', 'bundle', $3)", + ) + .bind(task_id) + .bind(now) + .bind(job_id) + .execute(pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO content_lock_deletion_task_snapshot + (deletion_job_id, verification_task_id, creator, bundle_id, + pubky_lock_resource, criterion_id, status_at_cutoff, + paykit_admission_required, payment_in_hours, + invoice_created_at, payment_deadline) + VALUES ($1, $2, 'creator', 'bundle', 'pubkycreator/pub/locks.app/lock.json', + 'payment', 'pending', TRUE, 1, $3, $4)", + ) + .bind(job_id) + .bind(task_id) + .bind(now) + .bind(now + time::Duration::hours(1)) + .execute(pool) + .await + .unwrap(); + } } diff --git a/locks-service/src/infrastructure/runtime_master_key.rs b/locks-service/src/infrastructure/runtime_master_key.rs new file mode 100644 index 0000000..1f51158 --- /dev/null +++ b/locks-service/src/infrastructure/runtime_master_key.rs @@ -0,0 +1,85 @@ +use std::fmt; + +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; + +const CREATOR_AUTHORITY_KEY_CONTEXT: &str = + "pubky-locks v1 runtime master key: creator authority secrets"; +const FINAL_CREDENTIAL_KEY_CONTEXT: &str = + "pubky-locks v1 runtime master key: final deletion credentials"; + +/// Root key for deriving independent runtime encryption keys. +#[derive(Clone)] +pub struct RuntimeMasterKey { + bytes: [u8; 32], +} + +impl fmt::Debug for RuntimeMasterKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("RuntimeMasterKey") + .field(&"") + .finish() + } +} + +#[derive(Debug, thiserror::Error)] +#[error("runtime master key must be an unpadded base64url-encoded 32-byte key")] +pub struct InvalidRuntimeMasterKey; + +impl RuntimeMasterKey { + pub fn from_base64url(value: &str) -> Result { + let bytes = URL_SAFE_NO_PAD + .decode(value) + .map_err(|_| InvalidRuntimeMasterKey)?; + let bytes = bytes.try_into().map_err(|_| InvalidRuntimeMasterKey)?; + Ok(Self { bytes }) + } + + pub fn creator_authority_key(&self) -> [u8; 32] { + blake3::derive_key(CREATOR_AUTHORITY_KEY_CONTEXT, &self.bytes) + } + + pub fn final_credential_key(&self) -> [u8; 32] { + blake3::derive_key(FINAL_CREDENTIAL_KEY_CONTEXT, &self.bytes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn derives_stable_distinct_keys_for_closed_runtime_domains() { + let encoded = URL_SAFE_NO_PAD.encode([7u8; 32]); + let first = RuntimeMasterKey::from_base64url(&encoded).unwrap(); + let second = RuntimeMasterKey::from_base64url(&encoded).unwrap(); + + assert_eq!( + first.creator_authority_key(), + second.creator_authority_key() + ); + assert_eq!(first.final_credential_key(), second.final_credential_key()); + assert_ne!(first.creator_authority_key(), first.final_credential_key()); + assert_ne!(first.creator_authority_key(), [7u8; 32]); + assert_ne!(first.final_credential_key(), [7u8; 32]); + } + + #[test] + fn rejects_invalid_or_wrong_length_values_without_exposing_input() { + for value in ["not-a-key***", &URL_SAFE_NO_PAD.encode([7u8; 31])] { + let error = RuntimeMasterKey::from_base64url(value).unwrap_err(); + let debug = format!("{error:?}"); + assert!(!debug.contains(value)); + } + } + + #[test] + fn debug_output_redacts_root_key() { + let encoded = URL_SAFE_NO_PAD.encode([9u8; 32]); + let key = RuntimeMasterKey::from_base64url(&encoded).unwrap(); + let debug = format!("{key:?}"); + + assert_eq!(debug, "RuntimeMasterKey(\"\")"); + assert!(!debug.contains(&encoded)); + } +} diff --git a/locks-service/tests/content_lock_deletions.rs b/locks-service/tests/content_lock_deletions.rs index e302635..8224055 100644 --- a/locks-service/tests/content_lock_deletions.rs +++ b/locks-service/tests/content_lock_deletions.rs @@ -1,22 +1,35 @@ -use std::{collections::BTreeMap, str::FromStr}; +use std::{collections::BTreeMap, str::FromStr, sync::Arc}; use locks_core::{ - ids::{CreatorPubky, GuardedResourceHash}, + ids::{BundleId, CreatorPubky, GuardedResourceHash, PubkyLockResource, TaskId}, lock_policy::{ AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, GuardedResource, LockLogic, - LockServerConfig, + LockServerConfig, VerifierType, }, + verification::{Proof, SUBMITTED_PROOF_BUNDLE_VERSION, SubmittedProofBundle}, }; use locks_service::{ application::{ models::{ + AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, ContentLockDeletionFailureCode, ContentLockDeletionJob, ContentLockDeletionPhase, - ContentLockDeletionState, PrepareForceDeletionResult, + ContentLockDeletionState, PrepareForceDeletionResult, VerificationTaskRecord, + VerificationTaskStatus, }, - ports::ContentLockDeletionRepository, + ports::{ + AccessCredentialStore, Clock, ContentLockDeletionRepository, VerificationTaskClaimer, + VerificationTaskRepository, + }, + }, + infrastructure::memory::{ + access_credentials::InMemoryAccessCredentialStore, + content_lock_deletions::InMemoryContentLockDeletionRepository, + verification_task_claims::InMemoryVerificationTaskClaimer, + verification_task_deletion_fence::InMemoryVerificationTaskDeletionFence, + verification_tasks::InMemoryVerificationTaskRepository, }, - infrastructure::memory::content_lock_deletions::InMemoryContentLockDeletionRepository, }; +use serde_json::json; use time::macros::datetime; use uuid::Uuid; @@ -24,9 +37,20 @@ const CREATOR: &str = "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy const NOW: time::OffsetDateTime = datetime!(2026-08-12 05:00:00 UTC); const LEASE_END: time::OffsetDateTime = datetime!(2026-08-12 05:05:00 UTC); +#[derive(Debug)] +struct FixedClock(time::OffsetDateTime); + +impl Clock for FixedClock { + fn now(&self) -> time::OffsetDateTime { + self.0 + } +} + #[tokio::test] async fn frozen_manifest_identity_is_immutable_and_creator_lock_unique() { - let repository = InMemoryContentLockDeletionRepository::new(); + let repository = InMemoryContentLockDeletionRepository::with_verification_task_fence(Arc::new( + InMemoryVerificationTaskDeletionFence::with_clock(Arc::new(FixedClock(NOW))), + )); let lock = content_lock(); let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock.clone(), NOW).unwrap(); @@ -62,7 +86,9 @@ async fn frozen_manifest_identity_is_immutable_and_creator_lock_unique() { #[tokio::test] async fn due_claims_reclaim_with_fresh_tokens_and_fence_stale_writes() { - let repository = InMemoryContentLockDeletionRepository::new(); + let repository = InMemoryContentLockDeletionRepository::with_verification_task_fence(Arc::new( + InMemoryVerificationTaskDeletionFence::with_clock(Arc::new(FixedClock(NOW))), + )); let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); repository.insert_job(job.clone()).await.unwrap(); @@ -195,7 +221,9 @@ fn failure_codes_are_a_closed_stable_vocabulary() { #[tokio::test] async fn retry_due_time_and_force_receipts_are_durable_repository_facts() { - let repository = InMemoryContentLockDeletionRepository::new(); + let repository = InMemoryContentLockDeletionRepository::with_verification_task_fence(Arc::new( + InMemoryVerificationTaskDeletionFence::with_clock(Arc::new(FixedClock(NOW))), + )); let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); repository.insert_job(job.clone()).await.unwrap(); let claimed = repository @@ -239,6 +267,1094 @@ async fn retry_due_time_and_force_receipts_are_durable_repository_facts() { ); } +#[tokio::test] +async fn in_memory_deletion_enrolls_existing_ordinary_credentials_and_blocks_late_insertion() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let lock = content_lock(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock.clone(), NOW).unwrap(); + let task = verification_task(&job, VerificationTaskStatus::Completed); + verification_tasks + .insert_verification_task(task.clone()) + .await + .unwrap(); + + let ordinary = AccessCredential::new("ordinary-before-deletion"); + let ordinary_lookup = AccessCredentialLookupKey::derive(&ordinary); + let original_expiry = NOW + time::Duration::minutes(10); + access + .insert_access_credential( + &job.lock_id, + ordinary_lookup.clone(), + AccessCredentialRecord { + creator: job.creator.clone(), + bundle_id: task.submitted_proof_bundle.bundle_id.clone(), + expires_at: original_expiry, + }, + ) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + + let first = access + .prepare_deletion_read( + &ordinary_lookup, + "/priv/locks.app/content/post.json", + NOW, + NOW + time::Duration::seconds(30), + ) + .await + .unwrap() + .unwrap(); + let replay = access + .prepare_deletion_read( + &ordinary_lookup, + "/priv/locks.app/content/post.json", + NOW + time::Duration::minutes(1), + NOW + time::Duration::minutes(2), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(first.claim_token, None); + assert_eq!(replay, first); + assert_eq!( + access + .get_access_credential(&ordinary_lookup) + .await + .unwrap() + .unwrap() + .expires_at, + original_expiry + ); + assert!( + access + .prepare_deletion_read( + &ordinary_lookup, + "/priv/locks.app/content/not-in-frozen-manifest.json", + NOW, + NOW + time::Duration::seconds(30), + ) + .await + .unwrap() + .is_none() + ); + + assert_eq!( + access + .insert_access_credential( + &job.lock_id, + AccessCredentialLookupKey::derive(&AccessCredential::new("late")), + AccessCredentialRecord { + creator: job.creator.clone(), + bundle_id: BundleId::from_str("000G40R40M30E209185GR38E1V").unwrap(), + expires_at: original_expiry, + }, + ) + .await, + Err(locks_service::application::errors::ApplicationError::ContentLockDeletionInProgress) + ); + + let drain_existing_claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::DrainExistingCredentials, + ) + .await; + assert!(matches!( + deletions + .advance_phase( + job.job_id, + "worker-final", + drain_existing_claim.claim_token, + NOW, + ContentLockDeletionPhase::IssueFinalCredentials, + ) + .await, + Err( + locks_service::application::errors::ApplicationError::InvalidContentLockDeletionState { .. } + ) + )); + + assert!(matches!( + deletions + .prepare_force_deletion(&job.creator, &job.lock_id, NOW) + .await + .unwrap(), + PrepareForceDeletionResult::Active(_) + )); + assert_eq!( + access + .get_access_credential(&ordinary_lookup) + .await + .unwrap() + .unwrap() + .expires_at, + original_expiry + ); +} + +#[tokio::test] +async fn in_memory_cutoff_is_captured_under_the_shared_fence_not_from_the_caller() { + let authoritative_cutoff = NOW + time::Duration::minutes(2); + let fence = Arc::new(InMemoryVerificationTaskDeletionFence::with_clock(Arc::new( + FixedClock(authoritative_cutoff), + ))); + let verification_tasks = Arc::new(InMemoryVerificationTaskRepository::with_deletion_fence( + Arc::clone(&fence), + )); + let access = Arc::new( + InMemoryAccessCredentialStore::with_verification_task_repository_and_deletion_fence( + verification_tasks.clone(), + Arc::clone(&fence), + ), + ); + let deletions = + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + Arc::clone(&access), + fence, + ); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let task = verification_task(&job, VerificationTaskStatus::Pending); + let credential = AccessCredential::new("expired-while-waiting-for-cutoff-fence"); + let lookup = AccessCredentialLookupKey::derive(&credential); + verification_tasks + .insert_verification_task(task.clone()) + .await + .unwrap(); + access + .insert_access_credential( + &job.lock_id, + lookup.clone(), + AccessCredentialRecord { + creator: job.creator.clone(), + bundle_id: task.submitted_proof_bundle.bundle_id, + expires_at: NOW + time::Duration::minutes(1), + }, + ) + .await + .unwrap(); + + deletions.insert_job(job.clone()).await.unwrap(); + + assert_eq!( + deletions + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .unwrap() + .deletion_started_at, + authoritative_cutoff + ); + assert!(!access.deletion_credential_enrolled(&lookup).await.unwrap()); +} + +#[tokio::test] +async fn failed_access_registration_leaves_no_job_or_task_ownership() { + let lock = content_lock(); + let first_job = ContentLockDeletionJob::new(Uuid::new_v4(), lock.clone(), NOW).unwrap(); + let first_fence = Arc::new(InMemoryVerificationTaskDeletionFence::with_clock(Arc::new( + FixedClock(NOW), + ))); + let first_tasks = Arc::new(InMemoryVerificationTaskRepository::with_deletion_fence( + Arc::clone(&first_fence), + )); + let access = Arc::new( + InMemoryAccessCredentialStore::with_verification_task_repository_and_deletion_fence( + first_tasks, + Arc::clone(&first_fence), + ), + ); + let first_deletions = + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + Arc::clone(&access), + first_fence, + ); + tokio::time::timeout( + std::time::Duration::from_secs(1), + first_deletions.insert_job(first_job), + ) + .await + .expect("first deletion admission must not deadlock") + .unwrap(); + + let second_fence = Arc::new(InMemoryVerificationTaskDeletionFence::with_clock(Arc::new( + FixedClock(NOW), + ))); + let second_tasks = Arc::new(InMemoryVerificationTaskRepository::with_deletion_fence( + Arc::clone(&second_fence), + )); + let second_job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + let task = verification_task(&second_job, VerificationTaskStatus::Pending); + second_tasks + .insert_verification_task(task.clone()) + .await + .unwrap(); + let claimer = + InMemoryVerificationTaskClaimer::with_deletion_fence(vec![task], Arc::clone(&second_fence)); + let second_deletions = + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + access, + second_fence, + ); + + assert_eq!( + tokio::time::timeout( + std::time::Duration::from_secs(1), + second_deletions.insert_job(second_job.clone()), + ) + .await + .expect("failed deletion admission must not deadlock"), + Err(locks_service::application::errors::ApplicationError::ContentLockDeletionInProgress) + ); + assert!( + second_deletions + .get_job(&second_job.creator, &second_job.lock_id) + .await + .unwrap() + .is_none() + ); + assert!( + tokio::time::timeout( + std::time::Duration::from_secs(1), + claimer.claim_next_verification_task("worker", NOW, LEASE_END), + ) + .await + .expect("failed deletion admission must release task ownership locks") + .unwrap() + .is_some() + ); +} + +#[tokio::test] +async fn in_memory_final_credential_is_exactly_replayable_and_reads_are_lease_fenced() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let completed = verification_task(&job, VerificationTaskStatus::Pending) + .transition_to(VerificationTaskStatus::InProgress, NOW, None) + .unwrap() + .transition_to(VerificationTaskStatus::Completed, NOW, None) + .unwrap(); + let bundle_id = completed.submitted_proof_bundle.bundle_id.clone(); + verification_tasks + .insert_verification_task(completed) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let claimed = advance_to_final_issuance(&deletions, &access, job.job_id).await; + let issuance_deadline = NOW + time::Duration::minutes(15); + let read_deadline = NOW + time::Duration::minutes(30); + assert!( + access + .initialize_final_access_windows( + job.job_id, + "worker-final", + claimed.claim_token, + NOW, + issuance_deadline, + read_deadline, + ) + .await + .unwrap() + ); + assert!( + access + .initialize_final_access_windows( + job.job_id, + "worker-final", + claimed.claim_token, + NOW, + NOW + time::Duration::minutes(20), + NOW + time::Duration::minutes(40), + ) + .await + .unwrap() + ); + + let candidate = AccessCredential::new("final-secret-bearer"); + let first = access + .issue_or_replay_final_credential(&job.creator, &bundle_id, NOW, candidate.clone()) + .await + .unwrap() + .unwrap(); + let replay = access + .issue_or_replay_final_credential( + &job.creator, + &bundle_id, + NOW + time::Duration::minutes(1), + AccessCredential::new("different-candidate-must-not-win"), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(first, replay); + assert_eq!(first.credential, candidate); + assert_eq!(first.expires_at, read_deadline); + assert!(!format!("{access:?}").contains("final-secret-bearer")); + let boundary_replay = access + .issue_or_replay_final_credential( + &job.creator, + &bundle_id, + issuance_deadline, + AccessCredential::new("boundary-candidate-must-not-win"), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(boundary_replay, first); + + let advanced = deletions + .advance_phase( + job.job_id, + "worker-final", + claimed.claim_token, + NOW, + ContentLockDeletionPhase::DrainFinalReads, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(advanced.phase, ContentLockDeletionPhase::DrainFinalReads); + let phase_replay = access + .issue_or_replay_final_credential( + &job.creator, + &bundle_id, + issuance_deadline, + AccessCredential::new("post-advance-candidate-must-not-win"), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(phase_replay, first); + + let lookup = AccessCredentialLookupKey::derive(&first.credential); + let path = "/priv/locks.app/content/post.json"; + let first_claim = access + .prepare_deletion_read(&lookup, path, NOW, NOW + time::Duration::minutes(1)) + .await + .unwrap() + .unwrap() + .claim_token + .unwrap(); + let equality_reclaim = access + .prepare_deletion_read( + &lookup, + path, + NOW + time::Duration::seconds(30), + NOW + time::Duration::minutes(2), + ) + .await + .unwrap() + .unwrap() + .claim_token + .unwrap(); + assert_ne!(equality_reclaim, first_claim); + assert!( + !access + .release_deletion_read(&lookup, path, Uuid::new_v4(), NOW) + .await + .unwrap() + ); + assert!( + !access + .release_deletion_read( + &lookup, + path, + first_claim, + NOW + time::Duration::seconds(30), + ) + .await + .unwrap() + ); + assert!( + access + .release_deletion_read( + &lookup, + path, + equality_reclaim, + NOW + time::Duration::seconds(30), + ) + .await + .unwrap() + ); + let stale_claim = access + .prepare_deletion_read( + &lookup, + path, + NOW + time::Duration::seconds(30), + NOW + time::Duration::hours(1), + ) + .await + .unwrap() + .unwrap() + .claim_token + .unwrap(); + let reclaim_time = NOW + time::Duration::seconds(60); + assert!( + !access + .consume_deletion_read(&lookup, path, stale_claim, reclaim_time) + .await + .unwrap() + ); + let recovered_claim = access + .prepare_deletion_read(&lookup, path, reclaim_time, NOW + time::Duration::hours(1)) + .await + .unwrap() + .unwrap() + .claim_token + .unwrap(); + assert!( + !access + .consume_deletion_read(&lookup, path, stale_claim, reclaim_time) + .await + .unwrap() + ); + assert!( + access + .consume_deletion_read(&lookup, path, recovered_claim, reclaim_time) + .await + .unwrap() + ); + assert!( + access + .prepare_deletion_read( + &lookup, + path, + NOW + time::Duration::minutes(3), + NOW + time::Duration::minutes(4), + ) + .await + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn mutable_task_completion_cannot_resolve_the_immutable_deletion_snapshot() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let pending = verification_task(&job, VerificationTaskStatus::Pending); + let bundle_id = pending.submitted_proof_bundle.bundle_id.clone(); + let task_id = pending.task_id; + verification_tasks + .insert_verification_task(pending.clone()) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + + let drain_claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::DrainPayments, + ) + .await; + let completed = pending + .transition_to(VerificationTaskStatus::InProgress, NOW, None) + .unwrap() + .transition_to(VerificationTaskStatus::Completed, NOW, None) + .unwrap(); + verification_tasks + .update_verification_task(completed) + .await + .unwrap(); + + assert!( + !access + .final_credential_available(&job.creator, &bundle_id, NOW) + .await + .unwrap() + ); + assert!( + access + .issue_or_replay_final_credential( + &job.creator, + &bundle_id, + NOW, + AccessCredential::new("mutable-completion-must-not-win"), + ) + .await + .unwrap() + .is_none() + ); + assert!( + access + .resolve_deletion_payment( + job.job_id, + "worker-final", + drain_claim.claim_token, + NOW, + &task_id, + VerificationTaskStatus::Completed, + ) + .await + .unwrap() + ); + assert!( + access + .complete_deletion_payment_aggregate( + job.job_id, + "worker-final", + drain_claim.claim_token, + NOW, + ) + .await + .unwrap() + ); + + deletions + .advance_phase( + job.job_id, + "worker-final", + drain_claim.claim_token, + NOW, + ContentLockDeletionPhase::DrainExistingCredentials, + ) + .await + .unwrap() + .unwrap(); + let existing_drain_claim = deletions + .claim_next("worker-final", NOW, LEASE_END) + .await + .unwrap() + .unwrap(); + deletions + .advance_phase( + job.job_id, + "worker-final", + existing_drain_claim.claim_token, + NOW, + ContentLockDeletionPhase::IssueFinalCredentials, + ) + .await + .unwrap() + .unwrap(); + let final_claim = deletions + .claim_next("worker-final", NOW, LEASE_END) + .await + .unwrap() + .unwrap(); + access + .initialize_final_access_windows( + job.job_id, + "worker-final", + final_claim.claim_token, + NOW, + NOW + time::Duration::minutes(15), + NOW + time::Duration::minutes(30), + ) + .await + .unwrap(); + assert!( + access + .final_credential_available(&job.creator, &bundle_id, NOW) + .await + .unwrap() + ); +} + +#[tokio::test] +async fn in_memory_payment_drain_waits_for_pending_non_paykit_snapshot() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let mut pending = verification_task(&job, VerificationTaskStatus::Pending); + pending.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::DevStatic; + verification_tasks + .insert_verification_task(pending) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let drain_claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::DrainPayments, + ) + .await; + + assert!(matches!( + tokio::time::timeout( + std::time::Duration::from_secs(1), + deletions.advance_phase( + job.job_id, + "worker-final", + drain_claim.claim_token, + NOW, + ContentLockDeletionPhase::DrainExistingCredentials, + ), + ) + .await + .expect("pending non-Paykit guard must not deadlock"), + Err( + locks_service::application::errors::ApplicationError::InvalidContentLockDeletionState { .. } + ) + )); +} + +#[tokio::test] +async fn in_memory_payment_drain_waits_for_pending_paykit_snapshot() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + verification_tasks + .insert_verification_task(verification_task(&job, VerificationTaskStatus::Pending)) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let drain_claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::DrainPayments, + ) + .await; + + assert!(matches!( + tokio::time::timeout( + std::time::Duration::from_secs(1), + deletions.advance_phase( + job.job_id, + "worker-final", + drain_claim.claim_token, + NOW, + ContentLockDeletionPhase::DrainExistingCredentials, + ), + ) + .await + .expect("pending Paykit guard must not deadlock"), + Err( + locks_service::application::errors::ApplicationError::InvalidContentLockDeletionState { .. } + ) + )); +} + +#[tokio::test] +async fn in_memory_payment_drain_waits_for_completed_aggregate() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + verification_tasks + .insert_verification_task(verification_task(&job, VerificationTaskStatus::Completed)) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let drain_claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::DrainPayments, + ) + .await; + + assert!( + !access + .complete_deletion_payment_aggregate( + job.job_id, + "different-worker", + drain_claim.claim_token, + NOW, + ) + .await + .unwrap() + ); + assert!( + !access + .complete_deletion_payment_aggregate(job.job_id, "worker-final", Uuid::new_v4(), NOW,) + .await + .unwrap() + ); + assert!(matches!( + tokio::time::timeout( + std::time::Duration::from_secs(1), + deletions.advance_phase( + job.job_id, + "worker-final", + drain_claim.claim_token, + NOW, + ContentLockDeletionPhase::DrainExistingCredentials, + ), + ) + .await + .expect("payment aggregate guard must not deadlock"), + Err( + locks_service::application::errors::ApplicationError::InvalidContentLockDeletionState { .. } + ) + )); +} + +#[tokio::test] +async fn in_memory_phase_and_success_finish_guards_preserve_access_obligations() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let completed = verification_task(&job, VerificationTaskStatus::Pending) + .transition_to(VerificationTaskStatus::InProgress, NOW, None) + .unwrap() + .transition_to(VerificationTaskStatus::Completed, NOW, None) + .unwrap(); + let bundle_id = completed.submitted_proof_bundle.bundle_id.clone(); + verification_tasks + .insert_verification_task(completed) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + + let issue_claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::IssueFinalCredentials, + ) + .await; + access + .initialize_final_access_windows( + job.job_id, + "worker-final", + issue_claim.claim_token, + NOW, + NOW + time::Duration::minutes(15), + NOW + time::Duration::minutes(30), + ) + .await + .unwrap(); + let deadline = NOW + time::Duration::minutes(15); + let deadline_claim = deletions + .claim_next( + "worker-final", + deadline, + deadline + time::Duration::minutes(5), + ) + .await + .unwrap() + .unwrap(); + assert!(matches!( + deletions + .advance_phase( + job.job_id, + "worker-final", + deadline_claim.claim_token, + deadline, + ContentLockDeletionPhase::DrainFinalReads, + ) + .await, + Err( + locks_service::application::errors::ApplicationError::InvalidContentLockDeletionState { .. } + ) + )); + assert!( + access + .issue_or_replay_final_credential( + &job.creator, + &bundle_id, + deadline, + AccessCredential::new("fresh-at-deadline-must-not-issue"), + ) + .await + .unwrap() + .is_none() + ); + assert!(matches!( + deletions + .finish( + job.job_id, + "worker-final", + deadline_claim.claim_token, + deadline, + None + ) + .await, + Err( + locks_service::application::errors::ApplicationError::InvalidContentLockDeletionState { .. } + ) + )); +} + +#[tokio::test] +async fn in_memory_final_credential_eligibility_does_not_change_when_cutoff_credential_is_deleted() +{ + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let completed = verification_task(&job, VerificationTaskStatus::Pending) + .transition_to(VerificationTaskStatus::InProgress, NOW, None) + .unwrap() + .transition_to(VerificationTaskStatus::Completed, NOW, None) + .unwrap(); + let bundle_id = completed.submitted_proof_bundle.bundle_id.clone(); + verification_tasks + .insert_verification_task(completed) + .await + .unwrap(); + + let ordinary = AccessCredential::new("active-at-cutoff"); + let ordinary_lookup = AccessCredentialLookupKey::derive(&ordinary); + let original_expiry = NOW + time::Duration::minutes(10); + access + .insert_access_credential( + &job.lock_id, + ordinary_lookup.clone(), + AccessCredentialRecord { + creator: job.creator.clone(), + bundle_id: bundle_id.clone(), + expires_at: original_expiry, + }, + ) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let drain_existing_claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::DrainExistingCredentials, + ) + .await; + assert!(matches!( + deletions + .advance_phase( + job.job_id, + "worker-final", + drain_existing_claim.claim_token, + NOW, + ContentLockDeletionPhase::IssueFinalCredentials, + ) + .await, + Err( + locks_service::application::errors::ApplicationError::InvalidContentLockDeletionState { .. } + ) + )); + + let after_expiry = original_expiry; + access + .delete_access_credential(&ordinary_lookup) + .await + .unwrap(); + let after_expiry_claim = deletions + .claim_next( + "worker-final", + after_expiry, + after_expiry + time::Duration::minutes(5), + ) + .await + .unwrap() + .unwrap(); + deletions + .advance_phase( + job.job_id, + "worker-final", + after_expiry_claim.claim_token, + after_expiry, + ContentLockDeletionPhase::IssueFinalCredentials, + ) + .await + .unwrap() + .unwrap(); + let claimed = deletions + .claim_next( + "worker-final", + after_expiry, + after_expiry + time::Duration::minutes(5), + ) + .await + .unwrap() + .unwrap(); + assert!( + access + .initialize_final_access_windows( + job.job_id, + "worker-final", + claimed.claim_token, + after_expiry, + after_expiry + time::Duration::minutes(15), + after_expiry + time::Duration::minutes(30), + ) + .await + .unwrap() + ); + + access + .delete_access_credential(&ordinary_lookup) + .await + .unwrap(); + + assert!( + !access + .final_credential_available(&job.creator, &bundle_id, after_expiry) + .await + .unwrap() + ); + assert!( + access + .issue_or_replay_final_credential( + &job.creator, + &bundle_id, + after_expiry, + AccessCredential::new("must-remain-ineligible"), + ) + .await + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn in_memory_final_credential_rejects_completed_non_paykit_snapshot() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let mut task = verification_task(&job, VerificationTaskStatus::Completed); + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::DevStatic; + let bundle_id = task.submitted_proof_bundle.bundle_id.clone(); + verification_tasks + .insert_verification_task(task.clone()) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let claimed = advance_to_final_issuance(&deletions, &access, job.job_id).await; + assert!( + access + .initialize_final_access_windows( + job.job_id, + "worker-final", + claimed.claim_token, + NOW, + NOW + time::Duration::minutes(15), + NOW + time::Duration::minutes(30), + ) + .await + .unwrap() + ); + + assert!( + !access + .final_credential_available(&job.creator, &bundle_id, NOW) + .await + .unwrap() + ); + assert!( + access + .issue_or_replay_final_credential( + &job.creator, + &bundle_id, + NOW, + AccessCredential::new("must-not-be-issued"), + ) + .await + .unwrap() + .is_none() + ); +} + +fn in_memory_access_stack() -> ( + Arc, + Arc, + InMemoryContentLockDeletionRepository, +) { + let fence = Arc::new(InMemoryVerificationTaskDeletionFence::with_clock(Arc::new( + FixedClock(NOW), + ))); + let verification_tasks = Arc::new(InMemoryVerificationTaskRepository::with_deletion_fence( + Arc::clone(&fence), + )); + let access = Arc::new( + InMemoryAccessCredentialStore::with_verification_task_repository_and_deletion_fence( + verification_tasks.clone(), + Arc::clone(&fence), + ), + ); + let deletions = + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + Arc::clone(&access), + fence, + ); + (verification_tasks, access, deletions) +} + +async fn advance_to_phase( + deletions: &InMemoryContentLockDeletionRepository, + access: &InMemoryAccessCredentialStore, + job_id: Uuid, + target: ContentLockDeletionPhase, +) -> locks_service::application::models::ClaimedContentLockDeletionJob { + for next_phase in [ + ContentLockDeletionPhase::StartPaymentDrain, + ContentLockDeletionPhase::DrainPayments, + ContentLockDeletionPhase::DrainExistingCredentials, + ContentLockDeletionPhase::IssueFinalCredentials, + ] { + let claimed = deletions + .claim_next("worker-final", NOW, LEASE_END) + .await + .unwrap() + .unwrap(); + if next_phase == ContentLockDeletionPhase::DrainExistingCredentials { + assert!( + access + .complete_deletion_payment_aggregate( + job_id, + "worker-final", + claimed.claim_token, + NOW, + ) + .await + .unwrap() + ); + } + deletions + .advance_phase(job_id, "worker-final", claimed.claim_token, NOW, next_phase) + .await + .unwrap() + .unwrap(); + if next_phase == target { + return deletions + .claim_next("worker-final", NOW, LEASE_END) + .await + .unwrap() + .unwrap(); + } + } + panic!("unsupported test target phase"); +} + +async fn advance_to_final_issuance( + deletions: &InMemoryContentLockDeletionRepository, + access: &InMemoryAccessCredentialStore, + job_id: Uuid, +) -> locks_service::application::models::ClaimedContentLockDeletionJob { + advance_to_phase( + deletions, + access, + job_id, + ContentLockDeletionPhase::IssueFinalCredentials, + ) + .await +} + +fn verification_task( + job: &ContentLockDeletionJob, + status: VerificationTaskStatus, +) -> VerificationTaskRecord { + VerificationTaskRecord { + task_id: TaskId::from_str("018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d10").unwrap(), + creator: job.creator.clone(), + submitted_proof_bundle: SubmittedProofBundle { + version: SUBMITTED_PROOF_BUNDLE_VERSION, + bundle_id: BundleId::from_str("000G40R40M30E209185GR38E1W").unwrap(), + pubky_lock_resource: PubkyLockResource::from_str(&format!( + "{}/pub/locks.app/{}.json", + job.creator, job.lock_id + )) + .unwrap(), + reader_public_key: None, + proofs: vec![Proof { + criterion_id: "criterion-1".to_owned(), + verifier_type: VerifierType::PaykitPayment, + payload: json!({}), + }], + }, + status, + submitted_at: NOW, + started_at: None, + completed_at: None, + failure_message: None, + } +} + fn content_lock() -> ContentLock { ContentLock { version: CONTENT_LOCK_VERSION, diff --git a/scripts/test-compose-bootstrap.sh b/scripts/test-compose-bootstrap.sh index 3557937..8317c28 100755 --- a/scripts/test-compose-bootstrap.sh +++ b/scripts/test-compose-bootstrap.sh @@ -3,7 +3,7 @@ set -eu repo_root="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" entrypoint="$repo_root/docker/locks-server-compose-entrypoint.sh" -key_name="PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY" +key_name="PUBKY_LOCK_RUNTIME_MASTER_KEY" env -u "$key_name" docker compose -f "$repo_root/docker-compose.yml" config --quiet @@ -23,7 +23,7 @@ EOF cat > "$bin_dir/locks-server" <<'EOF' #!/bin/sh set -eu -printf '%s' "$PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY" > "$LOCKS_TEST_KEY_CAPTURE" +printf '%s' "$PUBKY_LOCK_RUNTIME_MASTER_KEY" > "$LOCKS_TEST_KEY_CAPTURE" EOF chmod +x "$bin_dir/locks-server" @@ -46,7 +46,7 @@ file_mode() { } run_entrypoint -key_file="$service_home/creator-authority-encryption-key" +key_file="$service_home/runtime-master-key" test -f "$key_file" test "$(wc -c < "$key_file" | tr -d ' ')" -eq 43 grep -Eq '^[A-Za-z0-9_-]{43}$' "$key_file" @@ -61,7 +61,22 @@ run_entrypoint test "$(cat "$capture")" = "$first_key" override='AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' -PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY="$override" \ +if PUBKY_LOCK_RUNTIME_MASTER_KEY="$override" \ + PATH="$bin_dir:$PATH" \ + LOCKS_SERVICE_HOME="$service_home" \ + LOCKS_COMPOSE_CONFIG="$tmp/config.compose.toml" \ + LOCKS_TEST_KEY_CAPTURE="$capture" \ + sh "$entrypoint" >"$tmp/mismatched-key.stdout" 2>"$tmp/mismatched-key.stderr"; then + echo "entrypoint replaced an existing runtime master key" >&2 + exit 1 +fi +grep -q "does not match the persisted runtime master key" "$tmp/mismatched-key.stderr" +test "$(cat "$key_file")" = "$first_key" + +# Explicitly discarding local encrypted state includes discarding its key. A +# valid override may establish the key only once that reset has happened. +rm "$key_file" +PUBKY_LOCK_RUNTIME_MASTER_KEY="$override" \ PATH="$bin_dir:$PATH" \ LOCKS_SERVICE_HOME="$service_home" \ LOCKS_COMPOSE_CONFIG="$tmp/config.compose.toml" \ @@ -69,6 +84,49 @@ PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY="$override" \ LOCKS_TEST_KEY_CAPTURE="$capture" \ sh "$entrypoint" test "$(cat "$capture")" = "$override" -test "$(cat "$key_file")" = "$first_key" +test "$(cat "$key_file")" = "$override" +test "$(file_mode "$key_file")" = 600 + +run_entrypoint +test "$(cat "$capture")" = "$override" + +invalid_override='AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' +if PUBKY_LOCK_RUNTIME_MASTER_KEY="$invalid_override" \ + PATH="$bin_dir:$PATH" \ + LOCKS_SERVICE_HOME="$service_home" \ + LOCKS_COMPOSE_CONFIG="$tmp/config.compose.toml" \ + LOCKS_TEST_KEY_CAPTURE="$capture" \ + sh "$entrypoint" >"$tmp/invalid-key.stdout" 2>"$tmp/invalid-key.stderr"; then + echo "entrypoint accepted a padded-or-wrong-length runtime master key" >&2 + exit 1 +fi +grep -q "must be an unpadded base64url-encoded 32-byte key" "$tmp/invalid-key.stderr" +test "$(cat "$key_file")" = "$override" + +# The final base64url character for 32 bytes carries only two data bits. `B` +# has non-zero trailing bits and must not be accepted as an alias for `A`. +noncanonical_override='AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB' +if PUBKY_LOCK_RUNTIME_MASTER_KEY="$noncanonical_override" \ + PATH="$bin_dir:$PATH" \ + LOCKS_SERVICE_HOME="$service_home" \ + LOCKS_COMPOSE_CONFIG="$tmp/config.compose.toml" \ + LOCKS_TEST_KEY_CAPTURE="$capture" \ + sh "$entrypoint" >"$tmp/noncanonical-key.stdout" 2>"$tmp/noncanonical-key.stderr"; then + echo "entrypoint accepted a noncanonical runtime master key" >&2 + exit 1 +fi +grep -q "must be an unpadded base64url-encoded 32-byte key" \ + "$tmp/noncanonical-key.stderr" +test "$(cat "$key_file")" = "$override" + +retired_key_file="$service_home/creator-authority-encryption-key" +: > "$retired_key_file" +if run_entrypoint >"$tmp/retired-key.stdout" 2>"$tmp/retired-key.stderr"; then + echo "entrypoint accepted retired creator-authority key" >&2 + exit 1 +fi +grep -q "retired creator-authority key detected" "$tmp/retired-key.stderr" +grep -q "discard and reacquire creator authority rows or recreate the local database" \ + "$tmp/retired-key.stderr" printf 'compose bootstrap regression passed\n'