From 480e76c1f9eb2420f4b13c3c2280a194b80fdaa3 Mon Sep 17 00:00:00 2001 From: dzdidi Date: Tue, 11 Aug 2026 08:11:35 -0300 Subject: [PATCH 1/6] docs: synchronize payment lifecycle deletion contract Signed-off-by: dzdidi --- ...26-08-10-graceful-content-lock-deletion.md | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/plans/2026-08-10-graceful-content-lock-deletion.md b/docs/plans/2026-08-10-graceful-content-lock-deletion.md index 6f6402b..f200d59 100644 --- a/docs/plans/2026-08-10-graceful-content-lock-deletion.md +++ b/docs/plans/2026-08-10-graceful-content-lock-deletion.md @@ -153,7 +153,28 @@ POST /payment-requests/status { "creator": "pubky...", "bundle_id": "..." } ``` -Returns orthogonal `request_state` and `payment_state`, immutable invoice/deadline timestamps, confirmations, and amount match. Exact enum spellings and invalid/recovery-state HTTP mapping are an implementation-contract gate and must be synchronized in both plans before code. +Returns orthogonal `request_state` and `payment_state`, immutable invoice/deadline timestamps, confirmations, and amount match. + +The canonical persisted `request_state` is one of these exact closed snake-case values, mapped one-to-one from Paykit SDK lifecycle state: + +- `proposed` +- `proposal_expired` +- `accepted` +- `rejected` +- `canceled` +- `proof_submitted` +- `active_recurring` +- `recovery_required` +- `invalid_conflict` + +Drain classification uses the persisted state without inference from invoice delivery or Bitcoin observation: + +- `accepted` is accepted and blocking; +- `rejected`, `canceled`, and `proposal_expired` are terminal and non-blocking; +- `proposed` is unanswered and requires durable cancellation enqueue; +- `recovery_required`, `invalid_conflict`, `proof_submitted`, and `active_recurring` fail drain classification rather than being collapsed into another lifecycle. + +For the later HTTP slice, `recovery_required` maps to `503 unavailable`; `invalid_conflict`, `proof_submitted`, and `active_recurring` map to `409 conflict`. These mappings do not alter the canonical lifecycle persisted by this projection. ### Drain cleanup From c1cf97a35fd8dda3c81d7a319fd56ca05a52dcad Mon Sep 17 00:00:00 2001 From: dzdidi Date: Tue, 11 Aug 2026 10:52:42 -0300 Subject: [PATCH 2/6] docs: synchronize payment drain HTTP contract Signed-off-by: dzdidi --- ...26-08-10-graceful-content-lock-deletion.md | 50 ++++++++++++++++--- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/docs/plans/2026-08-10-graceful-content-lock-deletion.md b/docs/plans/2026-08-10-graceful-content-lock-deletion.md index f200d59..eaf5e8b 100644 --- a/docs/plans/2026-08-10-graceful-content-lock-deletion.md +++ b/docs/plans/2026-08-10-graceful-content-lock-deletion.md @@ -144,7 +144,18 @@ POST /payment-request-drain-lookups { "lock_resource": "..." } ``` -Returns aggregate factual state only; no Bundle IDs, readers, Payment Request IDs, addresses, or raw errors. +Both drain endpoints return `200` with the same closed aggregate body: + +```json +{ + "status": "active", + "accepted_count": 0, + "terminal_count": 0, + "cancellation_enqueued_count": 0 +} +``` + +`status` is exactly `active` or `completed`. The response contains no drain ID, replay flag, Bundle ID, reader, Payment Request ID, address, payment reference, or raw error. Exact replay returns the same aggregate body. ### Per-Bundle status @@ -153,7 +164,18 @@ POST /payment-requests/status { "creator": "pubky...", "bundle_id": "..." } ``` -Returns orthogonal `request_state` and `payment_state`, immutable invoice/deadline timestamps, confirmations, and amount match. +Returns this exact closed body with orthogonal lifecycle and payment facts: + +```json +{ + "request_state": "proposed", + "payment_state": "undetected", + "invoice_created_at": "", + "payment_deadline": "", + "confirmations": 0, + "amount_matched": false +} +``` The canonical persisted `request_state` is one of these exact closed snake-case values, mapped one-to-one from Paykit SDK lifecycle state: @@ -167,6 +189,15 @@ The canonical persisted `request_state` is one of these exact closed snake-case - `recovery_required` - `invalid_conflict` +`payment_state` is exactly one of: + +- `undetected` +- `detected` +- `confirmed` +- `expired` + +`expired` is returned when the invoice has a durable `payment_expired_at`; otherwise the persisted observation state maps one-to-one to `undetected`, `detected`, or `confirmed`. `confirmations` and `amount_matched` remain orthogonal factual fields. + Drain classification uses the persisted state without inference from invoice delivery or Bitcoin observation: - `accepted` is accepted and blocking; @@ -176,6 +207,13 @@ Drain classification uses the persisted state without inference from invoice del For the later HTTP slice, `recovery_required` maps to `503 unavailable`; `invalid_conflict`, `proof_submitted`, and `active_recurring` map to `409 conflict`. These mappings do not alter the canonical lifecycle persisted by this projection. +The stable drain-classification error envelopes are: + +- `409 {"error":{"code":"conflict","message":"request conflicts with persisted payment state"}}` +- `503 {"error":{"code":"unavailable","message":"payment request state is unavailable"}}` + +Absent drain lookups and absent per-Bundle statuses reuse `404 {"error":{"code":"not_found","message":"requested resource was not found"}}`. + ### Drain cleanup Paykit Server needs an idempotent signed operation to remove only the completed operational drain row after Locks has completed all external deletion effects. Exact route/body is an implementation-contract gate; it must not remove financial invoice/payment history. @@ -500,11 +538,9 @@ Cross-service acceptance must additionally prove: These do not reopen accepted product semantics, but code must not start for the affected slice until both plans are patched identically: -1. Exact `request_state` and `payment_state` wire enum values and mappings for Paykit recovery/conflict conditions. -2. Exact aggregate drain response fields/status values. -3. Exact signed route/body for deleting completed Paykit operational drain state. -4. Exact Locks stable `failure_code` vocabulary. -5. Exact configuration keys for retry attempts/backoff and final credential windows, within the accepted defaults/maxima. +1. Exact signed route/body for deleting completed Paykit operational drain state. +2. Exact Locks stable `failure_code` vocabulary. +3. Exact configuration keys for retry attempts/backoff and final credential windows, within the accepted defaults/maxima. ## Out of scope From 682868d1193e6a9b986806216bea6bc14224b642 Mon Sep 17 00:00:00 2001 From: dzdidi Date: Wed, 12 Aug 2026 04:57:48 -0300 Subject: [PATCH 3/6] docs: bind payment drain cleanup to opaque tokens Signed-off-by: dzdidi --- .../2026-08-10-graceful-content-lock-deletion.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/plans/2026-08-10-graceful-content-lock-deletion.md b/docs/plans/2026-08-10-graceful-content-lock-deletion.md index eaf5e8b..bf5123c 100644 --- a/docs/plans/2026-08-10-graceful-content-lock-deletion.md +++ b/docs/plans/2026-08-10-graceful-content-lock-deletion.md @@ -214,9 +214,15 @@ The stable drain-classification error envelopes are: Absent drain lookups and absent per-Bundle statuses reuse `404 {"error":{"code":"not_found","message":"requested resource was not found"}}`. -### Drain cleanup +### Operational drain cleanup -Paykit Server needs an idempotent signed operation to remove only the completed operational drain row after Locks has completed all external deletion effects. Exact route/body is an implementation-contract gate; it must not remove financial invoice/payment history. +Drain creation and lookup responses include an opaque `cleanup_token`: the canonical unpadded base64url encoding of 32 server-keyed, domain-separated bytes bound to the immutable drain identity. The token is not an internal drain ID, is not reversible, is stable across restart/exact replay, and must never be logged. Add `POST /payment-request-drain-cleanups`, authenticated by the existing canonical Locks signature boundary. It accepts the exact query-free body: + +```json +{"cleanup_token":"<43-character-unpadded-base64url>","lock_resource":"pubky/pub/locks.app/.json"} +``` + +On success it returns the exact closed response `200 {"status":"removed"}`. Cleanup is cycle-bound and idempotent: deleting the matching completed drain advances the publication generation once and durably retains the consumed token as the generation boundary's cleanup receipt; replay of that token while no newer drain exists returns the same response without advancing again. A token that cannot be verified against either the current drain or its retained cleanup receipt—including an arbitrary token for a never-known lock or a delayed old token after a newer drain exists—returns the existing coarse `409 conflict` envelope and cannot delete or advance the newer cycle. An active matching drain also returns `409 conflict`. Authenticated envelope mismatch, corrupt receipt/generation state, or unavailable persistence returns the existing coarse `503 unavailable` envelope. The operation rejects query strings, unknown body fields, padding, non-canonical base64url, and token lengths other than exactly 32 decoded bytes. It deletes only a completed operational drain after Locks external cleanup succeeds and must never delete invoices, Bitcoin observations, Payment Request events, cancellation intents, or financial audit history. No lock, Bundle, internal drain ID, invoice, reader, or Payment Request identifier appears in the response or error envelope. ## HTTP creator contract @@ -538,9 +544,8 @@ Cross-service acceptance must additionally prove: These do not reopen accepted product semantics, but code must not start for the affected slice until both plans are patched identically: -1. Exact signed route/body for deleting completed Paykit operational drain state. -2. Exact Locks stable `failure_code` vocabulary. -3. Exact configuration keys for retry attempts/backoff and final credential windows, within the accepted defaults/maxima. +1. Exact Locks stable `failure_code` vocabulary. +2. Exact configuration keys for retry attempts/backoff and final credential windows, within the accepted defaults/maxima. ## Out of scope From 3e71f306282b0cd8dfedbbb51a1c7f40dc2ed051 Mon Sep 17 00:00:00 2001 From: dzdidi Date: Wed, 12 Aug 2026 05:53:09 -0300 Subject: [PATCH 4/6] feat(deletion): add durable content lock deletion state Signed-off-by: dzdidi --- ...26-08-10-graceful-content-lock-deletion.md | 5 +- locks-core/src/content_lock_deletion.rs | 83 +++ locks-core/src/lib.rs | 1 + locks-core/tests/content_lock_deletion.rs | 77 ++ locks-server/src/api/errors.rs | 1 + .../0011_content_lock_deletions.sql | 69 ++ locks-service/src/application/errors.rs | 6 + .../models/content_lock_deletion.rs | 176 +++++ locks-service/src/application/models/mod.rs | 2 + .../ports/content_lock_deletion.rs | 81 ++ locks-service/src/application/ports/mod.rs | 2 + .../use_cases/complete_verification_task.rs | 1 + .../memory/content_lock_deletions.rs | 267 +++++++ .../src/infrastructure/memory/mod.rs | 1 + .../postgres/content_lock_deletions.rs | 694 ++++++++++++++++++ .../src/infrastructure/postgres/migrations.rs | 21 + .../src/infrastructure/postgres/mod.rs | 2 + locks-service/tests/content_lock_deletions.rs | 277 +++++++ 18 files changed, 1763 insertions(+), 3 deletions(-) create mode 100644 locks-core/src/content_lock_deletion.rs create mode 100644 locks-core/tests/content_lock_deletion.rs create mode 100644 locks-service/migrations/0011_content_lock_deletions.sql create mode 100644 locks-service/src/application/models/content_lock_deletion.rs create mode 100644 locks-service/src/application/ports/content_lock_deletion.rs create mode 100644 locks-service/src/infrastructure/memory/content_lock_deletions.rs create mode 100644 locks-service/src/infrastructure/postgres/content_lock_deletions.rs create mode 100644 locks-service/tests/content_lock_deletions.rs diff --git a/docs/plans/2026-08-10-graceful-content-lock-deletion.md b/docs/plans/2026-08-10-graceful-content-lock-deletion.md index bf5123c..a818683 100644 --- a/docs/plans/2026-08-10-graceful-content-lock-deletion.md +++ b/docs/plans/2026-08-10-graceful-content-lock-deletion.md @@ -246,7 +246,7 @@ Reject `force=true&graceful=true`, unknown fields, malformed booleans, and dupli GET /creator/content-locks/{lock_id}/deletion ``` -Authenticated response contains Lock ID and `status`; include `failure_code` only for failed jobs. Do not expose phases, leases, retries, Bundle IDs, readers, credentials, paths, Paykit IDs, or dependency errors. +Authenticated response contains Lock ID and `status`; include `failure_code` only for failed jobs. The closed stable vocabulary is exactly `tombstone_missing`, `tombstone_replaced`, `retry_exhausted`, and `state_corrupt`. Do not expose phases, leases, retries, Bundle IDs, readers, credentials, paths, Paykit IDs, or dependency errors. ## Internal state model @@ -544,8 +544,7 @@ Cross-service acceptance must additionally prove: These do not reopen accepted product semantics, but code must not start for the affected slice until both plans are patched identically: -1. Exact Locks stable `failure_code` vocabulary. -2. Exact configuration keys for retry attempts/backoff and final credential windows, within the accepted defaults/maxima. +1. Exact configuration keys for retry attempts/backoff and final credential windows, within the accepted defaults/maxima. ## Out of scope diff --git a/locks-core/src/content_lock_deletion.rs b/locks-core/src/content_lock_deletion.rs new file mode 100644 index 0000000..fddafef --- /dev/null +++ b/locks-core/src/content_lock_deletion.rs @@ -0,0 +1,83 @@ +use serde::{Deserialize, Deserializer, Serialize}; +use time::{OffsetDateTime, UtcOffset}; + +use crate::ids::LockId; + +/// Supported public content-lock deletion tombstone version. +pub const CONTENT_LOCK_DELETION_TOMBSTONE_VERSION: u16 = 1; +const CONTENT_LOCK_DELETION_TOMBSTONE_TYPE: &str = "content_lock_deletion"; + +/// Exact public replacement for a content lock while graceful deletion runs. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContentLockDeletionTombstone { + #[serde(deserialize_with = "deserialize_version")] + version: u16, + #[serde(rename = "type", deserialize_with = "deserialize_type")] + kind: String, + /// Identifier of the withdrawn canonical content lock. + pub lock_id: LockId, + /// Durable proof-admission cutoff, encoded as RFC3339 UTC. + #[serde( + serialize_with = "time::serde::rfc3339::serialize", + deserialize_with = "deserialize_utc_timestamp" + )] + pub deletion_started_at: OffsetDateTime, +} + +impl ContentLockDeletionTombstone { + /// Creates the exact supported tombstone payload. + pub fn new(lock_id: LockId, deletion_started_at: OffsetDateTime) -> Self { + Self { + version: CONTENT_LOCK_DELETION_TOMBSTONE_VERSION, + kind: CONTENT_LOCK_DELETION_TOMBSTONE_TYPE.to_owned(), + lock_id, + deletion_started_at: deletion_started_at.to_offset(UtcOffset::UTC), + } + } + + /// Returns the supported tombstone version. + pub fn version(&self) -> u16 { + self.version + } +} + +fn deserialize_version<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let version = u16::deserialize(deserializer)?; + if version == CONTENT_LOCK_DELETION_TOMBSTONE_VERSION { + Ok(version) + } else { + Err(serde::de::Error::custom( + "unsupported content lock deletion tombstone version", + )) + } +} + +fn deserialize_type<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let kind = String::deserialize(deserializer)?; + if kind == CONTENT_LOCK_DELETION_TOMBSTONE_TYPE { + Ok(kind) + } else { + Err(serde::de::Error::custom( + "unsupported content lock deletion tombstone type", + )) + } +} + +fn deserialize_utc_timestamp<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let timestamp = time::serde::rfc3339::deserialize(deserializer)?; + if timestamp.offset() == UtcOffset::UTC { + Ok(timestamp) + } else { + Err(serde::de::Error::custom("timestamp must use UTC offset Z")) + } +} diff --git a/locks-core/src/lib.rs b/locks-core/src/lib.rs index fca1f54..fc37d7f 100644 --- a/locks-core/src/lib.rs +++ b/locks-core/src/lib.rs @@ -1,3 +1,4 @@ +pub mod content_lock_deletion; pub mod creator_publishing; pub mod ids; pub mod lock_policy; diff --git a/locks-core/tests/content_lock_deletion.rs b/locks-core/tests/content_lock_deletion.rs new file mode 100644 index 0000000..50399ce --- /dev/null +++ b/locks-core/tests/content_lock_deletion.rs @@ -0,0 +1,77 @@ +use std::str::FromStr; + +use locks_core::content_lock_deletion::{ + CONTENT_LOCK_DELETION_TOMBSTONE_VERSION, ContentLockDeletionTombstone, +}; +use locks_core::ids::LockId; +use serde_json::json; +use time::macros::datetime; + +const LOCK_ID: &str = "000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG"; + +#[test] +fn tombstone_serializes_the_exact_closed_protocol_shape() { + let tombstone = ContentLockDeletionTombstone::new( + LockId::from_str(LOCK_ID).unwrap(), + datetime!(2026-08-12 05:00:00 UTC), + ); + + assert_eq!( + serde_json::to_value(&tombstone).unwrap(), + json!({ + "version": CONTENT_LOCK_DELETION_TOMBSTONE_VERSION, + "type": "content_lock_deletion", + "lock_id": LOCK_ID, + "deletion_started_at": "2026-08-12T05:00:00Z", + }) + ); +} + +#[test] +fn tombstone_rejects_unknown_version_type_fields_and_non_utc_time() { + let valid = json!({ + "version": 1, + "type": "content_lock_deletion", + "lock_id": LOCK_ID, + "deletion_started_at": "2026-08-12T05:00:00Z", + }); + assert!(serde_json::from_value::(valid.clone()).is_ok()); + + for invalid in [ + { + let mut value = valid.clone(); + value["version"] = json!(2); + value + }, + { + let mut value = valid.clone(); + value["type"] = json!("content_lock"); + value + }, + { + let mut value = valid.clone(); + value["extra"] = json!(true); + value + }, + { + let mut value = valid; + value["deletion_started_at"] = json!("2026-08-12T06:00:00+01:00"); + value + }, + ] { + assert!(serde_json::from_value::(invalid).is_err()); + } +} + +#[test] +fn tombstone_constructor_normalizes_offsets_to_utc() { + let tombstone = ContentLockDeletionTombstone::new( + LockId::from_str(LOCK_ID).unwrap(), + datetime!(2026-08-12 06:00:00 +01:00), + ); + + assert_eq!( + serde_json::to_value(tombstone).unwrap()["deletion_started_at"], + "2026-08-12T05:00:00Z" + ); +} diff --git a/locks-server/src/api/errors.rs b/locks-server/src/api/errors.rs index 399cc27..475f623 100644 --- a/locks-server/src/api/errors.rs +++ b/locks-server/src/api/errors.rs @@ -229,6 +229,7 @@ impl From for ApiError { Self::new(ApiErrorCode::RateLimited, "rate limit exceeded") } ApplicationError::Storage { .. } + | ApplicationError::InvalidContentLockDeletionState { .. } | ApplicationError::Verifier { .. } | ApplicationError::CredentialGeneration { .. } | ApplicationError::ContentLockCanonicalization { .. } diff --git a/locks-service/migrations/0011_content_lock_deletions.sql b/locks-service/migrations/0011_content_lock_deletions.sql new file mode 100644 index 0000000..418315e --- /dev/null +++ b/locks-service/migrations/0011_content_lock_deletions.sql @@ -0,0 +1,69 @@ +CREATE TABLE content_lock_deletion_jobs ( + job_id UUID PRIMARY KEY, + creator TEXT NOT NULL, + lock_id TEXT NOT NULL, + frozen_content_lock JSONB NOT NULL, + deletion_started_at TIMESTAMPTZ NOT NULL, + state TEXT NOT NULL DEFAULT 'queued', + phase TEXT NOT NULL DEFAULT 'withdraw', + attempt_count BIGINT NOT NULL DEFAULT 0, + next_attempt_at TIMESTAMPTZ, + force_requested_at TIMESTAMPTZ, + failure_code TEXT, + claimed_by TEXT, + claim_token UUID, + claim_expires_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT content_lock_deletion_jobs_creator_lock_unique UNIQUE (creator, lock_id), + CONSTRAINT content_lock_deletion_jobs_state_valid CHECK ( + state IN ('queued', 'running', 'completed', 'failed') + ), + CONSTRAINT content_lock_deletion_jobs_phase_valid CHECK ( + phase IN ( + 'withdraw', + 'start_payment_drain', + 'drain_payments', + 'drain_existing_credentials', + 'issue_final_credentials', + 'drain_final_reads', + 'delete_content', + 'delete_tombstone', + 'purge_operational_state' + ) + ), + CONSTRAINT content_lock_deletion_jobs_attempt_count_valid CHECK (attempt_count >= 0), + CONSTRAINT content_lock_deletion_jobs_claim_valid CHECK ( + (state = 'running' + AND claimed_by IS NOT NULL + AND claim_token IS NOT NULL + AND claim_expires_at IS NOT NULL + AND next_attempt_at IS NULL) + OR + (state <> 'running' + AND claimed_by IS NULL + AND claim_token IS NULL + AND claim_expires_at IS NULL) + ), + CONSTRAINT content_lock_deletion_jobs_failure_valid CHECK ( + (state = 'failed' AND failure_code IN ( + 'tombstone_missing', + 'tombstone_replaced', + 'retry_exhausted', + 'state_corrupt' + )) + OR + (state <> 'failed' AND failure_code IS NULL) + ) +); + +CREATE INDEX content_lock_deletion_jobs_due_idx + ON content_lock_deletion_jobs (deletion_started_at) + WHERE state IN ('queued', 'running'); + +CREATE TABLE content_lock_force_deletion_receipts ( + creator TEXT NOT NULL, + lock_id TEXT NOT NULL, + forced_at TIMESTAMPTZ NOT NULL, + CONSTRAINT content_lock_force_deletion_receipts_pkey PRIMARY KEY (creator, lock_id) +); diff --git a/locks-service/src/application/errors.rs b/locks-service/src/application/errors.rs index 98e7ed9..6815be6 100644 --- a/locks-service/src/application/errors.rs +++ b/locks-service/src/application/errors.rs @@ -18,6 +18,12 @@ pub enum ApplicationError { /// Full creator-scoped guarded path for structured internal handling. guarded_path: String, }, + /// Persisted content-lock deletion state violates its internal invariants. + #[error("invalid content lock deletion state: {message}")] + InvalidContentLockDeletionState { + /// Secret-free invariant failure detail. + message: String, + }, /// An update-only operation targeted a missing record. #[error("missing {record} record")] MissingRecord { diff --git a/locks-service/src/application/models/content_lock_deletion.rs b/locks-service/src/application/models/content_lock_deletion.rs new file mode 100644 index 0000000..9f61ebc --- /dev/null +++ b/locks-service/src/application/models/content_lock_deletion.rs @@ -0,0 +1,176 @@ +use std::str::FromStr; + +use locks_core::{ + ids::{CreatorPubky, LockId}, + lock_policy::ContentLock, +}; +use time::OffsetDateTime; +use uuid::Uuid; + +use crate::application::errors::ApplicationError; + +/// Internal deletion workflow state. Public API status conversion is intentionally separate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ContentLockDeletionState { + Queued, + Running, + Completed, + Failed, +} + +/// Closed creator-visible failure vocabulary. Raw dependency errors never cross this boundary. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ContentLockDeletionFailureCode { + TombstoneMissing, + TombstoneReplaced, + RetryExhausted, + StateCorrupt, +} + +impl ContentLockDeletionFailureCode { + /// Returns the exact stable public/database value. + pub fn as_str(self) -> &'static str { + match self { + Self::TombstoneMissing => "tombstone_missing", + Self::TombstoneReplaced => "tombstone_replaced", + Self::RetryExhausted => "retry_exhausted", + Self::StateCorrupt => "state_corrupt", + } + } +} + +impl FromStr for ContentLockDeletionFailureCode { + type Err = ApplicationError; + + fn from_str(value: &str) -> Result { + match value { + "tombstone_missing" => Ok(Self::TombstoneMissing), + "tombstone_replaced" => Ok(Self::TombstoneReplaced), + "retry_exhausted" => Ok(Self::RetryExhausted), + "state_corrupt" => Ok(Self::StateCorrupt), + _ => Err(ApplicationError::InvalidContentLockDeletionState { + message: "unknown content lock deletion failure code".to_owned(), + }), + } + } +} + +/// Internal orchestration phase. These values are not public API. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ContentLockDeletionPhase { + Withdraw, + StartPaymentDrain, + DrainPayments, + DrainExistingCredentials, + IssueFinalCredentials, + DrainFinalReads, + DeleteContent, + DeleteTombstone, + PurgeOperationalState, +} + +impl ContentLockDeletionPhase { + /// Returns true only for the immediate forward workflow transition. + pub fn permits(self, next: Self) -> bool { + matches!( + (self, next), + (Self::Withdraw, Self::StartPaymentDrain) + | (Self::StartPaymentDrain, Self::DrainPayments) + | (Self::DrainPayments, Self::DrainExistingCredentials) + | (Self::DrainExistingCredentials, Self::IssueFinalCredentials) + | (Self::IssueFinalCredentials, Self::DrainFinalReads) + | (Self::DrainFinalReads, Self::DeleteContent) + | (Self::DeleteContent, Self::DeleteTombstone) + | (Self::DeleteTombstone, Self::PurgeOperationalState) + ) + } +} + +/// Durable graceful content-lock deletion job and immutable frozen manifest. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContentLockDeletionJob { + pub job_id: Uuid, + pub creator: CreatorPubky, + pub lock_id: LockId, + pub frozen_content_lock: ContentLock, + pub deletion_started_at: OffsetDateTime, + pub state: ContentLockDeletionState, + pub phase: ContentLockDeletionPhase, + pub attempt_count: u32, + pub next_attempt_at: Option, + pub force_requested_at: Option, + pub failure_code: Option, +} + +/// Claimed job plus the fresh lease-incarnation token required for fenced writes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ClaimedContentLockDeletionJob { + pub job: ContentLockDeletionJob, + pub claim_token: Uuid, +} + +impl ContentLockDeletionJob { + /// Creates a queued deletion job from a canonical frozen content lock. + pub fn new( + job_id: Uuid, + frozen_content_lock: ContentLock, + deletion_started_at: OffsetDateTime, + ) -> Result { + let lock_id = frozen_content_lock.lock_id().map_err(|error| { + ApplicationError::ContentLockCanonicalization { + message: error.to_string(), + } + })?; + Ok(Self { + job_id, + creator: frozen_content_lock.creator.clone(), + lock_id, + frozen_content_lock, + deletion_started_at, + state: ContentLockDeletionState::Queued, + phase: ContentLockDeletionPhase::Withdraw, + attempt_count: 0, + next_attempt_at: None, + force_requested_at: None, + failure_code: None, + }) + } + + /// Recomputes the frozen lock identity and verifies the durable key fields. + pub fn validate_frozen_identity(&self) -> Result<(), ApplicationError> { + let actual = self.frozen_content_lock.lock_id().map_err(|error| { + ApplicationError::ContentLockCanonicalization { + message: error.to_string(), + } + })?; + if actual != self.lock_id || self.frozen_content_lock.creator != self.creator { + return Err(ApplicationError::InvalidContentLockDeletionState { + message: "frozen content lock identity does not match deletion job".to_owned(), + }); + } + Ok(()) + } + + /// Validates lifecycle fields against whether persistence has a complete active lease. + pub fn validate_state(&self, has_active_lease: bool) -> Result<(), ApplicationError> { + let valid = match self.state { + ContentLockDeletionState::Queued => !has_active_lease && self.failure_code.is_none(), + ContentLockDeletionState::Running => { + has_active_lease && self.next_attempt_at.is_none() && self.failure_code.is_none() + } + ContentLockDeletionState::Completed => { + !has_active_lease && self.next_attempt_at.is_none() && self.failure_code.is_none() + } + ContentLockDeletionState::Failed => { + !has_active_lease && self.next_attempt_at.is_none() && self.failure_code.is_some() + } + }; + if valid { + Ok(()) + } else { + Err(ApplicationError::InvalidContentLockDeletionState { + message: "deletion lifecycle fields are inconsistent".to_owned(), + }) + } + } +} diff --git a/locks-service/src/application/models/mod.rs b/locks-service/src/application/models/mod.rs index ed27c07..907bc24 100644 --- a/locks-service/src/application/models/mod.rs +++ b/locks-service/src/application/models/mod.rs @@ -1,4 +1,5 @@ mod access; +mod content_lock_deletion; mod content_lock_ownership; mod creator_authority; mod frontend_session; @@ -6,6 +7,7 @@ mod guarded_resource; mod verification; pub use access::*; +pub use content_lock_deletion::*; pub use content_lock_ownership::*; pub use creator_authority::*; pub use frontend_session::*; diff --git a/locks-service/src/application/ports/content_lock_deletion.rs b/locks-service/src/application/ports/content_lock_deletion.rs new file mode 100644 index 0000000..f7770d5 --- /dev/null +++ b/locks-service/src/application/ports/content_lock_deletion.rs @@ -0,0 +1,81 @@ +use async_trait::async_trait; +use locks_core::ids::{CreatorPubky, LockId}; +use time::OffsetDateTime; +use uuid::Uuid; + +use crate::application::{ + errors::ApplicationError, + models::{ + ClaimedContentLockDeletionJob, ContentLockDeletionFailureCode, ContentLockDeletionJob, + ContentLockDeletionPhase, + }, +}; + +/// Durable repository and fenced worker lease boundary for content-lock deletion jobs. +#[async_trait] +pub trait ContentLockDeletionRepository: Send + Sync { + async fn insert_job(&self, job: ContentLockDeletionJob) -> Result<(), ApplicationError>; + + async fn get_job( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result, ApplicationError>; + + async fn claim_next( + &self, + worker_id: &str, + now: OffsetDateTime, + claim_expires_at: OffsetDateTime, + ) -> Result, ApplicationError>; + + async fn schedule_retry( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + next_attempt_at: OffsetDateTime, + ) -> Result, ApplicationError>; + + async fn advance_phase( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + next_phase: ContentLockDeletionPhase, + ) -> Result, ApplicationError>; + + /// Persists terminal completion or a stable secret-free failure under the exact lease. + async fn finish( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + failure_code: Option, + ) -> Result, ApplicationError>; + + /// Permanently records force escalation. Returns true only on the first request. + async fn request_force( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + requested_at: OffsetDateTime, + ) -> Result; + + /// Idempotently records the permanent minimal force-deletion receipt. + async fn record_force_receipt( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + forced_at: OffsetDateTime, + ) -> Result<(), ApplicationError>; + + async fn has_force_receipt( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result; +} diff --git a/locks-service/src/application/ports/mod.rs b/locks-service/src/application/ports/mod.rs index fff16d3..97ecdbf 100644 --- a/locks-service/src/application/ports/mod.rs +++ b/locks-service/src/application/ports/mod.rs @@ -1,6 +1,7 @@ pub mod semantics {} mod access; +mod content_lock_deletion; mod content_lock_ownership; mod creator_authority; mod entitlement; @@ -10,6 +11,7 @@ mod runtime; mod verification; pub use access::*; +pub use content_lock_deletion::*; pub use content_lock_ownership::*; pub use creator_authority::*; pub use entitlement::*; diff --git a/locks-service/src/application/use_cases/complete_verification_task.rs b/locks-service/src/application/use_cases/complete_verification_task.rs index deb1571..c8ec7c2 100644 --- a/locks-service/src/application/use_cases/complete_verification_task.rs +++ b/locks-service/src/application/use_cases/complete_verification_task.rs @@ -357,6 +357,7 @@ fn viewer_safe_failure_message(error: &ApplicationError) -> &'static str { ApplicationError::Storage { .. } | ApplicationError::DuplicateRecord { .. } | ApplicationError::ContentLockPathConflict { .. } + | ApplicationError::InvalidContentLockDeletionState { .. } | ApplicationError::MissingRecord { .. } | ApplicationError::InvalidVerificationTaskTransition { .. } | ApplicationError::VerificationPending diff --git a/locks-service/src/infrastructure/memory/content_lock_deletions.rs b/locks-service/src/infrastructure/memory/content_lock_deletions.rs new file mode 100644 index 0000000..238ee31 --- /dev/null +++ b/locks-service/src/infrastructure/memory/content_lock_deletions.rs @@ -0,0 +1,267 @@ +use std::collections::{HashMap, HashSet}; + +use async_trait::async_trait; +use locks_core::ids::{CreatorPubky, LockId}; +use time::OffsetDateTime; +use tokio::sync::RwLock; +use uuid::Uuid; + +use crate::application::{ + errors::ApplicationError, + models::{ + ClaimedContentLockDeletionJob, ContentLockDeletionFailureCode, ContentLockDeletionJob, + ContentLockDeletionPhase, ContentLockDeletionState, + }, + ports::ContentLockDeletionRepository, +}; + +type JobKey = (CreatorPubky, LockId); + +#[derive(Debug, Clone)] +struct StoredJob { + job: ContentLockDeletionJob, + claimed_by: Option, + claim_token: Option, + claim_expires_at: Option, +} + +/// In-memory deletion repository with the same lease-fencing semantics as PostgreSQL. +#[derive(Debug, Default)] +pub struct InMemoryContentLockDeletionRepository { + jobs: RwLock>, + force_receipts: RwLock>, +} + +impl InMemoryContentLockDeletionRepository { + pub fn new() -> Self { + Self::default() + } +} + +#[async_trait] +impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { + async fn insert_job(&self, job: ContentLockDeletionJob) -> Result<(), ApplicationError> { + job.validate_frozen_identity()?; + job.validate_state(false)?; + let key = (job.creator.clone(), job.lock_id.clone()); + let mut jobs = self.jobs.write().await; + if jobs.contains_key(&key) || jobs.values().any(|stored| stored.job.job_id == job.job_id) { + return Err(ApplicationError::DuplicateRecord { + record: "content_lock_deletion_job", + }); + } + jobs.insert( + key, + StoredJob { + job, + claimed_by: None, + claim_token: None, + claim_expires_at: None, + }, + ); + Ok(()) + } + + async fn get_job( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result, ApplicationError> { + let stored = self + .jobs + .read() + .await + .get(&(creator.clone(), lock_id.clone())) + .cloned(); + if let Some(stored) = stored { + stored.job.validate_frozen_identity()?; + let has_active_lease = stored.claimed_by.is_some() + && stored.claim_token.is_some() + && stored.claim_expires_at.is_some(); + stored.job.validate_state(has_active_lease)?; + Ok(Some(stored.job)) + } else { + Ok(None) + } + } + + async fn claim_next( + &self, + worker_id: &str, + now: OffsetDateTime, + claim_expires_at: OffsetDateTime, + ) -> Result, ApplicationError> { + let mut jobs = self.jobs.write().await; + let Some(stored) = jobs + .values_mut() + .filter(|stored| is_claimable(stored, now)) + .min_by_key(|stored| stored.job.deletion_started_at) + else { + return Ok(None); + }; + let claim_token = Uuid::new_v4(); + stored.job.state = ContentLockDeletionState::Running; + stored.job.attempt_count = stored.job.attempt_count.saturating_add(1); + stored.job.next_attempt_at = None; + stored.claimed_by = Some(worker_id.to_owned()); + stored.claim_token = Some(claim_token); + stored.claim_expires_at = Some(claim_expires_at); + Ok(Some(ClaimedContentLockDeletionJob { + job: stored.job.clone(), + claim_token, + })) + } + + async fn schedule_retry( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + next_attempt_at: OffsetDateTime, + ) -> Result, ApplicationError> { + let mut jobs = self.jobs.write().await; + let Some(stored) = jobs + .values_mut() + .find(|stored| owns_claim(stored, job_id, worker_id, claim_token, now)) + else { + return Ok(None); + }; + stored.job.state = ContentLockDeletionState::Queued; + stored.job.next_attempt_at = Some(next_attempt_at); + clear_claim(stored); + Ok(Some(stored.job.clone())) + } + + async fn advance_phase( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + next_phase: ContentLockDeletionPhase, + ) -> Result, ApplicationError> { + let mut jobs = self.jobs.write().await; + let Some(stored) = jobs + .values_mut() + .find(|stored| owns_claim(stored, job_id, worker_id, claim_token, now)) + else { + return Ok(None); + }; + if !stored.job.phase.permits(next_phase) { + return Err(ApplicationError::InvalidContentLockDeletionState { + message: "deletion phase must advance to its immediate successor".to_owned(), + }); + } + stored.job.phase = next_phase; + stored.job.state = ContentLockDeletionState::Queued; + stored.job.attempt_count = 0; + stored.job.next_attempt_at = None; + stored.job.failure_code = None; + clear_claim(stored); + Ok(Some(stored.job.clone())) + } + + async fn finish( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + failure_code: Option, + ) -> Result, ApplicationError> { + let mut jobs = self.jobs.write().await; + let Some(stored) = jobs + .values_mut() + .find(|stored| owns_claim(stored, job_id, worker_id, claim_token, now)) + else { + return Ok(None); + }; + stored.job.state = if failure_code.is_some() { + ContentLockDeletionState::Failed + } else { + ContentLockDeletionState::Completed + }; + stored.job.failure_code = failure_code; + stored.job.next_attempt_at = None; + clear_claim(stored); + Ok(Some(stored.job.clone())) + } + + async fn request_force( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + requested_at: OffsetDateTime, + ) -> Result { + let mut jobs = self.jobs.write().await; + let Some(stored) = jobs.get_mut(&(creator.clone(), lock_id.clone())) else { + return Ok(false); + }; + if stored.job.force_requested_at.is_some() { + return Ok(false); + } + stored.job.force_requested_at = Some(requested_at); + Ok(true) + } + + async fn record_force_receipt( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + _forced_at: OffsetDateTime, + ) -> Result<(), ApplicationError> { + self.force_receipts + .write() + .await + .insert((creator.clone(), lock_id.clone())); + Ok(()) + } + + async fn has_force_receipt( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result { + Ok(self + .force_receipts + .read() + .await + .contains(&(creator.clone(), lock_id.clone()))) + } +} + +fn is_claimable(stored: &StoredJob, now: OffsetDateTime) -> bool { + match stored.job.state { + ContentLockDeletionState::Queued => stored + .job + .next_attempt_at + .is_none_or(|next_attempt_at| next_attempt_at <= now), + ContentLockDeletionState::Running => stored + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at < now), + ContentLockDeletionState::Completed | ContentLockDeletionState::Failed => false, + } +} + +fn owns_claim( + stored: &StoredJob, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, +) -> bool { + stored.job.job_id == job_id + && stored.job.state == ContentLockDeletionState::Running + && stored.claimed_by.as_deref() == Some(worker_id) + && stored.claim_token == Some(claim_token) + && stored + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at >= now) +} + +fn clear_claim(stored: &mut StoredJob) { + stored.claimed_by = None; + stored.claim_token = None; + stored.claim_expires_at = None; +} diff --git a/locks-service/src/infrastructure/memory/mod.rs b/locks-service/src/infrastructure/memory/mod.rs index 83b00d7..d1a1e28 100644 --- a/locks-service/src/infrastructure/memory/mod.rs +++ b/locks-service/src/infrastructure/memory/mod.rs @@ -1,4 +1,5 @@ pub mod access_credentials; +pub mod content_lock_deletions; pub mod content_lock_ownership; pub mod content_locks; pub mod entitlements; diff --git a/locks-service/src/infrastructure/postgres/content_lock_deletions.rs b/locks-service/src/infrastructure/postgres/content_lock_deletions.rs new file mode 100644 index 0000000..3a94c7c --- /dev/null +++ b/locks-service/src/infrastructure/postgres/content_lock_deletions.rs @@ -0,0 +1,694 @@ +use std::str::FromStr; + +use async_trait::async_trait; +use locks_core::{ + ids::{CreatorPubky, LockId}, + lock_policy::ContentLock, +}; +use sqlx::{FromRow, PgPool, Postgres, Transaction}; +use time::OffsetDateTime; +use uuid::Uuid; + +use crate::application::{ + errors::ApplicationError, + models::{ + ClaimedContentLockDeletionJob, ContentLockDeletionFailureCode, ContentLockDeletionJob, + ContentLockDeletionPhase, ContentLockDeletionState, + }, + ports::ContentLockDeletionRepository, +}; + +const ROW_COLUMNS: &str = "job_id, creator, lock_id, frozen_content_lock, deletion_started_at, state, phase, attempt_count, next_attempt_at, force_requested_at, failure_code, claimed_by, claim_token, claim_expires_at"; + +#[derive(Debug, FromRow)] +struct DeletionJobRow { + job_id: Uuid, + creator: String, + lock_id: String, + frozen_content_lock: serde_json::Value, + deletion_started_at: OffsetDateTime, + state: String, + phase: String, + attempt_count: i64, + next_attempt_at: Option, + force_requested_at: Option, + failure_code: Option, + claimed_by: Option, + claim_token: Option, + claim_expires_at: Option, +} + +/// PostgreSQL-backed durable deletion job queue and permanent force receipt store. +#[derive(Debug, Clone)] +pub struct PostgresContentLockDeletionRepository { + pool: PgPool, +} + +impl PostgresContentLockDeletionRepository { + pub fn new(pool: PgPool) -> Self { + Self { pool } + } +} + +#[async_trait] +impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { + async fn insert_job(&self, job: ContentLockDeletionJob) -> Result<(), ApplicationError> { + job.validate_frozen_identity()?; + job.validate_state(false)?; + let frozen = serde_json::to_value(&job.frozen_content_lock).map_err(storage_display)?; + sqlx::query( + "INSERT INTO content_lock_deletion_jobs + (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, state, phase, + attempt_count, next_attempt_at, force_requested_at, failure_code) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", + ) + .bind(job.job_id) + .bind(job.creator.to_string()) + .bind(job.lock_id.to_string()) + .bind(frozen) + .bind(job.deletion_started_at) + .bind(state_to_database(job.state)) + .bind(phase_to_database(job.phase)) + .bind(i64::from(job.attempt_count)) + .bind(job.next_attempt_at) + .bind(job.force_requested_at) + .bind(job.failure_code.map(ContentLockDeletionFailureCode::as_str)) + .execute(&self.pool) + .await + .map_err(map_insert_error)?; + Ok(()) + } + + async fn get_job( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result, ApplicationError> { + let sql = format!( + "SELECT {ROW_COLUMNS} FROM content_lock_deletion_jobs WHERE creator = $1 AND lock_id = $2" + ); + sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .fetch_optional(&self.pool) + .await + .map_err(storage_error)? + .map(row_to_job) + .transpose() + } + + async fn claim_next( + &self, + worker_id: &str, + now: OffsetDateTime, + claim_expires_at: OffsetDateTime, + ) -> Result, ApplicationError> { + let claim_token = Uuid::new_v4(); + let sql = format!( + "UPDATE content_lock_deletion_jobs + SET state = 'running', claimed_by = $1, claim_token = $2, + claim_expires_at = $3, next_attempt_at = NULL, + attempt_count = attempt_count + 1, updated_at = $4 + WHERE job_id = ( + SELECT job_id FROM content_lock_deletion_jobs + WHERE (state = 'queued' AND (next_attempt_at IS NULL OR next_attempt_at <= $4)) + OR (state = 'running' AND claim_expires_at < $4) + ORDER BY deletion_started_at + FOR UPDATE SKIP LOCKED LIMIT 1 + ) + RETURNING {ROW_COLUMNS}" + ); + let row = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(worker_id) + .bind(claim_token) + .bind(claim_expires_at) + .bind(now) + .fetch_optional(&self.pool) + .await + .map_err(storage_error)?; + row.map(row_to_job) + .transpose() + .map(|job| job.map(|job| ClaimedContentLockDeletionJob { job, claim_token })) + } + + async fn schedule_retry( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + next_attempt_at: OffsetDateTime, + ) -> Result, ApplicationError> { + let sql = format!( + "UPDATE content_lock_deletion_jobs + SET state = 'queued', next_attempt_at = $5, claimed_by = NULL, + claim_token = NULL, claim_expires_at = NULL, updated_at = $4 + WHERE job_id = $1 AND state = 'running' AND claimed_by = $2 + AND claim_token = $3 AND claim_expires_at >= $4 + RETURNING {ROW_COLUMNS}" + ); + fetch_optional_job( + sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(job_id) + .bind(worker_id) + .bind(claim_token) + .bind(now) + .bind(next_attempt_at) + .fetch_optional(&self.pool) + .await + .map_err(storage_error)?, + ) + } + + async fn advance_phase( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + next_phase: ContentLockDeletionPhase, + ) -> Result, ApplicationError> { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let current = + load_owned_claim(&mut transaction, job_id, worker_id, claim_token, now).await?; + let Some(current) = current else { + transaction.rollback().await.map_err(storage_error)?; + return Ok(None); + }; + if !current.phase.permits(next_phase) { + return Err(ApplicationError::InvalidContentLockDeletionState { + message: "deletion phase must advance to its immediate successor".to_owned(), + }); + } + let sql = format!( + "UPDATE content_lock_deletion_jobs + SET phase = $2, state = 'queued', attempt_count = 0, next_attempt_at = NULL, + failure_code = NULL, claimed_by = NULL, claim_token = NULL, + claim_expires_at = NULL, updated_at = $3 + WHERE job_id = $1 RETURNING {ROW_COLUMNS}" + ); + let updated = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(job_id) + .bind(phase_to_database(next_phase)) + .bind(now) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(Some(row_to_job(updated)?)) + } + + async fn finish( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + failure_code: Option, + ) -> Result, ApplicationError> { + let state = if failure_code.is_some() { + "failed" + } else { + "completed" + }; + let sql = format!( + "UPDATE content_lock_deletion_jobs + SET state = $5, failure_code = $6, next_attempt_at = NULL, + claimed_by = NULL, claim_token = NULL, claim_expires_at = NULL, updated_at = $4 + WHERE job_id = $1 AND state = 'running' AND claimed_by = $2 + AND claim_token = $3 AND claim_expires_at >= $4 + RETURNING {ROW_COLUMNS}" + ); + fetch_optional_job( + sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(job_id) + .bind(worker_id) + .bind(claim_token) + .bind(now) + .bind(state) + .bind(failure_code.map(ContentLockDeletionFailureCode::as_str)) + .fetch_optional(&self.pool) + .await + .map_err(storage_error)?, + ) + } + + async fn request_force( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + requested_at: OffsetDateTime, + ) -> Result { + let result = sqlx::query( + "UPDATE content_lock_deletion_jobs SET force_requested_at = $3, updated_at = $3 + WHERE creator = $1 AND lock_id = $2 AND force_requested_at IS NULL", + ) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .bind(requested_at) + .execute(&self.pool) + .await + .map_err(storage_error)?; + Ok(result.rows_affected() == 1) + } + + async fn record_force_receipt( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + forced_at: OffsetDateTime, + ) -> Result<(), ApplicationError> { + sqlx::query( + "INSERT INTO content_lock_force_deletion_receipts (creator, lock_id, forced_at) + VALUES ($1, $2, $3) ON CONFLICT (creator, lock_id) DO NOTHING", + ) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .bind(forced_at) + .execute(&self.pool) + .await + .map_err(storage_error)?; + Ok(()) + } + + async fn has_force_receipt( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result { + sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM content_lock_force_deletion_receipts + WHERE creator = $1 AND lock_id = $2)", + ) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .fetch_one(&self.pool) + .await + .map_err(storage_error) + } +} + +async fn load_owned_claim( + transaction: &mut Transaction<'_, Postgres>, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, +) -> Result, ApplicationError> { + let sql = format!( + "SELECT {ROW_COLUMNS} FROM content_lock_deletion_jobs + WHERE job_id = $1 AND state = 'running' AND claimed_by = $2 + AND claim_token = $3 AND claim_expires_at >= $4 FOR UPDATE" + ); + sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(job_id) + .bind(worker_id) + .bind(claim_token) + .bind(now) + .fetch_optional(&mut **transaction) + .await + .map_err(storage_error)? + .map(row_to_job) + .transpose() +} + +fn fetch_optional_job( + row: Option, +) -> Result, ApplicationError> { + row.map(row_to_job).transpose() +} + +fn row_to_job(row: DeletionJobRow) -> Result { + let has_active_lease = match ( + row.claimed_by.is_some(), + row.claim_token.is_some(), + row.claim_expires_at.is_some(), + ) { + (false, false, false) => false, + (true, true, true) => true, + _ => return Err(invalid_state("deletion lease fields are inconsistent")), + }; + let job = ContentLockDeletionJob { + job_id: row.job_id, + creator: CreatorPubky::from_str(&row.creator).map_err(storage_display)?, + lock_id: LockId::from_str(&row.lock_id).map_err(storage_display)?, + frozen_content_lock: serde_json::from_value::(row.frozen_content_lock) + .map_err(storage_display)?, + deletion_started_at: row.deletion_started_at, + state: state_from_database(&row.state)?, + phase: phase_from_database(&row.phase)?, + attempt_count: u32::try_from(row.attempt_count).map_err(storage_display)?, + next_attempt_at: row.next_attempt_at, + force_requested_at: row.force_requested_at, + failure_code: row + .failure_code + .map(|code| code.parse::()) + .transpose()?, + }; + job.validate_frozen_identity()?; + job.validate_state(has_active_lease)?; + Ok(job) +} + +fn state_to_database(state: ContentLockDeletionState) -> &'static str { + match state { + ContentLockDeletionState::Queued => "queued", + ContentLockDeletionState::Running => "running", + ContentLockDeletionState::Completed => "completed", + ContentLockDeletionState::Failed => "failed", + } +} + +fn state_from_database(value: &str) -> Result { + match value { + "queued" => Ok(ContentLockDeletionState::Queued), + "running" => Ok(ContentLockDeletionState::Running), + "completed" => Ok(ContentLockDeletionState::Completed), + "failed" => Ok(ContentLockDeletionState::Failed), + _ => Err(invalid_state("unknown deletion state")), + } +} + +fn phase_to_database(phase: ContentLockDeletionPhase) -> &'static str { + match phase { + ContentLockDeletionPhase::Withdraw => "withdraw", + ContentLockDeletionPhase::StartPaymentDrain => "start_payment_drain", + ContentLockDeletionPhase::DrainPayments => "drain_payments", + ContentLockDeletionPhase::DrainExistingCredentials => "drain_existing_credentials", + ContentLockDeletionPhase::IssueFinalCredentials => "issue_final_credentials", + ContentLockDeletionPhase::DrainFinalReads => "drain_final_reads", + ContentLockDeletionPhase::DeleteContent => "delete_content", + ContentLockDeletionPhase::DeleteTombstone => "delete_tombstone", + ContentLockDeletionPhase::PurgeOperationalState => "purge_operational_state", + } +} + +fn phase_from_database(value: &str) -> Result { + match value { + "withdraw" => Ok(ContentLockDeletionPhase::Withdraw), + "start_payment_drain" => Ok(ContentLockDeletionPhase::StartPaymentDrain), + "drain_payments" => Ok(ContentLockDeletionPhase::DrainPayments), + "drain_existing_credentials" => Ok(ContentLockDeletionPhase::DrainExistingCredentials), + "issue_final_credentials" => Ok(ContentLockDeletionPhase::IssueFinalCredentials), + "drain_final_reads" => Ok(ContentLockDeletionPhase::DrainFinalReads), + "delete_content" => Ok(ContentLockDeletionPhase::DeleteContent), + "delete_tombstone" => Ok(ContentLockDeletionPhase::DeleteTombstone), + "purge_operational_state" => Ok(ContentLockDeletionPhase::PurgeOperationalState), + _ => Err(invalid_state("unknown deletion phase")), + } +} + +fn invalid_state(message: &str) -> ApplicationError { + ApplicationError::InvalidContentLockDeletionState { + message: message.to_owned(), + } +} + +fn map_insert_error(error: sqlx::Error) -> ApplicationError { + if error + .as_database_error() + .is_some_and(|error| error.is_unique_violation()) + { + ApplicationError::DuplicateRecord { + record: "content_lock_deletion_job", + } + } else { + storage_error(error) + } +} + +fn storage_error(error: sqlx::Error) -> ApplicationError { + storage_display(error) +} + +fn storage_display(error: impl std::fmt::Display) -> ApplicationError { + ApplicationError::Storage { + message: error.to_string(), + } +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, str::FromStr}; + + use locks_core::{ + ids::{CreatorPubky, GuardedResourceHash}, + lock_policy::{ + AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, GuardedResource, LockLogic, + LockServerConfig, + }, + }; + use time::macros::datetime; + use uuid::Uuid; + + use super::PostgresContentLockDeletionRepository; + use crate::{ + application::{ + models::{ + ContentLockDeletionFailureCode, ContentLockDeletionJob, ContentLockDeletionPhase, + ContentLockDeletionState, + }, + ports::ContentLockDeletionRepository, + }, + infrastructure::postgres::testing::TestDatabase, + }; + + const CREATOR: &str = "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy"; + const NOW: time::OffsetDateTime = datetime!(2026-08-12 05:00:00 UTC); + + #[tokio::test] + async fn persists_and_fences_the_full_job_lifecycle_across_repository_recreation() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + + let reopened = PostgresContentLockDeletionRepository::new(database.pool().clone()); + assert_eq!( + reopened.get_job(&job.creator, &job.lock_id).await.unwrap(), + Some(job.clone()) + ); + assert!(reopened.insert_job(job.clone()).await.is_err()); + let mut distinct_lock = content_lock(); + distinct_lock.access_policy.requested_credential_ttl_seconds = 901; + let mut distinct_job = + ContentLockDeletionJob::new(Uuid::new_v4(), distinct_lock, NOW).unwrap(); + distinct_job.job_id = job.job_id; + assert!(reopened.insert_job(distinct_job).await.is_err()); + + let first = reopened + .claim_next("worker-a", NOW, datetime!(2026-08-12 05:05:00 UTC)) + .await + .unwrap() + .unwrap(); + assert!( + reopened + .claim_next("worker-b", NOW, datetime!(2026-08-12 05:05:00 UTC),) + .await + .unwrap() + .is_none() + ); + let reclaimed = reopened + .claim_next( + "worker-b", + datetime!(2026-08-12 05:05:01 UTC), + datetime!(2026-08-12 05:10:00 UTC), + ) + .await + .unwrap() + .unwrap(); + assert_ne!(first.claim_token, reclaimed.claim_token); + assert!( + reopened + .schedule_retry( + job.job_id, + "worker-a", + first.claim_token, + datetime!(2026-08-12 05:05:01 UTC), + datetime!(2026-08-12 05:06:00 UTC), + ) + .await + .unwrap() + .is_none() + ); + assert!( + reopened + .advance_phase( + job.job_id, + "worker-a", + first.claim_token, + datetime!(2026-08-12 05:05:01 UTC), + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .is_none() + ); + assert!( + reopened + .finish( + job.job_id, + "worker-a", + first.claim_token, + datetime!(2026-08-12 05:05:01 UTC), + Some(ContentLockDeletionFailureCode::StateCorrupt), + ) + .await + .unwrap() + .is_none() + ); + + let advanced = reopened + .advance_phase( + job.job_id, + "worker-b", + reclaimed.claim_token, + datetime!(2026-08-12 05:06:00 UTC), + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(advanced.state, ContentLockDeletionState::Queued); + assert_eq!(advanced.attempt_count, 0); + + let final_claim = reopened + .claim_next( + "worker-c", + datetime!(2026-08-12 05:06:01 UTC), + datetime!(2026-08-12 05:11:00 UTC), + ) + .await + .unwrap() + .unwrap(); + let failed = reopened + .finish( + job.job_id, + "worker-c", + final_claim.claim_token, + datetime!(2026-08-12 05:07:00 UTC), + Some(ContentLockDeletionFailureCode::TombstoneMissing), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(failed.state, ContentLockDeletionState::Failed); + assert_eq!( + failed.failure_code, + Some(ContentLockDeletionFailureCode::TombstoneMissing) + ); + + assert!( + reopened + .request_force(&job.creator, &job.lock_id, NOW) + .await + .unwrap() + ); + assert!( + !reopened + .request_force(&job.creator, &job.lock_id, NOW) + .await + .unwrap() + ); + reopened + .record_force_receipt(&job.creator, &job.lock_id, NOW) + .await + .unwrap(); + reopened + .record_force_receipt(&job.creator, &job.lock_id, NOW) + .await + .unwrap(); + assert!( + reopened + .has_force_receipt(&job.creator, &job.lock_id) + .await + .unwrap() + ); + sqlx::query("DELETE FROM content_lock_deletion_jobs WHERE job_id = $1") + .bind(job.job_id) + .execute(database.pool()) + .await + .unwrap(); + assert!( + reopened + .has_force_receipt(&job.creator, &job.lock_id) + .await + .unwrap() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn concurrent_claims_return_a_job_once() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job).await.unwrap(); + + let (left, right) = tokio::join!( + repository.claim_next("worker-a", NOW, datetime!(2026-08-12 05:05:00 UTC)), + repository.claim_next("worker-b", NOW, datetime!(2026-08-12 05:05:00 UTC)), + ); + assert_eq!( + usize::from(left.unwrap().is_some()) + usize::from(right.unwrap().is_some()), + 1 + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn read_rejects_corrupt_frozen_manifest_identity() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET frozen_content_lock = jsonb_set( + frozen_content_lock, + '{access_policy,requested_credential_ttl_seconds}', + '901'::jsonb + ) + WHERE job_id = $1", + ) + .bind(job.job_id) + .execute(database.pool()) + .await + .unwrap(); + + assert!( + repository + .get_job(&job.creator, &job.lock_id) + .await + .is_err() + ); + + database.cleanup().await; + } + + fn content_lock() -> ContentLock { + ContentLock { + version: CONTENT_LOCK_VERSION, + creator: CreatorPubky::from_str(CREATOR).unwrap(), + primary_resource: Some( + GuardedResource::new( + "/priv/locks.app/content/post.json".to_owned(), + GuardedResourceHash::from_bytes([7; 32]), + "application/json".to_owned(), + 42, + ) + .unwrap(), + ), + secondary_resources: BTreeMap::new(), + criteria: vec![], + lock_logic: LockLogic::All { criteria: vec![] }, + access_policy: AccessPolicy { + requested_credential_ttl_seconds: 900, + }, + lock_server: LockServerConfig { override_: None }, + created_at: datetime!(2026-08-12 04:00:00 UTC), + } + } +} diff --git a/locks-service/src/infrastructure/postgres/migrations.rs b/locks-service/src/infrastructure/postgres/migrations.rs index a9bccc6..741d053 100644 --- a/locks-service/src/infrastructure/postgres/migrations.rs +++ b/locks-service/src/infrastructure/postgres/migrations.rs @@ -50,6 +50,8 @@ mod tests { assert_table_exists(&mut connection, "frontend_session_codes").await; assert_table_exists(&mut connection, "frontend_sessions").await; assert_table_exists(&mut connection, "content_lock_ownership").await; + assert_table_exists(&mut connection, "content_lock_deletion_jobs").await; + assert_table_exists(&mut connection, "content_lock_force_deletion_receipts").await; assert_column_exists(&mut connection, "verification_tasks", "creator").await; assert_column_exists(&mut connection, "verification_tasks", "bundle_id").await; assert_column_exists(&mut connection, "verification_tasks", "next_attempt_at").await; @@ -75,6 +77,19 @@ mod tests { assert_column_exists(&mut connection, "content_lock_ownership", "guarded_path").await; assert_column_exists(&mut connection, "content_lock_ownership", "lock_id").await; assert_column_exists(&mut connection, "content_lock_ownership", "status").await; + assert_column_exists( + &mut connection, + "content_lock_deletion_jobs", + "frozen_content_lock", + ) + .await; + assert_column_exists(&mut connection, "content_lock_deletion_jobs", "claim_token").await; + assert_column_exists( + &mut connection, + "content_lock_deletion_jobs", + "force_requested_at", + ) + .await; assert_unique_constraint_exists( &mut connection, "verification_tasks", @@ -87,6 +102,12 @@ mod tests { "content_lock_ownership_creator_path_unique", ) .await; + assert_unique_constraint_exists( + &mut connection, + "content_lock_deletion_jobs", + "content_lock_deletion_jobs_creator_lock_unique", + ) + .await; drop(connection); database.cleanup().await; diff --git a/locks-service/src/infrastructure/postgres/mod.rs b/locks-service/src/infrastructure/postgres/mod.rs index 2ddf224..3639734 100644 --- a/locks-service/src/infrastructure/postgres/mod.rs +++ b/locks-service/src/infrastructure/postgres/mod.rs @@ -7,6 +7,7 @@ //! adapters or explicit production indexes are designed. pub mod access_credentials; +pub mod content_lock_deletions; pub mod content_lock_ownership; pub mod creator_authority; pub mod creator_connect_flows; @@ -19,6 +20,7 @@ pub mod verification_task_claims; pub mod verification_tasks; pub use access_credentials::PostgresAccessCredentialStore; +pub use content_lock_deletions::PostgresContentLockDeletionRepository; pub use content_lock_ownership::PostgresContentLockOwnershipRepository; pub use creator_authority::{CreatorAuthoritySecretCipher, PostgresCreatorAuthorityStore}; pub use creator_connect_flows::PostgresCreatorConnectFlowStore; diff --git a/locks-service/tests/content_lock_deletions.rs b/locks-service/tests/content_lock_deletions.rs new file mode 100644 index 0000000..91a64a4 --- /dev/null +++ b/locks-service/tests/content_lock_deletions.rs @@ -0,0 +1,277 @@ +use std::{collections::BTreeMap, str::FromStr}; + +use locks_core::{ + ids::{CreatorPubky, GuardedResourceHash}, + lock_policy::{ + AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, GuardedResource, LockLogic, + LockServerConfig, + }, +}; +use locks_service::{ + application::{ + models::{ + ContentLockDeletionFailureCode, ContentLockDeletionJob, ContentLockDeletionPhase, + ContentLockDeletionState, + }, + ports::ContentLockDeletionRepository, + }, + infrastructure::memory::content_lock_deletions::InMemoryContentLockDeletionRepository, +}; +use time::macros::datetime; +use uuid::Uuid; + +const CREATOR: &str = "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy"; +const NOW: time::OffsetDateTime = datetime!(2026-08-12 05:00:00 UTC); +const LEASE_END: time::OffsetDateTime = datetime!(2026-08-12 05:05:00 UTC); + +#[tokio::test] +async fn frozen_manifest_identity_is_immutable_and_creator_lock_unique() { + let repository = InMemoryContentLockDeletionRepository::new(); + let lock = content_lock(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock.clone(), NOW).unwrap(); + + repository.insert_job(job.clone()).await.unwrap(); + assert_eq!( + repository + .get_job(&job.creator, &job.lock_id) + .await + .unwrap(), + Some(job.clone()) + ); + + let duplicate = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + assert!(repository.insert_job(duplicate).await.is_err()); + + let mut duplicate_id = + ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + duplicate_id.job_id = job.job_id; + assert!(repository.insert_job(duplicate_id).await.is_err()); + + assert!(job.validate_frozen_identity().is_ok()); + let mut corrupted = job; + corrupted + .frozen_content_lock + .access_policy + .requested_credential_ttl_seconds += 1; + assert!(corrupted.validate_frozen_identity().is_err()); + + let mut malformed = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + malformed.state = ContentLockDeletionState::Running; + assert!(repository.insert_job(malformed).await.is_err()); +} + +#[tokio::test] +async fn due_claims_reclaim_with_fresh_tokens_and_fence_stale_writes() { + let repository = InMemoryContentLockDeletionRepository::new(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + + let first = repository + .claim_next("worker-a", NOW, LEASE_END) + .await + .unwrap() + .unwrap(); + assert_eq!(first.job.state, ContentLockDeletionState::Running); + assert_eq!(first.job.attempt_count, 1); + assert!( + repository + .claim_next("worker-b", NOW, LEASE_END) + .await + .unwrap() + .is_none() + ); + + let reclaimed = repository + .claim_next( + "worker-b", + datetime!(2026-08-12 05:05:01 UTC), + datetime!(2026-08-12 05:10:00 UTC), + ) + .await + .unwrap() + .unwrap(); + assert_ne!(first.claim_token, reclaimed.claim_token); + assert_eq!(reclaimed.job.attempt_count, 2); + + assert!( + repository + .advance_phase( + job.job_id, + "worker-a", + first.claim_token, + datetime!(2026-08-12 05:05:01 UTC), + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .is_none() + ); + let advanced = repository + .advance_phase( + job.job_id, + "worker-b", + reclaimed.claim_token, + datetime!(2026-08-12 05:06:00 UTC), + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(advanced.state, ContentLockDeletionState::Queued); + assert_eq!(advanced.phase, ContentLockDeletionPhase::StartPaymentDrain); + assert_eq!(advanced.attempt_count, 0); + + let next_claim = repository + .claim_next( + "worker-c", + datetime!(2026-08-12 05:06:01 UTC), + datetime!(2026-08-12 05:11:00 UTC), + ) + .await + .unwrap() + .unwrap(); + assert!( + repository + .advance_phase( + job.job_id, + "worker-c", + next_claim.claim_token, + datetime!(2026-08-12 05:07:00 UTC), + ContentLockDeletionPhase::DeleteContent, + ) + .await + .is_err() + ); + let failed = repository + .finish( + job.job_id, + "worker-c", + next_claim.claim_token, + datetime!(2026-08-12 05:07:00 UTC), + Some(ContentLockDeletionFailureCode::TombstoneMissing), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(failed.state, ContentLockDeletionState::Failed); + assert_eq!( + failed.failure_code, + Some(ContentLockDeletionFailureCode::TombstoneMissing) + ); +} + +#[test] +fn failure_codes_are_a_closed_stable_vocabulary() { + for (code, wire) in [ + ( + ContentLockDeletionFailureCode::TombstoneMissing, + "tombstone_missing", + ), + ( + ContentLockDeletionFailureCode::TombstoneReplaced, + "tombstone_replaced", + ), + ( + ContentLockDeletionFailureCode::RetryExhausted, + "retry_exhausted", + ), + ( + ContentLockDeletionFailureCode::StateCorrupt, + "state_corrupt", + ), + ] { + assert_eq!(code.as_str(), wire); + assert_eq!( + wire.parse::().unwrap(), + code + ); + } + assert!( + "backend: secret" + .parse::() + .is_err() + ); +} + +#[tokio::test] +async fn retry_due_time_and_force_receipts_are_durable_repository_facts() { + let repository = InMemoryContentLockDeletionRepository::new(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + let claimed = repository + .claim_next("worker-a", NOW, LEASE_END) + .await + .unwrap() + .unwrap(); + let retry_at = datetime!(2026-08-12 05:06:00 UTC); + repository + .schedule_retry(job.job_id, "worker-a", claimed.claim_token, NOW, retry_at) + .await + .unwrap() + .unwrap(); + assert!( + repository + .claim_next("worker-b", NOW, LEASE_END) + .await + .unwrap() + .is_none() + ); + assert!( + repository + .claim_next("worker-b", retry_at, datetime!(2026-08-12 05:11:00 UTC)) + .await + .unwrap() + .is_some() + ); + + assert!( + repository + .request_force(&job.creator, &job.lock_id, NOW) + .await + .unwrap() + ); + assert!( + !repository + .request_force(&job.creator, &job.lock_id, NOW) + .await + .unwrap() + ); + repository + .record_force_receipt(&job.creator, &job.lock_id, NOW) + .await + .unwrap(); + repository + .record_force_receipt(&job.creator, &job.lock_id, NOW) + .await + .unwrap(); + assert!( + repository + .has_force_receipt(&job.creator, &job.lock_id) + .await + .unwrap() + ); +} + +fn content_lock() -> ContentLock { + ContentLock { + version: CONTENT_LOCK_VERSION, + creator: CreatorPubky::from_str(CREATOR).unwrap(), + primary_resource: Some( + GuardedResource::new( + "/priv/locks.app/content/post.json".to_owned(), + GuardedResourceHash::from_bytes([7; 32]), + "application/json".to_owned(), + 42, + ) + .unwrap(), + ), + secondary_resources: BTreeMap::new(), + criteria: vec![], + lock_logic: LockLogic::All { criteria: vec![] }, + access_policy: AccessPolicy { + requested_credential_ttl_seconds: 900, + }, + lock_server: LockServerConfig { override_: None }, + created_at: datetime!(2026-08-12 04:00:00 UTC), + } +} From a11ceef011f2611d339e1f24e9c43710a9fd0bfd Mon Sep 17 00:00:00 2001 From: dzdidi Date: Wed, 12 Aug 2026 08:19:15 -0300 Subject: [PATCH 5/6] feat(service): enforce deletion admission cutoff Signed-off-by: dzdidi --- ...20-locks-paykit-v1-integration-boundary.md | 20 +- docs/API.md | 9 +- docs/RUNTIME.md | 4 +- ...26-08-10-graceful-content-lock-deletion.md | 2 +- locks-e2e/tests/postgres_runtime.rs | 278 ++++++++++++++- locks-server/src/api/errors.rs | 26 +- locks-server/src/api/verification.rs | 48 +++ ...12_content_lock_deletion_task_snapshot.sql | 18 + locks-service/src/application/errors.rs | 3 + .../src/application/ports/verification.rs | 2 + .../use_cases/complete_verification_task.rs | 1 + .../use_cases/submit_proof_bundle.rs | 32 +- .../postgres/content_lock_deletions.rs | 325 +++++++++++++++++- .../src/infrastructure/postgres/migrations.rs | 2 + .../src/infrastructure/postgres/mod.rs | 2 + .../postgres/proof_admission.rs | 212 ++++++++++++ .../postgres/verification_task_claims.rs | 50 +++ .../postgres/verification_tasks.rs | 91 +++-- 18 files changed, 1057 insertions(+), 68 deletions(-) create mode 100644 locks-service/migrations/0012_content_lock_deletion_task_snapshot.sql create mode 100644 locks-service/src/infrastructure/postgres/proof_admission.rs diff --git a/docs/ADRs/0020-locks-paykit-v1-integration-boundary.md b/docs/ADRs/0020-locks-paykit-v1-integration-boundary.md index 6fa39c4..b3da988 100644 --- a/docs/ADRs/0020-locks-paykit-v1-integration-boundary.md +++ b/docs/ADRs/0020-locks-paykit-v1-integration-boundary.md @@ -46,14 +46,14 @@ The v1 content-lock criterion has verifier wire value `paykit-payment` and param 1. apply rate limiting; 2. validate proof shape; -3. load and validate the current canonical Lock Resource and payment policy; -4. resolve the current reader through Pubky discovery; -5. compare any persisted lifecycle under `{ creator, bundle_id }`; -6. return an exact persisted replay or reject changed submitted proof material with `409 task_state_conflict`; -7. require configured Paykit and create an invoice only for a new identity; and -8. insert the verification task with post-invoice race reconciliation. +3. compare any durable lifecycle or admission reservation under `{ creator, bundle_id }` before mutable Lock Resource lookup or reader discovery; +4. return an exact ready replay, resume an exact unready reservation from its persisted submission fields, or reject changed submitted proof material with `409 task_state_conflict`; +5. only for a genuinely new identity, load and validate the current canonical Lock Resource and payment policy and resolve the current reader through Pubky discovery; +6. require configured Paykit and atomically persist a hidden, unclaimable admission reservation under the per-lock deletion/admission fence; +7. create or idempotently replay the Paykit invoice; and +8. mark the reservation ready so the verification task becomes publicly visible and worker-claimable. -Exact and changed persisted replays do not call Paykit. Terminal lifecycle state is not restarted under the same identity; clients needing another attempt must generate a new Bundle ID. +Exact ready replay does not call Paykit. Exact unready replay requires configured Paykit and repeats the same persisted idempotent invoice request before marking the reservation ready. Changed replay conflicts without calling Paykit. Terminal lifecycle state is not restarted under the same identity; clients needing another attempt must generate a new Bundle ID. ### Invoice request @@ -69,7 +69,7 @@ For a new lifecycle identity, Locks sends RFC 8785 canonical JSON to `POST /invo Locks signs the exact canonical body bytes with its existing Ed25519 keypair and sends the unpadded-base64url signature in `X-Paykit-Signature`. -The durable Paykit invoice identity is `(creator, bundle_id)`, where Paykit derives `creator` from `lock_resource`. Exact replay must return the original generic success without repeating mutable lookups, allocation, address creation, or delivery side effects. A different binding under the same identity returns Paykit `409 Conflict`, which Locks maps to `409 task_state_conflict`. Locks accepts any Paykit 2xx response and ignores its body; other invoice failures return `502 paykit_invoice_creation_failed` without creating a new verification task. +The durable Paykit invoice identity is `(creator, bundle_id)`, where Paykit derives `creator` from `lock_resource`. Exact replay must return the original generic success without repeating mutable lookups, allocation, address creation, or delivery side effects. A different binding under the same identity returns Paykit `409 Conflict`, which Locks maps to `409 task_state_conflict`. Locks accepts any Paykit 2xx response and ignores its body; other invoice failures return `502 paykit_invoice_creation_failed` while the internal Locks reservation remains hidden and unclaimable for exact retry. ### Status request and access policy @@ -94,7 +94,7 @@ V1 has no invoice expiry, TTL, `expires_at`, or terminal Paykit payment-failure ### Runtime boundary -- A new payment lifecycle requires `[paykit]`; exact persisted replay does not. +- A new payment lifecycle and exact unready reconciliation require `[paykit]`; exact ready replay does not. - Paykit HTTP connect timeout is 5 seconds and whole-request timeout is 20 seconds. - An enabled in-process Paykit worker requires `claim_timeout_seconds > 20`. - `worker.poll_interval_ms` must be greater than zero whether the worker is enabled or disabled. @@ -114,7 +114,7 @@ V1 has no invoice expiry, TTL, `expires_at`, or terminal Paykit payment-failure - Both services must implement the same canonical-body signing contract. - Paykit must parse the public Locks payment criterion and therefore depends on its versioned shape. -- Exact submission replay intentionally performs current canonical lock and reader preflight before returning persisted lifecycle state. +- Incomplete admission reservations require durable reconciliation before deletion may start the Paykit drain. - Unpaid invoices and pending Locks tasks have no protocol expiry in v1 and therefore require operational retention policy outside the payment-status contract. ## Rejected alternatives diff --git a/docs/API.md b/docs/API.md index 5ce85ed..0c61d95 100644 --- a/docs/API.md +++ b/docs/API.md @@ -55,7 +55,7 @@ Gated-off routes are plain Axum `404 Not Found` responses because the route is i | `DELETE /frontend-sessions/current` | `204` empty response | Requires `Authorization: Bearer `. Mounted with creator authority acquisition. | Token is request-only and is deleted from the frontend session store. | `401 frontend_session_unavailable`, `401 frontend_session_expired`, `404` when route gated off | | `GET /.well-known/locks-server` | `200` JSON service identity | Public. Always mounted. CORS-enabled. | No secrets. Used by browser SDK to verify service, API version, and Lock Server Pubky identity. | n/a | | `GET /creator/authority-status` | `200` JSON secret-free authority status | Requires `Authorization: Bearer `. Creator is derived from the frontend session. | Response contains only creator, boolean status, auth kind, scopes, and optional expiry; no tokens, codes, authorization URLs, secrets, or DB/config values. | `401 frontend_session_unavailable`, `401 frontend_session_expired`, `404` only if route absent in older deployments | -| `POST /proof-bundles` | `200` JSON lifecycle | Public viewer route. A new `paykit-payment` lifecycle identity requires `[paykit]` runtime config; an exact persisted replay does not. | No bearer secrets, invoice data, or raw proof material in response. | `400 invalid_request`, `409 task_state_conflict`, `422 unsupported_verifier_type`, `422 paykit_not_configured`, `422 reader_pubky_unresolvable`, `429 rate_limited`, `502 paykit_invoice_creation_failed` | +| `POST /proof-bundles` | `200` JSON lifecycle | Public viewer route. A new or unready `paykit-payment` lifecycle identity requires `[paykit]` runtime config; an exact ready replay does not. | No bearer secrets, invoice data, or raw proof material in response. | `400 invalid_request`, `409 task_state_conflict`, `409 content_lock_deletion_in_progress`, `422 unsupported_verifier_type`, `422 paykit_not_configured`, `422 reader_pubky_unresolvable`, `429 rate_limited`, `502 paykit_invoice_creation_failed` | | `POST /verification-task-lookups` | `200` JSON lifecycle | Public viewer route. | No bearer secrets in response. | `400 invalid_request`, `404 verification_task_not_found` | | `POST /verification-task-completions` | `200` JSON lifecycle | Dev-only completion gate. | No bearer secrets in response. | `400 invalid_request`, `404 verification_task_not_found`, `409 task_state_conflict`, `404` when route gated off | | `POST /access-credentials` | `200` JSON credential | Public viewer route after entitlement. | Response intentionally contains raw viewer access credential exactly once. | `400 invalid_request`, `403 entitlement_not_authorized`, `404 verification_task_not_found` | @@ -101,12 +101,13 @@ Stable error codes and statuses mirror `locks-server/src/api/errors.rs` tests: | `creator_authority_unavailable` | 503 | Creator-granted homeserver authority is unavailable or could not be revalidated. | | `content_lock_path_conflict` | 409 | The creator-scoped guarded path already has an in-flight or published Content Lock owner. | | `task_state_conflict` | 409 | Submission or completion conflicts with existing task state. | +| `content_lock_deletion_in_progress` | 409 | A deletion cutoff committed before this new proof Bundle could be admitted. | | `unsupported_verifier_type` | 422 | Proof references a verifier unavailable in the current runtime. | | `paykit_not_configured` | 422 | A `paykit-payment` proof was submitted to a Lock Server without a `[paykit]` runtime section. | | `reader_pubky_unresolvable` | 422 | A `paykit-payment` proof had a syntactically valid `reader_public_key` that could not be resolved to a Pubky homeserver/PKARR record before invoice creation. | | `rate_limited` | 429 | Submission exceeded configured rate limits. | | `payload_too_large` | 413 | Raw guarded-resource upload exceeded `[content_locks].max_resource_bytes`. | -| `paykit_invoice_creation_failed` | 502 | Lock Server could not create the Paykit invoice; no verification task is created. | +| `paykit_invoice_creation_failed` | 502 | Lock Server could not create or replay the Paykit invoice; no verification task is publicly admitted or worker-claimable. | | `internal_error` | 500 | Unexpected server-side failure. | ## Service discovery @@ -526,7 +527,7 @@ Success response returns lifecycle metadata only. It does not return internal `t For non-payment verifier types, `reader_public_key` may be omitted. For `paykit-payment`, `reader_public_key` is required as a top-level field on `submitted_proof_bundle`; the payment proof payload itself must be `{}`. Payment submissions are v1 single-proof only: a bundle with more than one `paykit-payment` proof, or a mix of `paykit-payment` and any other proof type, is rejected with `400 invalid_request`. -Submission processing applies rate limiting, validates proof shape, loads the current canonical content lock referenced by `pubky_lock_resource`, verifies its lock identity and payment policy (including recipient/creator equality), and resolves `reader_public_key` through Pubky/PKARR/homeserver discovery. It then checks the permanent lifecycle identity `{ creator, bundle_id }`. An exact persisted replay returns the existing lifecycle; changed submitted proof material returns `409 task_state_conflict`. Neither case calls Paykit again. Only a new identity requires `[paykit]` configuration and calls `POST /invoices`. Task insertion retains race reconciliation after invoice creation. The signed Paykit invoice body is exactly: +Submission processing applies rate limiting and validates the closed proof shape before checking the permanent lifecycle identity `{ creator, bundle_id }`. With PostgreSQL runtime storage, exact persisted binding is classified before mutable lock lookup or reader discovery. An exact ready replay returns the existing lifecycle without calling Paykit; an exact unready replay requires `[paykit]` and repeats the idempotent invoice request from the persisted submission fields; changed submitted proof material returns `409 task_state_conflict`. Only a genuinely new identity loads and validates the current canonical content lock, verifies its lock identity and payment policy (including recipient/creator equality), resolves `reader_public_key` through Pubky/PKARR/homeserver discovery, and requires `[paykit]`. Locks then atomically persists a hidden, unclaimable admission reservation before calling `POST /invoices`; Paykit success makes that task publicly visible and worker-claimable. The signed Paykit invoice body is exactly: ```json { @@ -536,7 +537,7 @@ Submission processing applies rate limiting, validates proof shape, loads the cu } ``` -Any 2xx Paykit invoice response is accepted and its body is ignored. Paykit invoice `409 Conflict` maps to `409 task_state_conflict`. Other invoice failures return `502 paykit_invoice_creation_failed`; no verification task is created unless invoice creation was accepted. +Any 2xx Paykit invoice response is accepted and its body is ignored. With PostgreSQL runtime storage, Locks first commits an internal, unclaimable admission reservation serialized against deletion start. It calls Paykit only after that commit, then makes the task publicly visible and worker-claimable after Paykit accepts the invoice. Durable handle replay is classified before consulting the mutable public lock or reader discovery: exact ready replay returns the persisted lifecycle, exact unready replay resumes the same idempotent Paykit request from persisted fields, and changed replay conflicts. Graceful deletion snapshots these reservations and cannot start the Paykit drain until every snapshotted reservation is ready, so Paykit has durably created every pre-cutoff invoice before its drain fence activates. Paykit invoice `409 Conflict` maps to `409 task_state_conflict`. Other invoice failures return `502 paykit_invoice_creation_failed`; the internal reservation remains hidden and unclaimable for exact retry. Paykit status verification is worker-owned. The Lock Server sends canonical JSON `{ "creator": "pubky...", "bundle_id": "..." }` to `POST /transactions/status` with `X-Paykit-Signature` over those exact canonical body bytes. Valid response statuses are `undetected`, `detected`, and `confirmed`. Transport failures, timeouts, every non-2xx response (including `404` and authentication/authorization failures), and malformed success bodies are durably rescheduled as pending and are not retried again before the worker poll interval elapses. V1 has no terminal Paykit payment-failure status. diff --git a/docs/RUNTIME.md b/docs/RUNTIME.md index 5acb082..fdab140 100644 --- a/docs/RUNTIME.md +++ b/docs/RUNTIME.md @@ -137,7 +137,7 @@ server_url = "http://127.0.0.1:3001" minimum_confirmations = 0 ``` -`server_url` is the standalone Paykit Server base URL. Any configured path prefix is preserved when appending `invoices` and `transactions/status`, with or without a trailing slash. For a new `{ creator, bundle_id }` lifecycle identity, the Lock Server calls `POST /invoices` during `POST /proof-bundles` before creating a verification task; exact persisted submission replay does not call Paykit. Workers call `POST /transactions/status` while completing pending payment verification tasks. Both request bodies are canonical JSON signed through `X-Paykit-Signature` with the existing Lock Server keypair; therefore `credentials.lock_server_secret_key` must use the `keypair-seed:` format when `[paykit]` is configured. +`server_url` is the standalone Paykit Server base URL. Any configured path prefix is preserved when appending `invoices` and `transactions/status`, with or without a trailing slash. With PostgreSQL runtime storage, a new `{ creator, bundle_id }` first commits an internal, unclaimable admission reservation under the same per-lock database fence used by deletion start. The Lock Server then calls `POST /invoices`; only success makes the task publicly visible and worker-claimable. Durable handle replay is checked before mutable public-lock lookup and reader discovery. Exact replay of a ready task does not call Paykit; exact retry of an incomplete reservation uses its persisted canonical request fields to replay the same idempotent invoice request, even after the public lock is tombstoned. A deletion job cannot advance from withdrawal to Paykit drain start while any snapshotted reservation remains unready; this ensures Paykit's drain sees every pre-cutoff invoice. Workers call `POST /transactions/status` while completing ready payment verification tasks. Both request bodies are canonical JSON signed through `X-Paykit-Signature` with the existing Lock Server keypair; therefore `credentials.lock_server_secret_key` must use the `keypair-seed:` format when `[paykit]` is configured. Paykit HTTP connections have a 5-second connect timeout and every request has a 20-second whole-request timeout. Invoice timeouts fail submission with `paykit_invoice_creation_failed`; status-query timeouts remain pending/retryable. When `[paykit]` and the in-process worker are both enabled, `worker.claim_timeout_seconds` must be greater than 20 so a Paykit request cannot outlive the worker claim lease. External worker deployments must preserve the same timeout/lease relationship operationally. @@ -145,7 +145,7 @@ Every claimed verification task receives a fresh opaque claim token. Retry, comp `minimum_confirmations = 0` accepts a Paykit status of `detected` or `confirmed` when `amount_matched = true`. Values above zero require `status = "confirmed"` and at least that many confirmations. `undetected`, insufficient confirmations, or `amount_matched = false` keep the task pending/retryable. -Omitting `[paykit]` prevents creation of new payment lifecycle identities. In that state, non-payment verifier flows continue to run, an exact persisted `paykit-payment` submission replay can still return its lifecycle after current canonical preflight, and a new `paykit-payment` submission returns `422 paykit_not_configured`. Staging deployments should omit `[paykit]` until a Paykit Server is deployed and reachable for that environment. +Omitting `[paykit]` prevents creation or reconciliation of payment lifecycle identities. In that state, non-payment verifier flows continue to run, an exact ready `paykit-payment` replay can still return its persisted lifecycle, an exact unready replay returns `422 paykit_not_configured`, and a new `paykit-payment` submission returns the same `422`. Staging deployments should omit `[paykit]` until a Paykit Server is deployed and reachable for that environment. ## Runtime storage diff --git a/docs/plans/2026-08-10-graceful-content-lock-deletion.md b/docs/plans/2026-08-10-graceful-content-lock-deletion.md index a818683..6397e7b 100644 --- a/docs/plans/2026-08-10-graceful-content-lock-deletion.md +++ b/docs/plans/2026-08-10-graceful-content-lock-deletion.md @@ -378,7 +378,7 @@ cargo test --workspace --no-run **RED:** Concurrent tests prove task-first commit joins snapshot, deletion-first commit rejects a new Bundle, exact old replay succeeds, and conflicting replay remains rejected. -**GREEN:** Use per-lock database serialization and one transaction for job persistence/task snapshot. Do not use viewer timestamps or tombstone publication as the cutoff. +**GREEN:** Use per-lock database serialization and one transaction for job persistence/task snapshot. For Paykit-backed submissions, atomically persist a hidden, unclaimable admission reservation before the external invoice call; classify durable exact replay/conflict before mutable lock lookup or reader resolution, and resume an unready reservation from its persisted canonical fields. Paykit success makes it ready. This guarantees that deletion either snapshots the durable obligation or commits first and prevents any Paykit call. Do not permit transition to `start_payment_drain` while a snapshotted reservation is unready: Paykit's active drain accepts exact replay only for invoices already created before drain start. Do not hold a database transaction across HTTP, and do not use viewer timestamps or tombstone publication as the cutoff. **Suggested commit:** `feat(service): enforce deletion admission cutoff` diff --git a/locks-e2e/tests/postgres_runtime.rs b/locks-e2e/tests/postgres_runtime.rs index 4777329..26fcd04 100644 --- a/locks-e2e/tests/postgres_runtime.rs +++ b/locks-e2e/tests/postgres_runtime.rs @@ -1,10 +1,14 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::PathBuf; use std::str::FromStr; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use async_trait::async_trait; use axum::body::{Body, to_bytes}; use axum::extract::ConnectInfo; use axum::http::{Request, StatusCode, header}; +use axum::routing::post; use locks_core::ids::{ BundleId, CreatorPubky, GuardedResourceHash, LockServerPubky, PubkyLockResource, }; @@ -14,24 +18,28 @@ use locks_core::lock_policy::{ }; use locks_core::verification::{Proof, SUBMITTED_PROOF_BUNDLE_VERSION, SubmittedProofBundle}; use locks_server::api::routes::router; -use locks_server::app_state::AppState; +use locks_server::app_state::{AppState, ReaderPubkyResolver}; use locks_server::config::{ ContentLocksConfig, CreatorAuthorityAcquisitionConfig, DatabaseConfig, - LockServerCredentialsConfig, LockServerRuntimeConfig, LoggingConfig, PkdnsConfig, PubkyConfig, + FilesystemLockServerIdentityProvider, LockServerCredentialsConfig, LockServerIdentityProvider, + LockServerRuntimeConfig, LoggingConfig, PaykitConfig, PkdnsConfig, PubkyConfig, RateLimitsConfig, RuntimeConfig, RuntimeEnvironment, SecretsConfig, WorkerConfig, }; use locks_server::worker::{VerificationWorker, WorkerTick}; use locks_service::application::models::{ - AccessCredential, AccessCredentialLookupKey, ContentLockOwnershipStatus, - CreatorAuthorityAuthKind, CreatorAuthorityRecord, CreatorAuthoritySecret, - VerificationTaskStatus, + AccessCredential, AccessCredentialLookupKey, ContentLockDeletionJob, + ContentLockOwnershipStatus, CreatorAuthorityAuthKind, CreatorAuthorityRecord, + CreatorAuthoritySecret, VerificationTaskStatus, }; +use locks_service::application::ports::ContentLockDeletionRepository; use locks_service::infrastructure::memory::{ content_locks::InMemoryContentLockRepository, entitlements::InMemoryEntitlementRepository, guarded_resources::InMemoryGuardedResourceRepository, lock_service_pointers::InMemoryLockServicePointerRepository, }; -use locks_service::infrastructure::postgres::{CreatorAuthoritySecretCipher, run_migrations}; +use locks_service::infrastructure::postgres::{ + CreatorAuthoritySecretCipher, PostgresContentLockDeletionRepository, run_migrations, +}; use serde_json::{Value, json}; use sqlx::postgres::PgPoolOptions; use sqlx::{Connection, Executor, PgConnection, PgPool}; @@ -174,6 +182,212 @@ async fn postgres_runtime_encrypts_creator_authority_secrets_at_rest() { database.cleanup().await; } +#[tokio::test] +async fn deletion_first_proof_submission_returns_409_without_calling_paykit() { + let Some(database) = TestDatabase::create().await else { + return; + }; + let invoice_calls = Arc::new(AtomicUsize::new(0)); + let paykit_state = Arc::clone(&invoice_calls); + let paykit_app = axum::Router::new().route( + "/invoices", + post(move || { + let paykit_state = Arc::clone(&paykit_state); + async move { + paykit_state.fetch_add(1, Ordering::SeqCst); + StatusCode::OK + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let paykit_url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, paykit_app).await.unwrap() }); + + let temp_dir = tempfile::tempdir().unwrap(); + let secret_path = temp_dir.path().join("lock-server.keypair-seed"); + let public_key = FilesystemLockServerIdentityProvider + .generate_secret(&secret_path) + .unwrap(); + let mut config = test_config(); + config.credentials.lock_server_secret_key = secret_path; + config.credentials.lock_server_public_key = public_key; + config.paykit = Some(PaykitConfig { + server_url: paykit_url, + minimum_confirmations: 0, + }); + let state = AppState::new_with_postgres_runtime_and_creator_repositories( + config, + database.pool().clone(), + CreatorAuthoritySecretCipher::new([7; 32]), + Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryGuardedResourceRepository::new()), + Arc::new(InMemoryLockServicePointerRepository::new()), + Arc::new(InMemoryEntitlementRepository::new()), + ) + .with_reader_pubky_resolver(Arc::new(AlwaysResolvesReader)); + let lock = paykit_content_lock(); + seed_content_lock(&state, lock.clone()).await; + PostgresContentLockDeletionRepository::new(database.pool().clone()) + .insert_job( + ContentLockDeletionJob::new( + uuid::Uuid::new_v4(), + lock.clone(), + datetime!(2026-08-12 06:00:00 UTC), + ) + .unwrap(), + ) + .await + .unwrap(); + + let response = router(state) + .oneshot(json_request( + "POST", + "/proof-bundles", + json!({ "submitted_proof_bundle": paykit_submission_for(&lock) }), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CONFLICT); + assert_eq!( + response_json(response).await, + json!({ + "error": { + "code": "content_lock_deletion_in_progress", + "message": "content lock deletion is in progress" + } + }) + ); + assert_eq!(invoice_calls.load(Ordering::SeqCst), 0); + + database.cleanup().await; +} + +#[tokio::test] +async fn snapshotted_unready_paykit_replay_ignores_tombstoned_lock_and_reader_resolution() { + let Some(database) = TestDatabase::create().await else { + return; + }; + let invoice_calls = Arc::new(AtomicUsize::new(0)); + let paykit_state = Arc::clone(&invoice_calls); + let paykit_app = axum::Router::new().route( + "/invoices", + post(move || { + let call = paykit_state.fetch_add(1, Ordering::SeqCst); + async move { + if call == 0 { + StatusCode::INTERNAL_SERVER_ERROR + } else { + StatusCode::OK + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let paykit_url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, paykit_app).await.unwrap() }); + + let temp_dir = tempfile::tempdir().unwrap(); + let secret_path = temp_dir.path().join("lock-server.keypair-seed"); + let public_key = FilesystemLockServerIdentityProvider + .generate_secret(&secret_path) + .unwrap(); + let mut config = test_config(); + config.credentials.lock_server_secret_key = secret_path; + config.credentials.lock_server_public_key = public_key; + config.paykit = Some(PaykitConfig { + server_url: paykit_url, + minimum_confirmations: 0, + }); + let initial_state = AppState::new_with_postgres_runtime_and_creator_repositories( + config.clone(), + database.pool().clone(), + CreatorAuthoritySecretCipher::new([7; 32]), + Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryGuardedResourceRepository::new()), + Arc::new(InMemoryLockServicePointerRepository::new()), + Arc::new(InMemoryEntitlementRepository::new()), + ) + .with_reader_pubky_resolver(Arc::new(AlwaysResolvesReader)); + let lock = paykit_content_lock(); + let submitted = paykit_submission_for(&lock); + seed_content_lock(&initial_state, lock.clone()).await; + + let first = router(initial_state) + .oneshot(json_request( + "POST", + "/proof-bundles", + json!({ "submitted_proof_bundle": submitted.clone() }), + )) + .await + .unwrap(); + assert_eq!(first.status(), StatusCode::BAD_GATEWAY); + PostgresContentLockDeletionRepository::new(database.pool().clone()) + .insert_job( + ContentLockDeletionJob::new( + uuid::Uuid::new_v4(), + lock, + datetime!(2026-08-12 06:00:00 UTC), + ) + .unwrap(), + ) + .await + .unwrap(); + + let tombstoned_state = AppState::new_with_postgres_runtime_and_creator_repositories( + config, + database.pool().clone(), + CreatorAuthoritySecretCipher::new([7; 32]), + Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryGuardedResourceRepository::new()), + Arc::new(InMemoryLockServicePointerRepository::new()), + Arc::new(InMemoryEntitlementRepository::new()), + ) + .with_reader_pubky_resolver(Arc::new(NeverResolvesReader)); + let replay_router = router(tombstoned_state); + let replay = replay_router + .clone() + .oneshot(json_request( + "POST", + "/proof-bundles", + json!({ "submitted_proof_bundle": submitted.clone() }), + )) + .await + .unwrap(); + assert_eq!(replay.status(), StatusCode::OK); + assert_eq!(response_json(replay).await["status"], "pending"); + assert_eq!(invoice_calls.load(Ordering::SeqCst), 2); + + let ready_replay = replay_router + .clone() + .oneshot(json_request( + "POST", + "/proof-bundles", + json!({ "submitted_proof_bundle": submitted.clone() }), + )) + .await + .unwrap(); + assert_eq!(ready_replay.status(), StatusCode::OK); + assert_eq!(invoice_calls.load(Ordering::SeqCst), 2); + + let mut changed = submitted; + changed.reader_public_key = Some( + CreatorPubky::from_str("pubky7ir1ttte48bcp4zjychjyscicrwi1j34mtt91ptsafdbjmr8g9eo") + .unwrap(), + ); + let conflict = replay_router + .oneshot(json_request( + "POST", + "/proof-bundles", + json!({ "submitted_proof_bundle": changed }), + )) + .await + .unwrap(); + assert_eq!(conflict.status(), StatusCode::CONFLICT); + assert_eq!(invoice_calls.load(Ordering::SeqCst), 2); + + database.cleanup().await; +} + struct TestDatabase { pool: PgPool, schema_name: String, @@ -397,6 +611,58 @@ fn content_lock() -> ContentLock { } } +fn paykit_content_lock() -> ContentLock { + let mut lock = content_lock(); + lock.criteria = vec![Criterion { + criterion_id: "criterion-1".to_owned(), + verifier_type: VerifierType::PaykitPayment, + params: json!({ + "recipient_pubky": creator().to_string(), + "amount": "50000", + "asset": "BTC", + "payment_in": 24 + }), + }]; + lock +} + +fn paykit_submission_for(content_lock: &ContentLock) -> SubmittedProofBundle { + SubmittedProofBundle { + version: SUBMITTED_PROOF_BUNDLE_VERSION, + bundle_id: bundle_id(), + pubky_lock_resource: PubkyLockResource::new( + content_lock.creator.clone(), + content_lock.content_lock_path().unwrap(), + ), + reader_public_key: Some(creator()), + proofs: vec![Proof { + criterion_id: "criterion-1".to_owned(), + verifier_type: VerifierType::PaykitPayment, + payload: json!({}), + }], + } +} + +#[derive(Debug)] +struct AlwaysResolvesReader; + +#[async_trait] +impl ReaderPubkyResolver for AlwaysResolvesReader { + async fn reader_has_homeserver(&self, _reader: &CreatorPubky) -> bool { + true + } +} + +#[derive(Debug)] +struct NeverResolvesReader; + +#[async_trait] +impl ReaderPubkyResolver for NeverResolvesReader { + async fn reader_has_homeserver(&self, _reader: &CreatorPubky) -> bool { + false + } +} + fn creator() -> CreatorPubky { CreatorPubky::from_str("pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy").unwrap() } diff --git a/locks-server/src/api/errors.rs b/locks-server/src/api/errors.rs index 475f623..d13b91a 100644 --- a/locks-server/src/api/errors.rs +++ b/locks-server/src/api/errors.rs @@ -24,6 +24,7 @@ pub enum ApiErrorCode { FrontendSessionExpired, FrontendSessionStateMismatch, ContentLockPathConflict, + ContentLockDeletionInProgress, TaskStateConflict, UnsupportedVerifierType, PaykitNotConfigured, @@ -55,6 +56,7 @@ impl ApiErrorCode { Self::FrontendSessionExpired => "frontend_session_expired", Self::FrontendSessionStateMismatch => "frontend_session_state_mismatch", Self::ContentLockPathConflict => "content_lock_path_conflict", + Self::ContentLockDeletionInProgress => "content_lock_deletion_in_progress", Self::TaskStateConflict => "task_state_conflict", Self::UnsupportedVerifierType => "unsupported_verifier_type", Self::PaykitNotConfigured => "paykit_not_configured", @@ -86,7 +88,9 @@ impl ApiErrorCode { StatusCode::UNAUTHORIZED } Self::FrontendSessionStateMismatch => StatusCode::BAD_REQUEST, - Self::ContentLockPathConflict | Self::TaskStateConflict => StatusCode::CONFLICT, + Self::ContentLockPathConflict + | Self::ContentLockDeletionInProgress + | Self::TaskStateConflict => StatusCode::CONFLICT, Self::UnsupportedVerifierType | Self::PaykitNotConfigured | Self::ReaderPubkyUnresolvable => StatusCode::UNPROCESSABLE_ENTITY, @@ -199,6 +203,10 @@ impl From for ApiError { ApiErrorCode::ContentLockPathConflict, "content lock path is already owned", ), + ApplicationError::ContentLockDeletionInProgress => Self::new( + ApiErrorCode::ContentLockDeletionInProgress, + "content lock deletion is in progress", + ), ApplicationError::InvalidGuardedResource { .. } => { Self::new(ApiErrorCode::InvalidRequest, "invalid guarded resource") } @@ -510,6 +518,22 @@ mod tests { assert!(!json.to_string().contains("already-owned.txt")); } + #[test] + fn content_lock_deletion_cutoff_maps_to_409_stable_envelope() { + let api_error = ApiError::from(ApplicationError::ContentLockDeletionInProgress); + + assert_eq!(api_error.status_code(), StatusCode::CONFLICT); + assert_eq!( + serde_json::to_value(api_error.error_response()).unwrap(), + json!({ + "error": { + "code": "content_lock_deletion_in_progress", + "message": "content lock deletion is in progress" + } + }) + ); + } + #[test] fn invalid_guarded_resource_maps_to_400_stable_envelope() { let api_error = ApiError::from(ApplicationError::InvalidGuardedResource { diff --git a/locks-server/src/api/verification.rs b/locks-server/src/api/verification.rs index d13d11a..a701b31 100644 --- a/locks-server/src/api/verification.rs +++ b/locks-server/src/api/verification.rs @@ -20,6 +20,7 @@ use locks_service::application::use_cases::submit_proof_bundle::{ use locks_service::application::use_cases::validate_paykit_payment_submission::{ ValidatePaykitPaymentSubmissionRequest, ValidatePaykitPaymentSubmissionUseCase, }; +use locks_service::infrastructure::postgres::PostgresPaykitTaskAdmissionRepository; use locks_service::infrastructure::verifiers::registry::StaticCriterionVerifierRegistry; use crate::api::dtos::{ @@ -111,6 +112,16 @@ async fn maybe_prepare_paykit_submission( "paykit-payment requires reader_public_key", ) })?; + if let Some(pool) = state.postgres_pool() { + let admissions = PostgresPaykitTaskAdmissionRepository::new(pool.clone()); + if let Some(admission) = admissions.find_existing(submitted).await? { + if admission.requires_paykit { + create_paykit_invoice(state, &admission.task.submitted_proof_bundle).await?; + admissions.mark_ready(&admission.task).await?; + } + return Ok(Some(admission.task.into())); + } + } ValidatePaykitPaymentSubmissionUseCase::new(state.content_locks().as_ref()) .execute(ValidatePaykitPaymentSubmissionRequest { submitted_proof_bundle: submitted.clone(), @@ -126,6 +137,16 @@ async fn maybe_prepare_paykit_submission( "reader pubky is unresolvable", )); } + if let Some(pool) = state.postgres_pool() { + let task = submit_use_case.prepare_task(submitted.clone()).await?; + let admissions = PostgresPaykitTaskAdmissionRepository::new(pool.clone()); + let admission = admissions.reserve(task).await?; + if admission.requires_paykit { + create_paykit_invoice(state, &admission.task.submitted_proof_bundle).await?; + admissions.mark_ready(&admission.task).await?; + } + return Ok(Some(admission.task.into())); + } if let Some(existing) = submit_use_case.find_existing(submitted).await? { return Ok(Some(existing)); } @@ -146,6 +167,33 @@ async fn maybe_prepare_paykit_submission( Ok(None) } +async fn create_paykit_invoice( + state: &AppState, + submitted: &SubmittedProofBundle, +) -> Result<(), ApiError> { + let reader = submitted.reader_public_key.as_ref().ok_or_else(|| { + ApiError::new( + ApiErrorCode::InvalidRequest, + "paykit-payment requires reader_public_key", + ) + })?; + state + .paykit_http_client() + .ok_or_else(|| { + ApiError::new( + ApiErrorCode::PaykitNotConfigured, + "paykit is not configured", + ) + })? + .create_invoice(&PaykitInvoiceRequest { + bundle_id: submitted.bundle_id.to_string(), + lock_resource: submitted.pubky_lock_resource.to_string(), + reader: reader.to_string(), + }) + .await + .map_err(map_paykit_invoice_error) +} + fn map_paykit_invoice_error(error: PaykitClientError) -> ApiError { if matches!( error, diff --git a/locks-service/migrations/0012_content_lock_deletion_task_snapshot.sql b/locks-service/migrations/0012_content_lock_deletion_task_snapshot.sql new file mode 100644 index 0000000..15ac984 --- /dev/null +++ b/locks-service/migrations/0012_content_lock_deletion_task_snapshot.sql @@ -0,0 +1,18 @@ +CREATE TABLE content_lock_deletion_task_snapshot ( + deletion_job_id UUID NOT NULL REFERENCES content_lock_deletion_jobs(job_id) ON DELETE CASCADE, + verification_task_id UUID NOT NULL REFERENCES verification_tasks(task_id), + CONSTRAINT content_lock_deletion_task_snapshot_pkey + PRIMARY KEY (deletion_job_id, verification_task_id), + CONSTRAINT content_lock_deletion_task_snapshot_task_unique + UNIQUE (verification_task_id) +); + +CREATE TABLE paykit_task_admissions ( + verification_task_id UUID PRIMARY KEY REFERENCES verification_tasks(task_id) ON DELETE CASCADE, + ready BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + ready_at TIMESTAMPTZ, + CONSTRAINT paykit_task_admissions_ready_time_valid CHECK ( + (ready AND ready_at IS NOT NULL) OR (NOT ready AND ready_at IS NULL) + ) +); diff --git a/locks-service/src/application/errors.rs b/locks-service/src/application/errors.rs index 6815be6..75a775d 100644 --- a/locks-service/src/application/errors.rs +++ b/locks-service/src/application/errors.rs @@ -18,6 +18,9 @@ pub enum ApplicationError { /// Full creator-scoped guarded path for structured internal handling. guarded_path: String, }, + /// A graceful deletion cutoff already blocks new proof Bundle IDs for the lock. + #[error("content lock deletion in progress")] + ContentLockDeletionInProgress, /// Persisted content-lock deletion state violates its internal invariants. #[error("invalid content lock deletion state: {message}")] InvalidContentLockDeletionState { diff --git a/locks-service/src/application/ports/verification.rs b/locks-service/src/application/ports/verification.rs index ad706b9..eec1528 100644 --- a/locks-service/src/application/ports/verification.rs +++ b/locks-service/src/application/ports/verification.rs @@ -15,6 +15,8 @@ pub trait VerificationTaskRepository: Send + Sync { /// /// Returns `DuplicateRecord` if a task with the same Task ID or public /// verification attempt handle (`creator`, `bundle_id`) already exists. + /// PostgreSQL additionally returns `ContentLockDeletionInProgress` when the + /// authoritative deletion cutoff already exists for a new Bundle ID. async fn insert_verification_task( &self, task: VerificationTaskRecord, diff --git a/locks-service/src/application/use_cases/complete_verification_task.rs b/locks-service/src/application/use_cases/complete_verification_task.rs index c8ec7c2..fed03ae 100644 --- a/locks-service/src/application/use_cases/complete_verification_task.rs +++ b/locks-service/src/application/use_cases/complete_verification_task.rs @@ -357,6 +357,7 @@ fn viewer_safe_failure_message(error: &ApplicationError) -> &'static str { ApplicationError::Storage { .. } | ApplicationError::DuplicateRecord { .. } | ApplicationError::ContentLockPathConflict { .. } + | ApplicationError::ContentLockDeletionInProgress | ApplicationError::InvalidContentLockDeletionState { .. } | ApplicationError::MissingRecord { .. } | ApplicationError::InvalidVerificationTaskTransition { .. } diff --git a/locks-service/src/application/use_cases/submit_proof_bundle.rs b/locks-service/src/application/use_cases/submit_proof_bundle.rs index 48632d0..47d62ad 100644 --- a/locks-service/src/application/use_cases/submit_proof_bundle.rs +++ b/locks-service/src/application/use_cases/submit_proof_bundle.rs @@ -70,20 +70,7 @@ impl<'a> SubmitProofBundleUseCase<'a> { if let Some(existing) = self.find_existing(&submitted_proof_bundle).await? { return Ok(existing); } - let creator = submitted_proof_bundle.pubky_lock_resource.creator().clone(); - - let task_id = self.task_ids.generate_task_id().await?; - let submitted_at = self.clock.now(); - let task = VerificationTaskRecord { - task_id, - creator, - submitted_proof_bundle, - status: VerificationTaskStatus::Pending, - submitted_at, - started_at: None, - completed_at: None, - failure_message: None, - }; + let task = self.prepare_task(submitted_proof_bundle).await?; match self.tasks.insert_verification_task(task.clone()).await { Ok(()) => Ok(VerificationTaskLifecycleView::from(task)), @@ -109,6 +96,23 @@ impl<'a> SubmitProofBundleUseCase<'a> { Err(error) => Err(error), } } + + /// Builds a new pending task without persisting it. + pub async fn prepare_task( + &self, + submitted_proof_bundle: SubmittedProofBundle, + ) -> Result { + Ok(VerificationTaskRecord { + task_id: self.task_ids.generate_task_id().await?, + creator: submitted_proof_bundle.pubky_lock_resource.creator().clone(), + submitted_proof_bundle, + status: VerificationTaskStatus::Pending, + submitted_at: self.clock.now(), + started_at: None, + completed_at: None, + failure_message: None, + }) + } } #[cfg(test)] diff --git a/locks-service/src/infrastructure/postgres/content_lock_deletions.rs b/locks-service/src/infrastructure/postgres/content_lock_deletions.rs index 3a94c7c..11412ca 100644 --- a/locks-service/src/infrastructure/postgres/content_lock_deletions.rs +++ b/locks-service/src/infrastructure/postgres/content_lock_deletions.rs @@ -17,6 +17,7 @@ use crate::application::{ }, ports::ContentLockDeletionRepository, }; +use crate::infrastructure::postgres::proof_admission::lock_proof_admission; const ROW_COLUMNS: &str = "job_id, creator, lock_id, frozen_content_lock, deletion_started_at, state, phase, attempt_count, next_attempt_at, force_requested_at, failure_code, claimed_by, claim_token, claim_expires_at"; @@ -56,6 +57,13 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { job.validate_frozen_identity()?; job.validate_state(false)?; let frozen = serde_json::to_value(&job.frozen_content_lock).map_err(storage_display)?; + let lock_resource = format!( + "{}/pub/locks.app/{}.json", + job.creator, + job.lock_id.as_str() + ); + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, &job.creator, &job.lock_id).await?; sqlx::query( "INSERT INTO content_lock_deletion_jobs (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, state, phase, @@ -73,10 +81,24 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { .bind(job.next_attempt_at) .bind(job.force_requested_at) .bind(job.failure_code.map(ContentLockDeletionFailureCode::as_str)) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(map_insert_error)?; - Ok(()) + sqlx::query( + "INSERT INTO content_lock_deletion_task_snapshot + (deletion_job_id, verification_task_id) + SELECT $1, task_id + FROM verification_tasks + WHERE creator = $2 + AND submitted_proof_bundle->>'pubky_lock_resource' = $3", + ) + .bind(job.job_id) + .bind(job.creator.to_string()) + .bind(lock_resource) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error) } async fn get_job( @@ -180,6 +202,28 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { message: "deletion phase must advance to its immediate successor".to_owned(), }); } + if next_phase == ContentLockDeletionPhase::StartPaymentDrain { + let has_unready_paykit_admission = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS ( + SELECT 1 + FROM content_lock_deletion_task_snapshot AS snapshot + JOIN paykit_task_admissions AS admission + ON admission.verification_task_id = snapshot.verification_task_id + WHERE snapshot.deletion_job_id = $1 AND admission.ready = FALSE + )", + ) + .bind(job_id) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if has_unready_paykit_admission { + return Err(ApplicationError::InvalidContentLockDeletionState { + message: + "payment drain cannot start before reserved Paykit admissions are ready" + .to_owned(), + }); + } + } let sql = format!( "UPDATE content_lock_deletion_jobs SET phase = $2, state = 'queued', attempt_count = 0, next_attempt_at = NULL, @@ -432,11 +476,12 @@ mod tests { use std::{collections::BTreeMap, str::FromStr}; use locks_core::{ - ids::{CreatorPubky, GuardedResourceHash}, + ids::{BundleId, CreatorPubky, GuardedResourceHash, PubkyLockResource, TaskId}, lock_policy::{ AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, GuardedResource, LockLogic, - LockServerConfig, + LockServerConfig, VerifierType, }, + verification::{Proof, SUBMITTED_PROOF_BUNDLE_VERSION, SubmittedProofBundle}, }; use time::macros::datetime; use uuid::Uuid; @@ -444,18 +489,255 @@ mod tests { use super::PostgresContentLockDeletionRepository; use crate::{ application::{ + errors::ApplicationError, models::{ ContentLockDeletionFailureCode, ContentLockDeletionJob, ContentLockDeletionPhase, - ContentLockDeletionState, + ContentLockDeletionState, VerificationTaskRecord, VerificationTaskStatus, }, - ports::ContentLockDeletionRepository, + ports::{ContentLockDeletionRepository, VerificationTaskRepository}, }, - infrastructure::postgres::testing::TestDatabase, + infrastructure::postgres::{PostgresVerificationTaskRepository, testing::TestDatabase}, }; const CREATOR: &str = "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy"; const NOW: time::OffsetDateTime = datetime!(2026-08-12 05:00:00 UTC); + #[tokio::test] + async fn deletion_commit_order_is_the_authoritative_proof_admission_cutoff() { + let database = TestDatabase::create().await; + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let lock = content_lock(); + + let admitted_before = verification_task(&lock, BundleId::from_bytes([1; 16])); + tasks + .insert_verification_task(admitted_before.clone()) + .await + .unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock.clone(), NOW).unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + + let snapshotted: Vec = sqlx::query_scalar( + "SELECT verification_task_id + FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1", + ) + .bind(job.job_id) + .fetch_all(database.pool()) + .await + .unwrap(); + assert_eq!(snapshotted, vec![admitted_before.task_id.as_uuid()]); + assert_eq!( + tasks + .insert_verification_task(admitted_before.clone()) + .await, + Err(ApplicationError::DuplicateRecord { + record: "verification_task", + }) + ); + + let admitted_after = verification_task(&lock, BundleId::from_bytes([2; 16])); + assert_eq!( + tasks.insert_verification_task(admitted_after).await, + Err(ApplicationError::ContentLockDeletionInProgress) + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn concurrent_deletion_and_new_bundle_have_one_serialized_cutoff_order() { + for iteration in 0..20_u8 { + let database = TestDatabase::create().await; + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let lock = content_lock(); + let task = verification_task(&lock, BundleId::from_bytes([iteration; 16])); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + + let (deletion_result, task_result) = tokio::join!( + deletions.insert_job(job.clone()), + tasks.insert_verification_task(task.clone()) + ); + deletion_result.unwrap(); + + let snapshotted = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 AND verification_task_id = $2 + )", + ) + .bind(job.job_id) + .bind(task.task_id.as_uuid()) + .fetch_one(database.pool()) + .await + .unwrap(); + match task_result { + Ok(()) => assert!(snapshotted), + Err(ApplicationError::ContentLockDeletionInProgress) => assert!(!snapshotted), + other => panic!("unexpected concurrent admission result: {other:?}"), + } + + database.cleanup().await; + } + } + + #[tokio::test] + async fn durable_paykit_reservation_commits_before_deletion_and_is_snapshotted() { + use crate::infrastructure::postgres::PostgresPaykitTaskAdmissionRepository; + + let database = TestDatabase::create().await; + let lock = content_lock(); + let task = verification_task(&lock, BundleId::from_bytes([3; 16])); + let admissions = PostgresPaykitTaskAdmissionRepository::new(database.pool().clone()); + let admission = admissions.reserve(task.clone()).await.unwrap(); + assert!(admission.requires_paykit); + + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository + .insert_job(ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap()) + .await + .unwrap(); + + let snapshotted = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_deletion_task_snapshot + WHERE verification_task_id = $1 + )", + ) + .bind(task.task_id.as_uuid()) + .fetch_one(database.pool()) + .await + .unwrap(); + assert!(snapshotted); + + database.cleanup().await; + } + + #[tokio::test] + async fn deletion_first_rejects_durable_paykit_reservation_before_external_work() { + use crate::infrastructure::postgres::PostgresPaykitTaskAdmissionRepository; + + let database = TestDatabase::create().await; + let lock = content_lock(); + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository + .insert_job(ContentLockDeletionJob::new(Uuid::new_v4(), lock.clone(), NOW).unwrap()) + .await + .unwrap(); + + let admissions = PostgresPaykitTaskAdmissionRepository::new(database.pool().clone()); + let mut external_calls = 0; + let result = admissions + .reserve(verification_task(&lock, BundleId::from_bytes([4; 16]))) + .await; + if result.is_ok() { + external_calls += 1; + } + assert!(matches!( + result, + Err(ApplicationError::ContentLockDeletionInProgress) + )); + assert_eq!(external_calls, 0); + + database.cleanup().await; + } + + #[tokio::test] + async fn payment_drain_phase_waits_for_every_snapshotted_paykit_reservation() { + use crate::infrastructure::postgres::PostgresPaykitTaskAdmissionRepository; + + let database = TestDatabase::create().await; + let lock = content_lock(); + let task = verification_task(&lock, BundleId::from_bytes([6; 16])); + let admissions = PostgresPaykitTaskAdmissionRepository::new(database.pool().clone()); + let admission = admissions.reserve(task).await.unwrap(); + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository + .insert_job(ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap()) + .await + .unwrap(); + let claim = repository + .claim_next("worker", NOW, NOW + time::Duration::seconds(60)) + .await + .unwrap() + .unwrap(); + + let blocked = repository + .advance_phase( + claim.job.job_id, + "worker", + claim.claim_token, + NOW, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await; + assert!(matches!( + blocked, + Err(ApplicationError::InvalidContentLockDeletionState { message }) + if message == "payment drain cannot start before reserved Paykit admissions are ready" + )); + let still_running = repository + .get_job(&claim.job.creator, &claim.job.lock_id) + .await + .unwrap() + .unwrap(); + assert_eq!(still_running.state, ContentLockDeletionState::Running); + assert_eq!(still_running.phase, ContentLockDeletionPhase::Withdraw); + + admissions.mark_ready(&admission.task).await.unwrap(); + let advanced = repository + .advance_phase( + claim.job.job_id, + "worker", + claim.claim_token, + NOW, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(advanced.phase, ContentLockDeletionPhase::StartPaymentDrain); + + database.cleanup().await; + } + + #[tokio::test] + async fn paykit_admission_insert_failure_rolls_back_verification_task() { + use crate::infrastructure::postgres::PostgresPaykitTaskAdmissionRepository; + + let database = TestDatabase::create().await; + sqlx::query( + "CREATE FUNCTION reject_paykit_admission() RETURNS trigger AS $$ + BEGIN + RAISE EXCEPTION 'injected admission failure'; + END; + $$ LANGUAGE plpgsql", + ) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "CREATE TRIGGER reject_paykit_admission + BEFORE INSERT ON paykit_task_admissions + FOR EACH ROW EXECUTE FUNCTION reject_paykit_admission()", + ) + .execute(database.pool()) + .await + .unwrap(); + + let admissions = PostgresPaykitTaskAdmissionRepository::new(database.pool().clone()); + let task = verification_task(&content_lock(), BundleId::from_bytes([5; 16])); + assert!(admissions.reserve(task).await.is_err()); + let task_count = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM verification_tasks") + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(task_count, 0); + + database.cleanup().await; + } + #[tokio::test] async fn persists_and_fences_the_full_job_lifecycle_across_repository_recreation() { let database = TestDatabase::create().await; @@ -668,6 +950,35 @@ mod tests { database.cleanup().await; } + fn verification_task(lock: &ContentLock, bundle_id: BundleId) -> VerificationTaskRecord { + let lock_resource = PubkyLockResource::from_str(&format!( + "{}/pub/locks.app/{}.json", + lock.creator, + lock.lock_id().unwrap() + )) + .unwrap(); + VerificationTaskRecord { + task_id: TaskId::from_str(&Uuid::new_v4().to_string()).unwrap(), + creator: lock.creator.clone(), + submitted_proof_bundle: SubmittedProofBundle { + version: SUBMITTED_PROOF_BUNDLE_VERSION, + bundle_id, + pubky_lock_resource: lock_resource, + reader_public_key: None, + proofs: vec![Proof { + criterion_id: "criterion-1".to_owned(), + verifier_type: VerifierType::DevStatic, + payload: serde_json::json!({"satisfied": true}), + }], + }, + status: VerificationTaskStatus::Pending, + submitted_at: NOW, + started_at: None, + completed_at: None, + failure_message: None, + } + } + fn content_lock() -> ContentLock { ContentLock { version: CONTENT_LOCK_VERSION, diff --git a/locks-service/src/infrastructure/postgres/migrations.rs b/locks-service/src/infrastructure/postgres/migrations.rs index 741d053..98b492a 100644 --- a/locks-service/src/infrastructure/postgres/migrations.rs +++ b/locks-service/src/infrastructure/postgres/migrations.rs @@ -52,6 +52,8 @@ mod tests { assert_table_exists(&mut connection, "content_lock_ownership").await; assert_table_exists(&mut connection, "content_lock_deletion_jobs").await; assert_table_exists(&mut connection, "content_lock_force_deletion_receipts").await; + assert_table_exists(&mut connection, "content_lock_deletion_task_snapshot").await; + assert_table_exists(&mut connection, "paykit_task_admissions").await; assert_column_exists(&mut connection, "verification_tasks", "creator").await; assert_column_exists(&mut connection, "verification_tasks", "bundle_id").await; assert_column_exists(&mut connection, "verification_tasks", "next_attempt_at").await; diff --git a/locks-service/src/infrastructure/postgres/mod.rs b/locks-service/src/infrastructure/postgres/mod.rs index 3639734..43c6845 100644 --- a/locks-service/src/infrastructure/postgres/mod.rs +++ b/locks-service/src/infrastructure/postgres/mod.rs @@ -14,6 +14,8 @@ pub mod creator_connect_flows; pub mod errors; pub mod frontend_sessions; pub mod migrations; +mod proof_admission; +pub use proof_admission::PostgresPaykitTaskAdmissionRepository; #[cfg(test)] pub(crate) mod testing; pub mod verification_task_claims; diff --git a/locks-service/src/infrastructure/postgres/proof_admission.rs b/locks-service/src/infrastructure/postgres/proof_admission.rs new file mode 100644 index 0000000..3e68ca8 --- /dev/null +++ b/locks-service/src/infrastructure/postgres/proof_admission.rs @@ -0,0 +1,212 @@ +use locks_core::ids::{CreatorPubky, LockId}; +use sqlx::{PgPool, Postgres, Transaction}; + +use crate::application::errors::ApplicationError; +use crate::application::models::VerificationTaskRecord; +use locks_core::verification::SubmittedProofBundle; + +use super::verification_tasks::{ + VERIFICATION_TASK_ROW_COLUMNS, VerificationTaskRow, VerificationTaskWriteRow, row_to_task, +}; + +const PROOF_ADMISSION_LOCK_NAMESPACE: &str = "locks:proof-admission:v1"; + +/// Result of durably reserving one Paykit-backed proof admission. +#[derive(Debug)] +pub struct PaykitTaskAdmission { + /// The durable task associated with the public Bundle handle. + pub task: VerificationTaskRecord, + /// Whether the caller must create/reconcile the Paykit invoice before making the task claimable. + pub requires_paykit: bool, +} + +/// PostgreSQL coordinator for durable persist-before-Paykit admission. +#[derive(Debug, Clone)] +pub struct PostgresPaykitTaskAdmissionRepository { + pool: PgPool, +} + +impl PostgresPaykitTaskAdmissionRepository { + /// Creates a coordinator backed by the migrated PostgreSQL pool. + pub fn new(pool: PgPool) -> Self { + Self { pool } + } + + /// Returns durable replay state without consulting mutable lock or reader discovery state. + pub async fn find_existing( + &self, + submitted: &SubmittedProofBundle, + ) -> Result, ApplicationError> { + let sql = format!( + "SELECT {VERIFICATION_TASK_ROW_COLUMNS}, + COALESCE(admission.ready, TRUE) AS paykit_ready + FROM verification_tasks AS task + LEFT JOIN paykit_task_admissions AS admission + ON admission.verification_task_id = task.task_id + WHERE task.creator = $1 AND task.bundle_id = $2" + ); + let Some(existing) = sqlx::query_as::<_, PaykitAdmissionRow>(&sql) + .bind(submitted.pubky_lock_resource.creator().to_string()) + .bind(submitted.bundle_id.to_string()) + .fetch_optional(&self.pool) + .await + .map_err(storage_error)? + else { + return Ok(None); + }; + let ready = existing.paykit_ready; + let existing = row_to_task(existing.task)?; + if existing.submitted_proof_bundle != *submitted { + return Err(ApplicationError::VerificationTaskConflict); + } + Ok(Some(PaykitTaskAdmission { + task: existing, + requires_paykit: !ready, + })) + } + + /// Reserves a task before Paykit mutation, serialized against deletion start. + pub async fn reserve( + &self, + task: VerificationTaskRecord, + ) -> Result { + let row = VerificationTaskWriteRow::try_from(&task)?; + let lock_id = task.submitted_proof_bundle.pubky_lock_resource.lock_id(); + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, &task.creator, lock_id).await?; + + let existing_sql = format!( + "SELECT {VERIFICATION_TASK_ROW_COLUMNS}, + COALESCE(admission.ready, TRUE) AS paykit_ready + FROM verification_tasks AS task + LEFT JOIN paykit_task_admissions AS admission + ON admission.verification_task_id = task.task_id + WHERE task.creator = $1 AND task.bundle_id = $2" + ); + if let Some(existing) = sqlx::query_as::<_, PaykitAdmissionRow>(&existing_sql) + .bind(&row.creator) + .bind(&row.bundle_id) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)? + { + let ready = existing.paykit_ready; + let existing = row_to_task(existing.task)?; + if existing.submitted_proof_bundle != task.submitted_proof_bundle { + return Err(ApplicationError::VerificationTaskConflict); + } + transaction.commit().await.map_err(storage_error)?; + return Ok(PaykitTaskAdmission { + task: existing, + requires_paykit: !ready, + }); + } + + let deletion_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_deletion_jobs WHERE creator = $1 AND lock_id = $2 + )", + ) + .bind(&row.creator) + .bind(lock_id.to_string()) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if deletion_exists { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + + insert_task(&mut transaction, row).await?; + sqlx::query( + "INSERT INTO paykit_task_admissions (verification_task_id, ready) + VALUES ($1::uuid, FALSE)", + ) + .bind(task.task_id.to_string()) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + + Ok(PaykitTaskAdmission { + task, + requires_paykit: true, + }) + } + + /// Makes a reserved task claimable after Paykit confirms invoice creation/replay. + pub async fn mark_ready(&self, task: &VerificationTaskRecord) -> Result<(), ApplicationError> { + let result = sqlx::query( + "UPDATE paykit_task_admissions + SET ready = TRUE, ready_at = COALESCE(ready_at, now()) + WHERE verification_task_id = $1::uuid", + ) + .bind(task.task_id.to_string()) + .execute(&self.pool) + .await + .map_err(storage_error)?; + if result.rows_affected() == 0 { + return Err(ApplicationError::MissingRecord { + record: "paykit_task_admission", + }); + } + Ok(()) + } +} + +#[derive(sqlx::FromRow)] +struct PaykitAdmissionRow { + #[sqlx(flatten)] + task: VerificationTaskRow, + paykit_ready: bool, +} + +async fn insert_task( + transaction: &mut Transaction<'_, Postgres>, + row: VerificationTaskWriteRow, +) -> Result<(), ApplicationError> { + sqlx::query( + "INSERT INTO verification_tasks ( + task_id, creator, bundle_id, status, submitted_proof_bundle, + submitted_at, started_at, completed_at, failure_message + ) VALUES ($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9)", + ) + .bind(row.task_id) + .bind(row.creator) + .bind(row.bundle_id) + .bind(row.status) + .bind(row.submitted_proof_bundle) + .bind(row.submitted_at) + .bind(row.started_at) + .bind(row.completed_at) + .bind(row.failure_message) + .execute(&mut **transaction) + .await + .map_err(storage_error)?; + Ok(()) +} + +pub(super) async fn lock_proof_admission( + transaction: &mut Transaction<'_, Postgres>, + creator: &CreatorPubky, + lock_id: &LockId, +) -> Result<(), ApplicationError> { + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(proof_admission_lock_key(creator, lock_id)) + .execute(&mut **transaction) + .await + .map_err(storage_error)?; + Ok(()) +} + +fn proof_admission_lock_key(creator: &CreatorPubky, lock_id: &LockId) -> String { + format!( + "{PROOF_ADMISSION_LOCK_NAMESPACE}:{creator}:{}", + lock_id.as_str() + ) +} + +fn storage_error(error: sqlx::Error) -> ApplicationError { + ApplicationError::Storage { + message: error.to_string(), + } +} diff --git a/locks-service/src/infrastructure/postgres/verification_task_claims.rs b/locks-service/src/infrastructure/postgres/verification_task_claims.rs index cd874fa..53b9b7d 100644 --- a/locks-service/src/infrastructure/postgres/verification_task_claims.rs +++ b/locks-service/src/infrastructure/postgres/verification_task_claims.rs @@ -50,6 +50,11 @@ impl VerificationTaskClaimer for PostgresVerificationTaskClaimer { WHERE ((status = 'pending' AND (next_attempt_at IS NULL OR next_attempt_at <= $3)) OR (status = 'in_progress' AND claim_expires_at < $3)) + AND NOT EXISTS ( + SELECT 1 FROM paykit_task_admissions + WHERE verification_task_id = verification_tasks.task_id + AND ready = FALSE + ) AND creator = split_part(submitted_proof_bundle->>'pubky_lock_resource', '/', 1) AND bundle_id = submitted_proof_bundle->>'bundle_id' ORDER BY submitted_at @@ -230,6 +235,51 @@ mod tests { database.cleanup().await; } + #[tokio::test] + async fn paykit_reservation_is_not_claimable_until_marked_ready() { + use crate::infrastructure::postgres::PostgresPaykitTaskAdmissionRepository; + + let database = TestDatabase::create().await; + let claimer = PostgresVerificationTaskClaimer::new(database.pool().clone()); + let admissions = PostgresPaykitTaskAdmissionRepository::new(database.pool().clone()); + let pending = task( + "018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d15", + VerificationTaskStatus::Pending, + datetime!(2026-05-29 12:00:00 UTC), + ); + + let first = admissions.reserve(pending.clone()).await.unwrap(); + assert!(first.requires_paykit); + assert!( + claimer + .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .await + .unwrap() + .is_none() + ); + + let replay = admissions.reserve(pending.clone()).await.unwrap(); + assert!(replay.requires_paykit); + assert_eq!(replay.task, pending); + + admissions.mark_ready(&pending).await.unwrap(); + let ready_replay = admissions.reserve(pending.clone()).await.unwrap(); + assert!(!ready_replay.requires_paykit); + assert_eq!(ready_replay.task, pending); + assert_eq!( + claimer + .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .await + .unwrap() + .unwrap() + .task + .task_id, + pending.task_id + ); + + database.cleanup().await; + } + #[tokio::test] async fn does_not_claim_terminal_tasks() { let database = TestDatabase::create().await; diff --git a/locks-service/src/infrastructure/postgres/verification_tasks.rs b/locks-service/src/infrastructure/postgres/verification_tasks.rs index f206ccb..77a4513 100644 --- a/locks-service/src/infrastructure/postgres/verification_tasks.rs +++ b/locks-service/src/infrastructure/postgres/verification_tasks.rs @@ -9,6 +9,7 @@ use locks_core::verification::SubmittedProofBundle; use crate::application::errors::ApplicationError; use crate::application::models::{VerificationTaskRecord, VerificationTaskStatus}; use crate::application::ports::VerificationTaskRepository; +use crate::infrastructure::postgres::proof_admission::lock_proof_admission; /// Postgres-backed repository for Lock Server private verification task state. #[derive(Debug, Clone)] @@ -18,27 +19,27 @@ pub struct PostgresVerificationTaskRepository { #[derive(Debug, FromRow)] pub(super) struct VerificationTaskRow { - task_id: String, - creator: String, - bundle_id: String, - status: String, - submitted_proof_bundle: serde_json::Value, - submitted_at: time::OffsetDateTime, - started_at: Option, - completed_at: Option, - failure_message: Option, + pub(super) task_id: String, + pub(super) creator: String, + pub(super) bundle_id: String, + pub(super) status: String, + pub(super) submitted_proof_bundle: serde_json::Value, + pub(super) submitted_at: time::OffsetDateTime, + pub(super) started_at: Option, + pub(super) completed_at: Option, + pub(super) failure_message: Option, } -struct VerificationTaskWriteRow { - task_id: String, - creator: String, - bundle_id: String, - status: &'static str, - submitted_proof_bundle: serde_json::Value, - submitted_at: time::OffsetDateTime, - started_at: Option, - completed_at: Option, - failure_message: Option, +pub(super) struct VerificationTaskWriteRow { + pub(super) task_id: String, + pub(super) creator: String, + pub(super) bundle_id: String, + pub(super) status: &'static str, + pub(super) submitted_proof_bundle: serde_json::Value, + pub(super) submitted_at: time::OffsetDateTime, + pub(super) started_at: Option, + pub(super) completed_at: Option, + pub(super) failure_message: Option, } pub(super) const VERIFICATION_TASK_ROW_COLUMNS: &str = " @@ -66,6 +67,40 @@ impl VerificationTaskRepository for PostgresVerificationTaskRepository { task: VerificationTaskRecord, ) -> Result<(), ApplicationError> { let row = VerificationTaskWriteRow::try_from(&task)?; + let lock_id = task.submitted_proof_bundle.pubky_lock_resource.lock_id(); + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, &task.creator, lock_id).await?; + + let handle_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS ( + SELECT 1 FROM verification_tasks WHERE creator = $1 AND bundle_id = $2 + )", + ) + .bind(&row.creator) + .bind(&row.bundle_id) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if handle_exists { + return Err(ApplicationError::DuplicateRecord { + record: "verification_task", + }); + } + + let deletion_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_deletion_jobs WHERE creator = $1 AND lock_id = $2 + )", + ) + .bind(&row.creator) + .bind(lock_id.to_string()) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if deletion_exists { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + let result = sqlx::query( "INSERT INTO verification_tasks ( task_id, @@ -90,7 +125,7 @@ impl VerificationTaskRepository for PostgresVerificationTaskRepository { .bind(row.started_at) .bind(row.completed_at) .bind(row.failure_message) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(storage_error)?; @@ -100,7 +135,7 @@ impl VerificationTaskRepository for PostgresVerificationTaskRepository { }); } - Ok(()) + transaction.commit().await.map_err(storage_error) } async fn update_verification_task( @@ -150,7 +185,12 @@ impl VerificationTaskRepository for PostgresVerificationTaskRepository { let sql = format!( "SELECT {VERIFICATION_TASK_ROW_COLUMNS} FROM verification_tasks - WHERE task_id = $1::uuid" + WHERE task_id = $1::uuid + AND NOT EXISTS ( + SELECT 1 FROM paykit_task_admissions + WHERE verification_task_id = verification_tasks.task_id + AND ready = FALSE + )" ); let row = sqlx::query_as::<_, VerificationTaskRow>(&sql) .bind(task_id.to_string()) @@ -169,7 +209,12 @@ impl VerificationTaskRepository for PostgresVerificationTaskRepository { let sql = format!( "SELECT {VERIFICATION_TASK_ROW_COLUMNS} FROM verification_tasks - WHERE creator = $1 AND bundle_id = $2" + WHERE creator = $1 AND bundle_id = $2 + AND NOT EXISTS ( + SELECT 1 FROM paykit_task_admissions + WHERE verification_task_id = verification_tasks.task_id + AND ready = FALSE + )" ); let row = sqlx::query_as::<_, VerificationTaskRow>(&sql) .bind(creator.to_string()) From a8ca14d295f2d37205a9e8983d4646951f56f295 Mon Sep 17 00:00:00 2001 From: dzdidi Date: Wed, 12 Aug 2026 10:46:37 -0300 Subject: [PATCH 6/6] feat(deletion): add creator-authorized graceful content-lock deletion Signed-off-by: dzdidi --- docs/API.md | 24 +- docs/DOMAIN_MODEL.md | 2 +- ...26-08-10-graceful-content-lock-deletion.md | 14 +- locks-sdk/bindings/js/src/creator.rs | 147 +++- locks-sdk/bindings/js/src/lib.rs | 4 +- locks-sdk/src/creator.rs | 40 ++ locks-sdk/src/lib.rs | 5 +- locks-sdk/tests/public_api.rs | 57 +- locks-server/src/api/creator_publishing.rs | 355 +++++++++- locks-server/src/api/dtos.rs | 8 + locks-server/src/api/errors.rs | 3 + locks-server/src/api/routes.rs | 11 +- locks-server/src/api/routes/tests.rs | 637 +++++++++++++++++- locks-server/src/app_state/mod.rs | 31 +- .../0013_content_lock_publication_intents.sql | 8 + .../models/content_lock_deletion.rs | 11 + .../ports/content_lock_deletion.rs | 47 +- .../src/application/ports/lock_policy.rs | 8 + .../use_cases/complete_verification_task.rs | 8 + .../use_cases/create_content_lock.rs | 388 ++++++++++- .../use_cases/credential_flow_tests.rs | 8 + .../validate_paykit_payment_submission.rs | 8 + .../memory/content_lock_deletions.rs | 151 ++++- .../infrastructure/memory/content_locks.rs | 13 + .../postgres/content_lock_deletions.rs | 429 +++++++++++- .../src/infrastructure/postgres/migrations.rs | 1 + .../src/infrastructure/pubky/content_locks.rs | 52 +- locks-service/tests/content_lock_deletions.rs | 25 +- 28 files changed, 2369 insertions(+), 126 deletions(-) create mode 100644 locks-service/migrations/0013_content_lock_publication_intents.sql diff --git a/docs/API.md b/docs/API.md index 0c61d95..eafb5a7 100644 --- a/docs/API.md +++ b/docs/API.md @@ -19,7 +19,7 @@ The Lock Server has one non-production route family and one authenticated creato - `POST /verification-task-completions` - Requires `runtime.environment = "development"`. - `staging` and `production` never mount it. -- Authenticated creator publishing routes: `PUT /creator/priv-resources/content/`, `DELETE /creator/priv-resources/content/`, `POST /creator/content-locks`, `POST /creator/lock-service-config` +- Authenticated creator publishing routes: `PUT /creator/priv-resources/content/`, `DELETE /creator/priv-resources/content/`, `POST /creator/content-locks`, `DELETE /creator/content-locks/{lock_id}`, `GET /creator/content-locks/{lock_id}/deletion`, `POST /creator/lock-service-config` - Always Pubky homeserver-backed. - Can run in `development`, `staging`, or `production`. - Require `Authorization: Bearer `. @@ -47,7 +47,9 @@ Gated-off routes are plain Axum `404 Not Found` responses because the route is i | --- | --- | --- | --- | --- | | `PUT /creator/priv-resources/content/` | `200` JSON guarded-resource descriptor | Requires `Authorization: Bearer `. Raw bytes body; MIME from `Content-Type`. | No bearer secrets or raw bytes in response. | `400 invalid_request`, `401 frontend_session_unavailable`, `401 frontend_session_expired`, `413 payload_too_large`, `503 creator_authority_unavailable` | | `DELETE /creator/priv-resources/content/` | `204` empty response | Requires `Authorization: Bearer `. | No bearer secrets or raw bytes in response. | `401 frontend_session_unavailable`, `401 frontend_session_expired`, `404 guarded_resource_not_found`, `503 creator_authority_unavailable` | -| `POST /creator/content-locks` | `200` JSON content lock | Requires `Authorization: Bearer `. | No bearer secrets in response. | `400 invalid_request`, `401 frontend_session_unavailable`, `401 frontend_session_expired`, `404 guarded_resource_not_found`, `409 content_lock_path_conflict`, `503 creator_authority_unavailable` | +| `POST /creator/content-locks` | `200` JSON content lock | Requires `Authorization: Bearer `. | No bearer secrets in response. | `400 invalid_request`, `401 frontend_session_unavailable`, `401 frontend_session_expired`, `404 guarded_resource_not_found`, `409 content_lock_path_conflict`, `409 content_lock_deletion_in_progress`, `503 creator_authority_unavailable` | +| `DELETE /creator/content-locks/{lock_id}` | Graceful: `202` redacted lifecycle, or `200` completed absent postcondition. Replaying a failed graceful job requeues the same frozen manifest. `force=true`: synchronous `200` force summary when no active graceful job exists; an active graceful job is marked for worker escalation and returns `202`. | Requires `Authorization: Bearer `. Default and `graceful=true` are graceful; `force=true` is explicit and mutually exclusive. | No snapshots, task IDs, attempts, dependency errors, or force marker in lifecycle responses. Force summary contains only Lock ID, lock-deleted boolean, and failed guarded paths. | `400 invalid_request`, `400 invalid_identifier`, `401 frontend_session_unavailable`, `401 frontend_session_expired`, `503 creator_authority_unavailable` | +| `GET /creator/content-locks/{lock_id}/deletion` | `200` redacted lifecycle JSON | Requires `Authorization: Bearer `. | Contains only Lock ID, stable status, and an optional closed failure code. Permanent force receipts project as completed without exposing force mode. | `400 invalid_identifier`, `401 frontend_session_unavailable`, `401 frontend_session_expired`, `404 content_lock_deletion_not_found` | | `POST /creator/lock-service-config` | `200` JSON lock-service pointer | Requires `Authorization: Bearer `. | No bearer secrets in response. | `400 invalid_request`, `401 frontend_session_unavailable`, `401 frontend_session_expired`, `503 creator_authority_unavailable` | | `GET /connect` | `200` HTML Lock-Server-hosted connect shell | No bearer auth. Mounted when `[creator_authority_acquisition].enabled = true`; `return_to` must match `allowed_return_origins` or explicit wildcard policy. | HTML intentionally contains the secret-bearing Pubky authorization URL on Lock Server origin; response must not contain frontend session token, one-time code, or creator authority secret. | `400 invalid_request`, `503 creator_authority_unavailable`, `404` when route gated off | | `POST /connect/{flow_id}/complete` | `303` redirect to stored `return_to` | No bearer auth. Mounted when `[creator_authority_acquisition].enabled = true`; stored `return_to` is revalidated before redirect. | `Location` contains only callback `state` and one-time `code`; no authorization URL, frontend session token, or creator authority secret. | `400 invalid_request`, `404 creator_connect_flow_unavailable`, `410 creator_connect_flow_expired`, `503 creator_authority_unavailable`, `404` when route gated off | @@ -99,7 +101,7 @@ Stable error codes and statuses mirror `locks-server/src/api/errors.rs` tests: | `frontend_session_expired` | 401 | Frontend session token existed but expired. | | `frontend_session_state_mismatch` | 400 | One-time code exchange state did not match. | | `creator_authority_unavailable` | 503 | Creator-granted homeserver authority is unavailable or could not be revalidated. | -| `content_lock_path_conflict` | 409 | The creator-scoped guarded path already has an in-flight or published Content Lock owner. | +| `content_lock_path_conflict` | 409 | A creator-scoped guarded path already has an in-flight/published owner, or the canonical Content Lock publication itself is still in flight. | | `task_state_conflict` | 409 | Submission or completion conflicts with existing task state. | | `content_lock_deletion_in_progress` | 409 | A deletion cutoff committed before this new proof Bundle could be admitted. | | `unsupported_verifier_type` | 422 | Proof references a verifier unavailable in the current runtime. | @@ -327,6 +329,20 @@ Authorization: Bearer Success returns `204 No Content`. Missing resources return `404 guarded_resource_not_found`. +### `DELETE /creator/content-locks/{lock_id}` + +Requires the authenticated creator frontend session. With no query, or with `graceful=true`, it starts or replays the durable graceful deletion job. Queued and running work returns `202` with only `lock_id` and `status`; a completed-and-forgotten absent lock returns `200 { "lock_id": "...", "status": "completed" }`. + +`force=true` is explicit and cannot be combined with `graceful=true`. With no active graceful job, force deletion synchronously stores the permanent blocking receipt, removes the public lock/tombstone, and then best-effort deletes guarded resources. A terminal graceful job supplies its frozen manifest for this synchronous cleanup. It returns exactly `{ "lock_id": "...", "lock_deleted": true, "failed_resource_paths": ["..."] }`. With a queued or running graceful job, it revokes any current worker claim, durably requeues the job for force escalation, and returns that redacted job at `202`. An in-flight canonical publication returns redacted `409 content_lock_path_conflict`; force has not started and no receipt exists, so the creator may retry after publication reconciles. + +Unknown query keys, malformed or false booleans, and ambiguous modes return `400 invalid_request`. The accepted wire forms are exactly no query, `graceful=true`, or `force=true`. + +Rust SDK callers use `CreatorLocks::delete_content_lock(DeleteContentLockRequest { lock_id, mode })`, where `DeleteContentLockMode` is closed to `DefaultGraceful`, `ExplicitGraceful`, and `Force`. JS/WASM callers use `creator.deleteContentLock(lockId, options?)`; omitting `options` selects default graceful, while `new DeleteContentLockOptions(DeleteContentLockMode.ExplicitGraceful)` and `new DeleteContentLockOptions(DeleteContentLockMode.Force)` select the two explicit query modes. + +### `GET /creator/content-locks/{lock_id}/deletion` + +Returns only `{ "lock_id": "...", "status": "queued|running|completed|failed", "failure_code"?: "..." }`. The optional failure vocabulary is closed to `tombstone_missing`, `tombstone_replaced`, `retry_exhausted`, and `state_corrupt`. If neither a job nor permanent force receipt exists, it returns `404 content_lock_deletion_not_found`. + ### `POST /creator/content-locks` Fixtures: @@ -341,7 +357,7 @@ Creates or replaces a content lock from a resource set. A content lock may conta At least one resource is required. If a primary resource is present, its path must not also appear in `secondary_resources`. `secondary_resources` keys are full canonical private paths such as `/priv/locks.app/content/attachments/a.txt`. -With Pubky-backed repositories, this writes the public content lock JSON to the creator homeserver under its derived `content_lock_path`. Test-support composition may use in-memory repositories behind the same authenticated route contract. +With Pubky-backed repositories, this writes the public content lock JSON to the creator homeserver under its derived `content_lock_path`. Before external publication, Locks persists an opaque per-lock publication intent under the same PostgreSQL fence used by graceful deletion and force-receipt establishment. Deletion cannot start while that intent exists; force therefore cannot report permanent deletion and then lose a race to a late Pubky write. After an upsert error, Locks reads the canonical path: an exact expected lock is reconciled as published, proven absence permits reservation compensation, and a mismatched payload or failed read retains both ownership and intent for fail-closed operator reconciliation. The intent is removed only after path ownership is durably marked published or safely compensated. Test-support composition may use in-memory repositories behind the same authenticated route contract. Every referenced guarded resource must currently exist for the same creator/path and must match hash, content type, and size. If the creator has overwritten or deleted a guarded resource path, content lock creation rejects the stale descriptor. diff --git a/docs/DOMAIN_MODEL.md b/docs/DOMAIN_MODEL.md index b544c7c..72c806c 100644 --- a/docs/DOMAIN_MODEL.md +++ b/docs/DOMAIN_MODEL.md @@ -62,7 +62,7 @@ Completion is worker-owned in production-shaped runtime. The server runs an in-p The dev HTTP completion route, `POST /verification-task-completions`, is not a production route. It accepts `{ creator, bundle_id }`, resolves the internal task, and is mounted only when `runtime.environment = "development"`; `staging` and `production` never mount it. -Creator publishing routes are authenticated and Pubky homeserver-backed. `PUT /creator/priv-resources/content/`, `DELETE /creator/priv-resources/content/`, `POST /creator/content-locks`, and `POST /creator/lock-service-config` require `Authorization: Bearer `, derive creator from the Locks-local frontend session, and reject request-body creator spoofing. The raw guarded-resource upload body is bytes, not JSON/base64. +Creator publishing and deletion routes are authenticated and Pubky homeserver-backed. `PUT /creator/priv-resources/content/`, `DELETE /creator/priv-resources/content/`, `POST /creator/content-locks`, `DELETE /creator/content-locks/{lock_id}`, `GET /creator/content-locks/{lock_id}/deletion`, and `POST /creator/lock-service-config` require `Authorization: Bearer ` and derive creator from the Locks-local frontend session. The raw guarded-resource upload body is bytes, not JSON/base64. Deletion validates that any fetched Content Lock hashes to the requested Lock ID and belongs to that authenticated creator before freezing or deleting its manifest. Graceful-job creation/resume and permanent force-receipt creation share one canonical per-lock PostgreSQL fence, so active graceful work and a force receipt cannot coexist. Failed graceful replay requeues the same frozen job; synchronous force replaces terminal operational job state with the permanent receipt. Force against an active graceful job remains a durable worker escalation. Runtime health and readiness are Lock Server operator concerns. `GET /healthz` reports process liveness once the HTTP router is serving. `GET /readyz` reports whether runtime dependencies are usable: `ephemeral` readiness currently means in-memory process composition, while `persisted` readiness currently means the Postgres-backed runtime can ping its configured pool. Health/readiness responses must be small and secret-free; they must not expose database URLs, secret paths, Lock Server identities, worker IDs, task counts, Task IDs, claim metadata, submitted proof material, access credentials, or rate-limit counters. diff --git a/docs/plans/2026-08-10-graceful-content-lock-deletion.md b/docs/plans/2026-08-10-graceful-content-lock-deletion.md index 6397e7b..0e3519f 100644 --- a/docs/plans/2026-08-10-graceful-content-lock-deletion.md +++ b/docs/plans/2026-08-10-graceful-content-lock-deletion.md @@ -14,7 +14,7 @@ ## Status and provenance -- Plan status: **accepted product design; implementation not started**. +- Plan status: **accepted product design; Tasks 1–5 committed and Task 6 implemented pending commit; Tasks 7–10 remain**. - Repository inspected: `/home/u/Projects/Synonym/Pubky/locks-public`. - Planning base when written: clean `master` at `ba49a77`. - There has been no production deployment. New persistence may require a clean pre-production database; no historical backfill is required. @@ -59,11 +59,13 @@ 25. Creator-visible job status is only `queued|running|completed|failed`; failed responses include a stable secret-free `failure_code` only. 26. Missing or replaced tombstone before destructive work halts as failed. Creator restores the exact tombstone and repeats graceful DELETE to resume the same job. 27. Guarded paths are exclusive to one managed lock. Enforce unique `(creator, guarded_path)` ownership in PostgreSQL. There is no historical backfill. -28. Lock publication uses best-effort reservation compensation and accepts crash-orphaned ownership requiring operator cleanup; do not claim cross-system atomicity. +28. Lock publication uses best-effort ownership compensation and a durable opaque per-lock publication intent under the same PostgreSQL fence as deletion admission. Graceful/force deletion cannot start while publication is in flight. The intent is cleared only after ownership is durably published or failed publication is safely compensated. Process death can leave operator-reconciled intent/ownership state; do not claim cross-system atomicity. 29. Graceful final cleanup deletes guarded content first and tombstone last, purges Locks authorization/task/job state, asks Paykit to remove operational drain state, and releases path ownership. It forgets the deletion so the same canonical Lock ID may later be published fresh with new Bundle IDs. 30. Paykit retains terminal financial invoice/payment history; delayed old lifecycle events cannot reactivate a fresh publication. 31. New force deletion is synchronous: persist a permanent minimal blocking receipt, delete lock/tombstone first, then best-effort guarded resources. Do not drain Paykit/tasks/credentials. Return failed paths. A force-deleted Lock ID can never be republished. -32. `force=true` against an active graceful job persists `force_requested` and returns `202`; the worker escalates asynchronously under exclusive action ownership, skips drains, deletes tombstone then content, and finishes forced. +32. `force=true` against an active graceful job persists `force_requested`, revokes the current claim token/lease, requeues the same frozen job, and returns `202`; a fresh worker claim escalates asynchronously under exclusive action ownership, skips drains, deletes tombstone then content, and finishes forced. +33. Graceful job insertion/resume and permanent force-receipt establishment acquire the same canonical per-lock PostgreSQL fence. The durable result is either an active graceful job or a permanent force receipt, never both. Failed graceful replay requeues the same job and frozen manifest. Force against a terminal job atomically replaces that operational row with the permanent receipt before synchronous external deletion. +34. Any Content Lock fetched from Pubky for deletion must hash to the requested Lock ID and name the authenticated creator before its manifest is frozen or used for resource deletion. ### Source-derived constraints @@ -233,6 +235,8 @@ DELETE /creator/content-locks/{lock_id}?graceful=true Starts/replays/resumes graceful deletion and returns `202` for queued/running work. A completed-and-forgotten absent lock is an idempotent absent postcondition. +Queued/running deletion and deletion status use the closed body `{ "lock_id": "...", "status": "queued|running|completed|failed", "failure_code"?: "..." }`. If both the canonical lock and deletion job are absent, graceful DELETE returns `200` with `{ "lock_id": "...", "status": "completed" }`. + ```http DELETE /creator/content-locks/{lock_id}?force=true ``` @@ -240,6 +244,8 @@ DELETE /creator/content-locks/{lock_id}?force=true - No graceful job: synchronous `200` force summary. - Existing graceful job: persist `force_requested`, return `202` job status. +The synchronous force summary is exactly `{ "lock_id": "...", "lock_deleted": true, "failed_resource_paths": ["..."] }`. It does not expose a force mode or internal receipt. + Reject `force=true&graceful=true`, unknown fields, malformed booleans, and duplicate conflicting query values. ```http @@ -248,6 +254,8 @@ GET /creator/content-locks/{lock_id}/deletion Authenticated response contains Lock ID and `status`; include `failure_code` only for failed jobs. The closed stable vocabulary is exactly `tombstone_missing`, `tombstone_replaced`, `retry_exhausted`, and `state_corrupt`. Do not expose phases, leases, retries, Bundle IDs, readers, credentials, paths, Paykit IDs, or dependency errors. +If no job or force receipt exists, status returns `404 content_lock_deletion_not_found`. A permanent force receipt projects as `{ "lock_id": "...", "status": "completed" }` without exposing force mode. + ## Internal state model Internal phase names are not public API. The implementation should represent at least: diff --git a/locks-sdk/bindings/js/src/creator.rs b/locks-sdk/bindings/js/src/creator.rs index fc1900f..1031446 100644 --- a/locks-sdk/bindings/js/src/creator.rs +++ b/locks-sdk/bindings/js/src/creator.rs @@ -8,7 +8,7 @@ use crate::session::{BrowserPkarrResolver, fetch_authorized_empty, fetch_authori #[cfg(any(test, target_arch = "wasm32"))] use crate::session::{JsAuthorizedRequestPlan, JsRequestBody}; #[cfg(any(test, target_arch = "wasm32"))] -use locks_core::ids::LockServerPubky; +use locks_core::ids::{LockId, LockServerPubky}; #[cfg(any(test, target_arch = "wasm32"))] use std::str::FromStr; use wasm_bindgen::prelude::*; @@ -65,6 +65,32 @@ impl DeleteGuardedResourceOptions { } } +#[wasm_bindgen] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeleteContentLockMode { + DefaultGraceful, + ExplicitGraceful, + Force, +} + +#[wasm_bindgen] +pub struct DeleteContentLockOptions { + mode: DeleteContentLockMode, +} + +#[wasm_bindgen] +impl DeleteContentLockOptions { + #[wasm_bindgen(constructor)] + pub fn new(mode: DeleteContentLockMode) -> Self { + Self { mode } + } + + #[wasm_bindgen(getter)] + pub fn mode(&self) -> DeleteContentLockMode { + self.mode + } +} + #[derive(Debug, Clone, Default)] struct CreateContentLockRequestBuilderState { primary_resource: Option, @@ -365,6 +391,41 @@ impl Creator { fetch_authorized_empty(&request).await } + #[cfg(target_arch = "wasm32")] + #[wasm_bindgen(js_name = deleteContentLock)] + pub async fn delete_content_lock( + &self, + lock_id: String, + options: Option, + ) -> crate::js_error::JsResult { + let resolver = BrowserPkarrResolver::new_with_options(self.session.options()) + .map_err(|err| crate::js_error::invalid_input(err.to_string()))?; + let request = self + .build_delete_content_lock_request(&lock_id, options.as_ref()) + .map_err(crate::js_error::invalid_input)? + .prepare_with_pkarr_resolver(&resolver, None) + .await + .map_err(|err| crate::js_error::invalid_input(err.to_string()))?; + fetch_authorized_json(&request).await + } + + #[cfg(target_arch = "wasm32")] + #[wasm_bindgen(js_name = contentLockDeletionStatus)] + pub async fn content_lock_deletion_status( + &self, + lock_id: String, + ) -> crate::js_error::JsResult { + let resolver = BrowserPkarrResolver::new_with_options(self.session.options()) + .map_err(|err| crate::js_error::invalid_input(err.to_string()))?; + let request = self + .build_content_lock_deletion_status_request(&lock_id) + .map_err(crate::js_error::invalid_input)? + .prepare_with_pkarr_resolver(&resolver, None) + .await + .map_err(|err| crate::js_error::invalid_input(err.to_string()))?; + fetch_authorized_json(&request).await + } + #[cfg(target_arch = "wasm32")] #[wasm_bindgen(js_name = setLockServicePointer)] pub async fn set_lock_service_pointer( @@ -418,6 +479,44 @@ impl Creator { self.authorized_request_plan(request) } + #[cfg(any(test, target_arch = "wasm32"))] + pub(crate) fn build_delete_content_lock_request( + &self, + lock_id: &str, + options: Option<&DeleteContentLockOptions>, + ) -> Result { + let lock_id = LockId::from_str(lock_id).map_err(|err| format!("invalid lock id: {err}"))?; + let mode = match options.map(|options| options.mode) { + None | Some(DeleteContentLockMode::DefaultGraceful) => { + locks_sdk::DeleteContentLockMode::DefaultGraceful + } + Some(DeleteContentLockMode::ExplicitGraceful) => { + locks_sdk::DeleteContentLockMode::ExplicitGraceful + } + Some(DeleteContentLockMode::Force) => locks_sdk::DeleteContentLockMode::Force, + }; + Ok(self.authorized_request_plan( + self.session + .inner() + .creator() + .delete_content_lock(locks_sdk::DeleteContentLockRequest { lock_id, mode }), + )) + } + + #[cfg(any(test, target_arch = "wasm32"))] + pub(crate) fn build_content_lock_deletion_status_request( + &self, + lock_id: &str, + ) -> Result { + let lock_id = LockId::from_str(lock_id).map_err(|err| format!("invalid lock id: {err}"))?; + Ok(self.authorized_request_plan( + self.session + .inner() + .creator() + .get_content_lock_deletion(lock_id), + )) + } + #[cfg(any(test, target_arch = "wasm32"))] #[cfg_attr(target_arch = "wasm32", allow(dead_code))] pub(crate) fn build_set_lock_service_pointer_request( @@ -548,6 +647,52 @@ mod tests { assert_eq!(request.content_type, None); } + #[test] + fn content_lock_deletion_requests_delegate_to_closed_rust_sdk_routes() { + let creator = Creator::new(test_session()); + let lock_id = LockId::from_hash(locks_core::ids::LockHash::from_bytes([9; 32])); + + let graceful = creator + .build_delete_content_lock_request(&lock_id.to_string(), None) + .unwrap(); + assert_eq!(graceful.method, "DELETE"); + assert_eq!(graceful.path, format!("/creator/content-locks/{lock_id}")); + assert_eq!(graceful.authorization, "Bearer frontend-session-secret"); + + let explicit_graceful = creator + .build_delete_content_lock_request( + &lock_id.to_string(), + Some(&DeleteContentLockOptions::new( + DeleteContentLockMode::ExplicitGraceful, + )), + ) + .unwrap(); + assert_eq!( + explicit_graceful.path, + format!("/creator/content-locks/{lock_id}?graceful=true") + ); + + let force = creator + .build_delete_content_lock_request( + &lock_id.to_string(), + Some(&DeleteContentLockOptions::new(DeleteContentLockMode::Force)), + ) + .unwrap(); + assert_eq!( + force.path, + format!("/creator/content-locks/{lock_id}?force=true") + ); + + let status = creator + .build_content_lock_deletion_status_request(&lock_id.to_string()) + .unwrap(); + assert_eq!(status.method, "GET"); + assert_eq!( + status.path, + format!("/creator/content-locks/{lock_id}/deletion") + ); + } + #[test] fn create_content_lock_request_builder_primary_only_build_succeeds() { let builder = complete_builder(); diff --git a/locks-sdk/bindings/js/src/lib.rs b/locks-sdk/bindings/js/src/lib.rs index 6b0c589..a7b41b8 100644 --- a/locks-sdk/bindings/js/src/lib.rs +++ b/locks-sdk/bindings/js/src/lib.rs @@ -6,8 +6,8 @@ mod session; mod viewer; pub use creator::{ - CreateContentLockRequestBuilder, Creator, DeleteGuardedResourceOptions, - RegisterGuardedResourceOptions, SetLockServicePointerOptions, + CreateContentLockRequestBuilder, Creator, DeleteContentLockMode, DeleteContentLockOptions, + DeleteGuardedResourceOptions, RegisterGuardedResourceOptions, SetLockServicePointerOptions, }; pub use locks::{ ConnectCallback, ConnectUrlOptions, ExchangeFrontendSessionCodeOptions, Locks, LocksOptions, diff --git a/locks-sdk/src/creator.rs b/locks-sdk/src/creator.rs index 6edf4cc..104ee06 100644 --- a/locks-sdk/src/creator.rs +++ b/locks-sdk/src/creator.rs @@ -1,4 +1,5 @@ pub use locks_core::creator_publishing::{CreateContentLockRequest, SetLockServicePointerRequest}; +use locks_core::ids::LockId; use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode}; use serde::Serialize; use serde_json::Value; @@ -55,6 +56,19 @@ pub struct DeleteGuardedResourceRequest { pub path: String, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeleteContentLockMode { + DefaultGraceful, + ExplicitGraceful, + Force, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeleteContentLockRequest { + pub lock_id: LockId, + pub mode: DeleteContentLockMode, +} + impl CreatorLocks { pub fn new(session: LocksSession) -> Self { Self { session } @@ -100,6 +114,22 @@ impl CreatorLocks { self.delete_guarded_resource_request(request) } + pub fn delete_content_lock(&self, request: DeleteContentLockRequest) -> SdkRequest { + let suffix = match request.mode { + DeleteContentLockMode::DefaultGraceful => "", + DeleteContentLockMode::ExplicitGraceful => "?graceful=true", + DeleteContentLockMode::Force => "?force=true", + }; + self.empty_request( + "DELETE", + format!("/creator/content-locks/{}{suffix}", request.lock_id), + ) + } + + pub fn get_content_lock_deletion(&self, lock_id: LockId) -> SdkRequest { + self.empty_request("GET", format!("/creator/content-locks/{lock_id}/deletion")) + } + pub fn create_content_lock_request(&self, request: CreateContentLockRequest) -> SdkRequest { self.post_json("/creator/content-locks", request) } @@ -130,6 +160,16 @@ impl CreatorLocks { ), } } + + fn empty_request(&self, method: &'static str, path: String) -> SdkRequest { + SdkRequest { + method, + path, + authorization: self.session.authorization_header_value(), + content_type: String::new(), + body: SdkRequestBody::Bytes(Vec::new()), + } + } } pub(crate) fn encode_content_path(path: &str) -> String { diff --git a/locks-sdk/src/lib.rs b/locks-sdk/src/lib.rs index fe931a9..e387390 100644 --- a/locks-sdk/src/lib.rs +++ b/locks-sdk/src/lib.rs @@ -8,8 +8,9 @@ pub mod viewer; pub use client::LocksClient; pub use creator::{ - CreateContentLockRequest, CreatorLocks, DeleteGuardedResourceRequest, - RegisterGuardedResourceRequest, SdkRequest, SdkRequestBody, SetLockServicePointerRequest, + CreateContentLockRequest, CreatorLocks, DeleteContentLockMode, DeleteContentLockRequest, + DeleteGuardedResourceRequest, RegisterGuardedResourceRequest, SdkRequest, SdkRequestBody, + SetLockServicePointerRequest, }; pub use discovery::{ CreatorLockServicePointer, WellKnownLocksServer, content_lock_resource_url, diff --git a/locks-sdk/tests/public_api.rs b/locks-sdk/tests/public_api.rs index 5b9c580..9761f35 100644 --- a/locks-sdk/tests/public_api.rs +++ b/locks-sdk/tests/public_api.rs @@ -2,11 +2,11 @@ use std::str::FromStr; use locks_core::ids::{BundleId, CreatorPubky, LockServerPubky, PubkyLockResource}; use locks_sdk::{ - AccessCredentialResponse, CreatorLockServicePointer, DeleteGuardedResourceRequest, LocksClient, - LocksSession, ReadLockedResourceRequest, RegisterGuardedResourceRequest, - VerificationTaskHandleRequest, VerificationTaskLifecycleResponse, VerificationTaskStatus, - ViewerLocks, content_lock_resource_url, creator_lock_service_pointer_url, - lock_server_for_content_lock, + AccessCredentialResponse, CreatorLockServicePointer, DeleteContentLockMode, + DeleteContentLockRequest, DeleteGuardedResourceRequest, LocksClient, LocksSession, + ReadLockedResourceRequest, RegisterGuardedResourceRequest, VerificationTaskHandleRequest, + VerificationTaskLifecycleResponse, VerificationTaskStatus, ViewerLocks, + content_lock_resource_url, creator_lock_service_pointer_url, lock_server_for_content_lock, }; #[test] @@ -113,3 +113,50 @@ fn crate_root_exports_foundation_sdk_types() { assert_eq!(LocksSession::new("another").export_secret(), "another"); } + +#[test] +fn creator_content_lock_deletion_requests_use_closed_routes_and_modes() { + let creator = LocksSession::new("frontend-session-secret").creator(); + let lock_id = + locks_core::ids::LockId::from_str("000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG") + .unwrap(); + + let default_graceful = creator.delete_content_lock(DeleteContentLockRequest { + lock_id: lock_id.clone(), + mode: DeleteContentLockMode::DefaultGraceful, + }); + assert_eq!(default_graceful.method, "DELETE"); + assert_eq!( + default_graceful.path, + format!("/creator/content-locks/{lock_id}") + ); + assert_eq!( + default_graceful.authorization, + "Bearer frontend-session-secret" + ); + + let explicit_graceful = creator.delete_content_lock(DeleteContentLockRequest { + lock_id: lock_id.clone(), + mode: DeleteContentLockMode::ExplicitGraceful, + }); + assert_eq!( + explicit_graceful.path, + format!("/creator/content-locks/{lock_id}?graceful=true") + ); + + let force = creator.delete_content_lock(DeleteContentLockRequest { + lock_id: lock_id.clone(), + mode: DeleteContentLockMode::Force, + }); + assert_eq!( + force.path, + format!("/creator/content-locks/{lock_id}?force=true") + ); + + let status = creator.get_content_lock_deletion(lock_id.clone()); + assert_eq!(status.method, "GET"); + assert_eq!( + status.path, + format!("/creator/content-locks/{lock_id}/deletion") + ); +} diff --git a/locks-server/src/api/creator_publishing.rs b/locks-server/src/api/creator_publishing.rs index 23618db..d158fd6 100644 --- a/locks-server/src/api/creator_publishing.rs +++ b/locks-server/src/api/creator_publishing.rs @@ -1,8 +1,13 @@ use axum::Json; use axum::body::{Body, to_bytes}; -use axum::extract::rejection::JsonRejection; -use axum::extract::{Path, State}; +use axum::extract::rejection::{JsonRejection, QueryRejection}; +use axum::extract::{Path, Query, State}; use axum::http::{HeaderMap, StatusCode, header}; +use locks_core::ids::{ContentLockPath, CreatorPubky, LockId}; +use locks_service::application::models::{ + ContentLockDeletionFailureCode, ContentLockDeletionJob, ContentLockDeletionState, + PrepareForceDeletionResult, +}; use locks_service::application::use_cases::create_content_lock::{ CreateContentLockRequest, CreateContentLockUseCase, }; @@ -15,17 +20,358 @@ use locks_service::application::use_cases::register_guarded_resource::{ use locks_service::application::use_cases::set_lock_service_pointer::{ SetLockServicePointerRequest, SetLockServicePointerUseCase, }; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::str::FromStr; +use uuid::Uuid; use crate::api::auth::authenticated_creator_from_headers; use crate::api::dtos::{ AuthenticatedCreateContentLockHttpRequest, AuthenticatedSetLockServicePointerHttpRequest, - CreateContentLockHttpResponse, RegisterGuardedResourceHttpResponse, - SetLockServicePointerHttpResponse, + ContentLockDeletionStatusHttpResponse, CreateContentLockHttpResponse, + RegisterGuardedResourceHttpResponse, SetLockServicePointerHttpResponse, }; use crate::api::errors::{ApiError, ApiErrorCode}; use crate::api::extractors::{guarded_resource_path_from_tail, parse_json}; use crate::app_state::AppState; +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct DeleteContentLockQuery { + force: Option, + graceful: Option, +} + +pub(super) async fn delete_content_lock_for_authenticated_creator( + State(state): State, + Path(lock_id): Path, + headers: HeaderMap, + query: Result, QueryRejection>, +) -> Result<(StatusCode, Json), ApiError> { + let creator = authenticated_creator_from_headers(&state, &headers).await?; + let Query(query) = + query.map_err(|_| ApiError::new(ApiErrorCode::InvalidRequest, "invalid request"))?; + let force = match (query.force, query.graceful) { + (None, None | Some(true)) => false, + (Some(true), None) => true, + _ => { + return Err(ApiError::new( + ApiErrorCode::InvalidRequest, + "invalid request", + )); + } + }; + let lock_id = LockId::from_str(&lock_id) + .map_err(|_| ApiError::new(ApiErrorCode::InvalidIdentifier, "invalid lock id"))?; + + if force { + return force_delete_content_lock(&state, creator, lock_id).await; + } + + if state + .content_lock_deletions() + .has_force_receipt(&creator, &lock_id) + .await? + { + return Ok(( + StatusCode::OK, + Json(json!({ "lock_id": lock_id, "status": "completed" })), + )); + } + + if let Some(job) = state + .content_lock_deletions() + .get_job(&creator, &lock_id) + .await? + { + let job = if job.state == ContentLockDeletionState::Failed { + match state + .content_lock_deletions() + .resume_failed_job(&creator, &lock_id, state.clock().now()) + .await? + { + Some(resumed) => resumed, + None if state + .content_lock_deletions() + .has_force_receipt(&creator, &lock_id) + .await? => + { + return Ok(( + StatusCode::OK, + Json(json!({ "lock_id": lock_id, "status": "completed" })), + )); + } + None => job, + } + } else { + job + }; + let status = match job.state { + ContentLockDeletionState::Queued | ContentLockDeletionState::Running => { + StatusCode::ACCEPTED + } + ContentLockDeletionState::Completed | ContentLockDeletionState::Failed => { + StatusCode::OK + } + }; + return Ok(( + status, + Json(deletion_status_json(lock_id, job.state, job.failure_code)), + )); + } + + let path = ContentLockPath::from_lock_id(lock_id.clone()); + let Some(content_lock) = state + .content_locks() + .get_content_lock(&creator, &path) + .await? + else { + if state + .content_lock_deletions() + .publication_in_progress(&creator, &lock_id) + .await? + { + return Err(ApiError::new( + ApiErrorCode::ContentLockPathConflict, + "content lock publication is in progress", + )); + } + return Ok(( + StatusCode::OK, + Json(json!({ "lock_id": lock_id, "status": "completed" })), + )); + }; + + validate_content_lock_identity(&content_lock, &creator, &lock_id)?; + + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock, state.clock().now())?; + match state.content_lock_deletions().insert_job(job.clone()).await { + Ok(()) => Ok(( + StatusCode::ACCEPTED, + Json(deletion_status_json(lock_id, job.state, job.failure_code)), + )), + Err(locks_service::application::errors::ApplicationError::DuplicateRecord { + record: "content_lock_deletion_job", + }) => { + let persisted = state + .content_lock_deletions() + .get_job(&creator, &lock_id) + .await? + .ok_or_else(|| { + ApiError::new( + ApiErrorCode::InternalError, + "content lock deletion unavailable", + ) + })?; + let status = match persisted.state { + ContentLockDeletionState::Queued | ContentLockDeletionState::Running => { + StatusCode::ACCEPTED + } + ContentLockDeletionState::Completed | ContentLockDeletionState::Failed => { + StatusCode::OK + } + }; + Ok(( + status, + Json(deletion_status_json( + lock_id, + persisted.state, + persisted.failure_code, + )), + )) + } + Err( + locks_service::application::errors::ApplicationError::ContentLockDeletionInProgress, + ) if state + .content_lock_deletions() + .has_force_receipt(&creator, &lock_id) + .await? => + { + Ok(( + StatusCode::OK, + Json(json!({ "lock_id": lock_id, "status": "completed" })), + )) + } + Err(error) => Err(error.into()), + } +} + +async fn force_delete_content_lock( + state: &AppState, + creator: CreatorPubky, + lock_id: LockId, +) -> Result<(StatusCode, Json), ApiError> { + let path = ContentLockPath::from_lock_id(lock_id.clone()); + let existing_job = state + .content_lock_deletions() + .get_job(&creator, &lock_id) + .await?; + let published_content_lock = if existing_job.is_none() { + state + .content_locks() + .get_content_lock(&creator, &path) + .await? + } else { + None + }; + if let Some(content_lock) = published_content_lock.as_ref() { + validate_content_lock_identity(content_lock, &creator, &lock_id)?; + } + let content_lock = match state + .content_lock_deletions() + .prepare_force_deletion(&creator, &lock_id, state.clock().now()) + .await? + { + PrepareForceDeletionResult::PublicationInProgress => { + return Err(ApiError::new( + ApiErrorCode::ContentLockPathConflict, + "content lock publication is in progress", + )); + } + PrepareForceDeletionResult::Active(job) => { + return Ok(( + StatusCode::ACCEPTED, + Json(deletion_status_json(lock_id, job.state, job.failure_code)), + )); + } + PrepareForceDeletionResult::Synchronous(Some(job)) => Some(job.frozen_content_lock), + PrepareForceDeletionResult::Synchronous(None) => published_content_lock, + }; + + if let Some(content_lock) = content_lock.as_ref() { + validate_content_lock_identity(content_lock, &creator, &lock_id)?; + } + + state + .content_locks() + .delete_content_lock(&creator, &path) + .await?; + if state + .content_locks() + .get_content_lock(&creator, &path) + .await? + .is_some() + { + return Err(ApiError::new( + ApiErrorCode::InternalError, + "content lock deletion postcondition failed", + )); + } + + let mut failed_resource_paths = Vec::new(); + if let Some(content_lock) = content_lock { + let mut resource_paths = content_lock + .secondary_resources + .keys() + .cloned() + .collect::>(); + if let Some(primary) = content_lock.primary_resource { + resource_paths.push(primary.path); + } + resource_paths.sort(); + resource_paths.dedup(); + for resource_path in resource_paths { + if state + .guarded_resources() + .delete_guarded_resource(&creator, &resource_path) + .await + .is_err() + { + failed_resource_paths.push(resource_path); + } + } + } + + Ok(( + StatusCode::OK, + Json(json!({ + "lock_id": lock_id, + "lock_deleted": true, + "failed_resource_paths": failed_resource_paths + })), + )) +} + +fn validate_content_lock_identity( + content_lock: &locks_core::lock_policy::ContentLock, + creator: &CreatorPubky, + expected_lock_id: &LockId, +) -> Result<(), ApiError> { + let actual_lock_id = content_lock + .lock_id() + .map_err(|_| ApiError::new(ApiErrorCode::InternalError, "content lock unavailable"))?; + if &content_lock.creator != creator || &actual_lock_id != expected_lock_id { + return Err(ApiError::new( + ApiErrorCode::InternalError, + "content lock unavailable", + )); + } + Ok(()) +} + +fn deletion_status_json( + lock_id: LockId, + state: ContentLockDeletionState, + failure_code: Option, +) -> Value { + serde_json::to_value(deletion_status_response(lock_id, state, failure_code)) + .expect("deletion status response must serialize") +} + +pub(super) async fn get_content_lock_deletion_status_for_authenticated_creator( + State(state): State, + Path(lock_id): Path, + headers: HeaderMap, +) -> Result, ApiError> { + let creator = authenticated_creator_from_headers(&state, &headers).await?; + let lock_id = LockId::from_str(&lock_id) + .map_err(|_| ApiError::new(ApiErrorCode::InvalidIdentifier, "invalid lock id"))?; + if state + .content_lock_deletions() + .has_force_receipt(&creator, &lock_id) + .await? + { + return Ok(Json(ContentLockDeletionStatusHttpResponse { + lock_id, + status: "completed", + failure_code: None, + })); + } + if let Some(job) = state + .content_lock_deletions() + .get_job(&creator, &lock_id) + .await? + { + return Ok(Json(deletion_status_response( + lock_id, + job.state, + job.failure_code, + ))); + } + Err(ApiError::new( + ApiErrorCode::ContentLockDeletionNotFound, + "content lock deletion not found", + )) +} + +fn deletion_status_response( + lock_id: LockId, + state: ContentLockDeletionState, + failure_code: Option, +) -> ContentLockDeletionStatusHttpResponse { + let status = match state { + ContentLockDeletionState::Queued => "queued", + ContentLockDeletionState::Running => "running", + ContentLockDeletionState::Completed => "completed", + ContentLockDeletionState::Failed => "failed", + }; + ContentLockDeletionStatusHttpResponse { + lock_id, + status, + failure_code: failure_code.map(|code| code.as_str().to_owned()), + } +} + pub(super) async fn register_guarded_resource_empty_tail_for_authenticated_creator( State(state): State, headers: HeaderMap, @@ -98,6 +444,7 @@ pub(super) async fn create_content_lock_for_authenticated_creator( validate_content_lock_limits(&request, state.config().content_locks.clone())?; let use_case = CreateContentLockUseCase::new( state.content_locks().as_ref(), + state.content_lock_deletions().as_ref(), state.content_lock_ownership().as_ref(), state.guarded_resources().as_ref(), state.clock().as_ref(), diff --git a/locks-server/src/api/dtos.rs b/locks-server/src/api/dtos.rs index 16a0127..0fbd1a9 100644 --- a/locks-server/src/api/dtos.rs +++ b/locks-server/src/api/dtos.rs @@ -51,6 +51,14 @@ pub struct CreateContentLockHttpResponse { pub content_lock: ContentLock, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ContentLockDeletionStatusHttpResponse { + pub lock_id: LockId, + pub status: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub failure_code: Option, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct SetLockServicePointerHttpResponse { pub creator: CreatorPubky, diff --git a/locks-server/src/api/errors.rs b/locks-server/src/api/errors.rs index d13b91a..f695f49 100644 --- a/locks-server/src/api/errors.rs +++ b/locks-server/src/api/errors.rs @@ -25,6 +25,7 @@ pub enum ApiErrorCode { FrontendSessionStateMismatch, ContentLockPathConflict, ContentLockDeletionInProgress, + ContentLockDeletionNotFound, TaskStateConflict, UnsupportedVerifierType, PaykitNotConfigured, @@ -57,6 +58,7 @@ impl ApiErrorCode { Self::FrontendSessionStateMismatch => "frontend_session_state_mismatch", Self::ContentLockPathConflict => "content_lock_path_conflict", Self::ContentLockDeletionInProgress => "content_lock_deletion_in_progress", + Self::ContentLockDeletionNotFound => "content_lock_deletion_not_found", Self::TaskStateConflict => "task_state_conflict", Self::UnsupportedVerifierType => "unsupported_verifier_type", Self::PaykitNotConfigured => "paykit_not_configured", @@ -78,6 +80,7 @@ impl ApiErrorCode { Self::VerificationTaskNotFound | Self::GuardedResourceNotFound | Self::ContentLockNotFound + | Self::ContentLockDeletionNotFound | Self::CreatorConnectFlowUnavailable | Self::FrontendSessionCodeUnavailable => StatusCode::NOT_FOUND, Self::CreatorAuthorityUnavailable => StatusCode::SERVICE_UNAVAILABLE, diff --git a/locks-server/src/api/routes.rs b/locks-server/src/api/routes.rs index af7fce7..acbbe4b 100644 --- a/locks-server/src/api/routes.rs +++ b/locks-server/src/api/routes.rs @@ -8,8 +8,9 @@ use crate::api::creator_authority::{ exchange_frontend_session_code_route, frontend_session_signout_route, }; use crate::api::creator_publishing::{ - create_content_lock_for_authenticated_creator, + create_content_lock_for_authenticated_creator, delete_content_lock_for_authenticated_creator, delete_guarded_resource_for_authenticated_creator, + get_content_lock_deletion_status_for_authenticated_creator, register_guarded_resource_empty_tail_for_authenticated_creator, register_guarded_resource_for_authenticated_creator, set_lock_service_pointer_for_authenticated_creator, @@ -65,6 +66,14 @@ pub fn router(state: AppState) -> Router { "/creator/content-locks", post(create_content_lock_for_authenticated_creator), ) + .route( + "/creator/content-locks/{lock_id}", + delete(delete_content_lock_for_authenticated_creator), + ) + .route( + "/creator/content-locks/{lock_id}/deletion", + get(get_content_lock_deletion_status_for_authenticated_creator), + ) .route( "/creator/lock-service-config", post(set_lock_service_pointer_for_authenticated_creator), diff --git a/locks-server/src/api/routes/tests.rs b/locks-server/src/api/routes/tests.rs index c9e5877..fd3668c 100644 --- a/locks-server/src/api/routes/tests.rs +++ b/locks-server/src/api/routes/tests.rs @@ -10,7 +10,8 @@ use axum::body::{Body, to_bytes}; use axum::extract::ConnectInfo; use axum::http::{HeaderMap, Request, StatusCode, header}; use locks_core::ids::{ - BundleId, CreatorPubky, GuardedResourceHash, LockServerPubky, PubkyLockResource, + BundleId, ContentLockPath, CreatorPubky, GuardedResourceHash, LockId, LockServerPubky, + PubkyLockResource, }; use locks_core::lock_policy::{ AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, Criterion, GuardedResource, LockLogic, @@ -19,16 +20,17 @@ use locks_core::lock_policy::{ use locks_core::verification::{Proof, SUBMITTED_PROOF_BUNDLE_VERSION, SubmittedProofBundle}; use locks_service::application::errors::ApplicationError; use locks_service::application::models::{ - CreatorAuthorityAuthKind, CreatorAuthorityRecord, CreatorAuthoritySecret, - CreatorConnectAuthorizationUrl, CreatorConnectFlowId, FrontendSessionRecord, - FrontendSessionToken, GuardedResourceRecord, LegacyCreatorConnectFlowApproval, - PendingCreatorConnectFlowRecord, + ContentLockDeletionFailureCode, ContentLockDeletionState, CreatorAuthorityAuthKind, + CreatorAuthorityRecord, CreatorAuthoritySecret, CreatorConnectAuthorizationUrl, + CreatorConnectFlowId, FrontendSessionRecord, FrontendSessionToken, GuardedResourceRecord, + LegacyCreatorConnectFlowApproval, PendingCreatorConnectFlowRecord, }; use locks_service::application::ports::{Clock, LegacyCreatorConnectFlowClient}; use serde_json::{Value, json}; use sqlx::postgres::PgPoolOptions; use time::macros::datetime; use tower::ServiceExt; +use uuid::Uuid; use super::router; use crate::api::auth::parse_frontend_session_token; @@ -155,6 +157,631 @@ async fn cors_preflight_allows_browser_sdk_requests() { ); } +#[tokio::test] +async fn creator_content_lock_delete_requires_frontend_session() { + let lock_id = content_lock(true).lock_id().unwrap(); + let response = router(test_state()) + .oneshot(empty_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!( + response_json(response).await["error"]["code"], + "frontend_session_unavailable" + ); +} + +#[tokio::test] +async fn creator_content_lock_delete_rejects_ambiguous_or_unknown_modes() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let lock_id = content_lock(true).lock_id().unwrap(); + + for query in [ + "force=true&graceful=true", + "force=maybe", + "force=false", + "graceful=false", + "force=false&graceful=false", + "unknown=true", + "force=true&force=false", + ] { + let response = router(state.clone()) + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}?{query}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{query}"); + assert_eq!( + response_json(response).await["error"]["code"], + "invalid_request", + "{query}" + ); + } +} + +#[tokio::test] +async fn authenticated_other_creator_cannot_mutate_or_discover_target_deletion() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + seed_frontend_session(&state, "other-session", other_creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + seed_content_lock(&state, content_lock).await; + let app = router(state.clone()); + + let other_delete = app + .clone() + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "other-session", + )) + .await + .unwrap(); + assert_eq!(other_delete.status(), StatusCode::OK); + assert_eq!( + response_json(other_delete).await, + json!({ "lock_id": lock_id, "status": "completed" }) + ); + assert!( + state + .content_lock_deletions() + .get_job(&creator(), &lock_id) + .await + .unwrap() + .is_none() + ); + assert!( + state + .content_locks() + .get_content_lock(&creator(), &ContentLockPath::from_lock_id(lock_id.clone())) + .await + .unwrap() + .is_some() + ); + + let other_status = app + .oneshot(authenticated_json_request( + "GET", + &format!("/creator/content-locks/{lock_id}/deletion"), + json!(null), + "other-session", + )) + .await + .unwrap(); + assert_error_response( + other_status, + StatusCode::NOT_FOUND, + "content_lock_deletion_not_found", + ) + .await; +} + +#[tokio::test] +async fn graceful_delete_of_absent_lock_returns_completed_postcondition() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let lock_id = content_lock(true).lock_id().unwrap(); + + let response = router(state) + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response_json(response).await, + json!({ "lock_id": lock_id, "status": "completed" }) + ); +} + +#[tokio::test] +async fn graceful_delete_of_existing_lock_queues_one_frozen_job_and_replays() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + seed_content_lock(&state, content_lock.clone()).await; + let app = router(state.clone()); + + for _ in 0..2 { + let response = app + .clone() + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::ACCEPTED); + assert_eq!( + response_json(response).await, + json!({ "lock_id": lock_id, "status": "queued" }) + ); + } + + let job = state + .content_lock_deletions() + .get_job(&creator(), &lock_id) + .await + .unwrap() + .unwrap(); + assert_eq!(job.frozen_content_lock, content_lock); + assert_eq!(job.state, ContentLockDeletionState::Queued); +} + +#[tokio::test] +async fn force_delete_removes_resources_and_lock_records_receipt_and_replays() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + let path = content_lock.content_lock_path().unwrap(); + let resource = content_lock.primary_resource.clone().unwrap(); + seed_content_lock(&state, content_lock.clone()).await; + seed_guarded_resource(&state, &content_lock, b"guarded".to_vec()).await; + let app = router(state.clone()); + + for _ in 0..2 { + let response = app + .clone() + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}?force=true"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response_json(response).await, + json!({ + "lock_id": lock_id, + "lock_deleted": true, + "failed_resource_paths": [] + }) + ); + } + + assert!( + state + .content_locks() + .get_content_lock(&creator(), &path) + .await + .unwrap() + .is_none() + ); + assert!( + state + .guarded_resources() + .get_current_guarded_resource(&creator(), &resource.path) + .await + .unwrap() + .is_none() + ); + assert!( + state + .content_lock_deletions() + .has_force_receipt(&creator(), &lock_id) + .await + .unwrap() + ); + + let status = app + .oneshot(authenticated_json_request( + "GET", + &format!("/creator/content-locks/{lock_id}/deletion"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(status.status(), StatusCode::OK); + assert_eq!( + response_json(status).await, + json!({ "lock_id": lock_id, "status": "completed" }) + ); +} + +#[tokio::test] +async fn force_during_publication_intent_returns_redacted_conflict_without_receipt() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + let publication_token = Uuid::new_v4(); + state + .content_lock_deletions() + .begin_publication(&creator(), &lock_id, publication_token) + .await + .unwrap(); + + for query in ["?force=true", ""] { + let response = router(state.clone()) + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}{query}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::CONFLICT); + assert_eq!( + response_json(response).await, + json!({ + "error": { + "code": "content_lock_path_conflict", + "message": "content lock publication is in progress" + } + }) + ); + } + assert!( + !state + .content_lock_deletions() + .has_force_receipt(&creator(), &lock_id) + .await + .unwrap() + ); + assert!( + state + .content_lock_deletions() + .abandon_publication(&creator(), &lock_id, publication_token) + .await + .unwrap() + ); +} + +#[tokio::test] +async fn force_escalation_marks_existing_job_for_worker_owned_async_force() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + seed_content_lock(&state, content_lock.clone()).await; + seed_guarded_resource(&state, &content_lock, b"guarded".to_vec()).await; + let app = router(state.clone()); + + let graceful = app + .clone() + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(graceful.status(), StatusCode::ACCEPTED); + + let forced = app + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}?force=true"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(forced.status(), StatusCode::ACCEPTED); + assert_eq!( + response_json(forced).await, + json!({ "lock_id": lock_id, "status": "queued" }) + ); + + let job = state + .content_lock_deletions() + .get_job(&creator(), &lock_id) + .await + .unwrap() + .unwrap(); + assert_eq!(job.frozen_content_lock, content_lock); + assert!(job.force_requested_at.is_some()); + assert!( + !state + .content_lock_deletions() + .has_force_receipt(&creator(), &lock_id) + .await + .unwrap() + ); + assert!( + state + .content_locks() + .get_content_lock(&creator(), &ContentLockPath::from_lock_id(lock_id)) + .await + .unwrap() + .is_some() + ); +} + +#[tokio::test] +async fn force_after_terminal_graceful_failure_runs_synchronously_from_frozen_manifest() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + let path = content_lock.content_lock_path().unwrap(); + let resource = content_lock.primary_resource.clone().unwrap(); + seed_content_lock(&state, content_lock.clone()).await; + seed_guarded_resource(&state, &content_lock, b"guarded".to_vec()).await; + let app = router(state.clone()); + + let graceful = app + .clone() + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(graceful.status(), StatusCode::ACCEPTED); + + let now = state.clock().now(); + let claimed = state + .content_lock_deletions() + .claim_next("test-worker", now, now + time::Duration::minutes(1)) + .await + .unwrap() + .unwrap(); + state + .content_lock_deletions() + .finish( + claimed.job.job_id, + "test-worker", + claimed.claim_token, + now, + Some(ContentLockDeletionFailureCode::TombstoneMissing), + ) + .await + .unwrap() + .unwrap(); + + let forced = app + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}?force=true"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(forced.status(), StatusCode::OK); + assert_eq!( + response_json(forced).await, + json!({ + "lock_id": lock_id, + "lock_deleted": true, + "failed_resource_paths": [] + }) + ); + assert!( + state + .content_locks() + .get_content_lock(&creator(), &path) + .await + .unwrap() + .is_none() + ); + assert!( + state + .guarded_resources() + .get_current_guarded_resource(&creator(), &resource.path) + .await + .unwrap() + .is_none() + ); + assert!( + state + .content_lock_deletions() + .has_force_receipt(&creator(), &lock_id) + .await + .unwrap() + ); +} + +#[tokio::test] +async fn graceful_replay_of_failed_job_requeues_same_frozen_manifest() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + seed_content_lock(&state, content_lock.clone()).await; + let app = router(state.clone()); + + let started = app + .clone() + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(started.status(), StatusCode::ACCEPTED); + let original = state + .content_lock_deletions() + .get_job(&creator(), &lock_id) + .await + .unwrap() + .unwrap(); + let now = state.clock().now(); + let claimed = state + .content_lock_deletions() + .claim_next("test-worker", now, now + time::Duration::minutes(1)) + .await + .unwrap() + .unwrap(); + state + .content_lock_deletions() + .finish( + claimed.job.job_id, + "test-worker", + claimed.claim_token, + now, + Some(ContentLockDeletionFailureCode::TombstoneMissing), + ) + .await + .unwrap() + .unwrap(); + + let replay = app + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(replay.status(), StatusCode::ACCEPTED); + assert_eq!( + response_json(replay).await, + json!({ "lock_id": lock_id, "status": "queued" }) + ); + let resumed = state + .content_lock_deletions() + .get_job(&creator(), &lock_id) + .await + .unwrap() + .unwrap(); + assert_eq!(resumed.job_id, original.job_id); + assert_eq!(resumed.frozen_content_lock, content_lock); + assert_eq!(resumed.state, ContentLockDeletionState::Queued); + assert_eq!(resumed.failure_code, None); +} + +#[tokio::test] +async fn graceful_and_force_reject_content_lock_stored_under_wrong_canonical_path() { + for force_query in ["", "?force=true"] { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let requested = content_lock(true); + let requested_lock_id = requested.lock_id().unwrap(); + let wrong_lock = content_lock(false); + let wrong_resource = wrong_lock.primary_resource.clone().unwrap(); + seed_guarded_resource(&state, &wrong_lock, b"guarded".to_vec()).await; + state + .content_locks() + .upsert_content_lock( + creator(), + ContentLockPath::from_lock_id(requested_lock_id.clone()), + wrong_lock, + ) + .await + .unwrap(); + + let response = router(state.clone()) + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{requested_lock_id}{force_query}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert!( + state + .content_lock_deletions() + .get_job(&creator(), &requested_lock_id) + .await + .unwrap() + .is_none() + ); + assert!( + !state + .content_lock_deletions() + .has_force_receipt(&creator(), &requested_lock_id) + .await + .unwrap() + ); + assert!( + state + .guarded_resources() + .get_current_guarded_resource(&creator(), &wrong_resource.path) + .await + .unwrap() + .is_some() + ); + } +} + +#[tokio::test] +async fn creator_content_lock_deletion_status_reports_job_and_absence() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + seed_content_lock(&state, content_lock).await; + let app = router(state.clone()); + + let started = app + .clone() + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(started.status(), StatusCode::ACCEPTED); + + let response = app + .oneshot(authenticated_json_request( + "GET", + &format!("/creator/content-locks/{lock_id}/deletion"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response_json(response).await, + json!({ "lock_id": lock_id, "status": "queued" }) + ); + + let missing_lock_id = LockId::from_hash(locks_core::ids::LockHash::from_bytes([99; 32])); + let missing = router(state) + .oneshot(authenticated_json_request( + "GET", + &format!("/creator/content-locks/{missing_lock_id}/deletion"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(missing.status(), StatusCode::NOT_FOUND); + assert_eq!( + response_json(missing).await["error"]["code"], + "content_lock_deletion_not_found" + ); +} + #[tokio::test] async fn readyz_returns_ready_for_ephemeral_runtime_without_secrets() { let response = router(test_state()) diff --git a/locks-server/src/app_state/mod.rs b/locks-server/src/app_state/mod.rs index c9e02cf..dd87d52 100644 --- a/locks-server/src/app_state/mod.rs +++ b/locks-server/src/app_state/mod.rs @@ -17,26 +17,28 @@ use locks_service::{ errors::ApplicationError, models::AccessCredentialPolicy, ports::{ - AccessCredentialStore, Clock, ContentLockOwnershipRepository, ContentLockRepository, - CreatorAuthorityManager, CreatorAuthorityStore, CreatorConnectFlowStore, - EntitlementRepository, FrontendSessionCodeStore, FrontendSessionStore, - GuardedResourceRepository, LegacyCreatorConnectFlowClient, - LockServicePointerRepository, VerificationTaskClaimer, VerificationTaskRepository, + AccessCredentialStore, Clock, ContentLockDeletionRepository, + ContentLockOwnershipRepository, ContentLockRepository, CreatorAuthorityManager, + CreatorAuthorityStore, CreatorConnectFlowStore, EntitlementRepository, + FrontendSessionCodeStore, FrontendSessionStore, GuardedResourceRepository, + LegacyCreatorConnectFlowClient, LockServicePointerRepository, VerificationTaskClaimer, + VerificationTaskRepository, }, }, infrastructure::{ memory::{ access_credentials::InMemoryAccessCredentialStore, + content_lock_deletions::InMemoryContentLockDeletionRepository, content_lock_ownership::InMemoryContentLockOwnershipRepository, verification_task_claims::InMemoryVerificationTaskClaimer, verification_tasks::InMemoryVerificationTaskRepository, }, postgres::{ CreatorAuthoritySecretCipher, PostgresAccessCredentialStore, - PostgresContentLockOwnershipRepository, PostgresCreatorAuthorityStore, - PostgresCreatorConnectFlowStore, PostgresFrontendSessionCodeStore, - PostgresFrontendSessionStore, PostgresVerificationTaskClaimer, - PostgresVerificationTaskRepository, + PostgresContentLockDeletionRepository, PostgresContentLockOwnershipRepository, + PostgresCreatorAuthorityStore, PostgresCreatorConnectFlowStore, + PostgresFrontendSessionCodeStore, PostgresFrontendSessionStore, + PostgresVerificationTaskClaimer, PostgresVerificationTaskRepository, }, pubky::{ AuthorizingPubkyHomeserverStorageClient, LegacyCookieCreatorAuthorityManager, @@ -150,6 +152,7 @@ pub struct AppState { guarded_resources: Arc, lock_service_pointers: Arc, content_lock_ownership: Arc, + content_lock_deletions: Arc, verification_tasks: Arc, verification_task_claimer: Arc, entitlements: Arc, @@ -514,6 +517,11 @@ impl AppState { )) }) }); + let content_lock_deletions: Arc = + match postgres_pool.as_ref() { + Some(pool) => Arc::new(PostgresContentLockDeletionRepository::new(pool.clone())), + None => Arc::new(InMemoryContentLockDeletionRepository::new()), + }; Self { config, @@ -523,6 +531,7 @@ impl AppState { guarded_resources: creator_repositories.guarded_resources, lock_service_pointers: creator_repositories.lock_service_pointers, content_lock_ownership: private_runtime.content_lock_ownership, + content_lock_deletions, verification_tasks: private_runtime.verification_tasks, verification_task_claimer: private_runtime.verification_task_claimer, entitlements: creator_repositories.entitlements, @@ -572,6 +581,10 @@ impl AppState { &self.content_lock_ownership } + pub fn content_lock_deletions(&self) -> &Arc { + &self.content_lock_deletions + } + pub fn lock_service_pointers(&self) -> &Arc { &self.lock_service_pointers } diff --git a/locks-service/migrations/0013_content_lock_publication_intents.sql b/locks-service/migrations/0013_content_lock_publication_intents.sql new file mode 100644 index 0000000..7d9712b --- /dev/null +++ b/locks-service/migrations/0013_content_lock_publication_intents.sql @@ -0,0 +1,8 @@ +CREATE TABLE content_lock_publication_intents ( + creator TEXT NOT NULL, + lock_id TEXT NOT NULL, + publication_token UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT content_lock_publication_intents_pkey PRIMARY KEY (creator, lock_id), + CONSTRAINT content_lock_publication_intents_token_unique UNIQUE (publication_token) +); \ No newline at end of file diff --git a/locks-service/src/application/models/content_lock_deletion.rs b/locks-service/src/application/models/content_lock_deletion.rs index 9f61ebc..56b24da 100644 --- a/locks-service/src/application/models/content_lock_deletion.rs +++ b/locks-service/src/application/models/content_lock_deletion.rs @@ -109,6 +109,17 @@ pub struct ClaimedContentLockDeletionJob { pub claim_token: Uuid, } +/// Durable decision made while serializing force deletion against graceful lifecycle state. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PrepareForceDeletionResult { + /// A pre-existing publication intent must reconcile before force can begin. + PublicationInProgress, + /// An active graceful job was durably marked for asynchronous force processing. + Active(ContentLockDeletionJob), + /// A permanent force receipt was established. A terminal frozen job is returned when present. + Synchronous(Option), +} + impl ContentLockDeletionJob { /// Creates a queued deletion job from a canonical frozen content lock. pub fn new( diff --git a/locks-service/src/application/ports/content_lock_deletion.rs b/locks-service/src/application/ports/content_lock_deletion.rs index f7770d5..972062e 100644 --- a/locks-service/src/application/ports/content_lock_deletion.rs +++ b/locks-service/src/application/ports/content_lock_deletion.rs @@ -7,13 +7,44 @@ use crate::application::{ errors::ApplicationError, models::{ ClaimedContentLockDeletionJob, ContentLockDeletionFailureCode, ContentLockDeletionJob, - ContentLockDeletionPhase, + ContentLockDeletionPhase, PrepareForceDeletionResult, }, }; /// Durable repository and fenced worker lease boundary for content-lock deletion jobs. #[async_trait] pub trait ContentLockDeletionRepository: Send + Sync { + /// Reserves canonical publication under the same per-lock fence used by force deletion. + async fn begin_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result<(), ApplicationError>; + + /// Finalizes the exact publication reservation after external publication and ownership commit. + async fn finish_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result; + + /// Removes the exact unfinalized reservation after a safely compensated publication failure. + async fn abandon_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result; + + /// Checks publication admission under the canonical per-lock fence. + async fn publication_in_progress( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result; + async fn insert_job(&self, job: ContentLockDeletionJob) -> Result<(), ApplicationError>; async fn get_job( @@ -57,21 +88,21 @@ pub trait ContentLockDeletionRepository: Send + Sync { failure_code: Option, ) -> Result, ApplicationError>; - /// Permanently records force escalation. Returns true only on the first request. - async fn request_force( + /// Requeues a failed job with its frozen manifest unless a permanent force receipt exists. + async fn resume_failed_job( &self, creator: &CreatorPubky, lock_id: &LockId, - requested_at: OffsetDateTime, - ) -> Result; + resumed_at: OffsetDateTime, + ) -> Result, ApplicationError>; - /// Idempotently records the permanent minimal force-deletion receipt. - async fn record_force_receipt( + /// Atomically escalates an active job or establishes the permanent synchronous-force receipt. + async fn prepare_force_deletion( &self, creator: &CreatorPubky, lock_id: &LockId, forced_at: OffsetDateTime, - ) -> Result<(), ApplicationError>; + ) -> Result; async fn has_force_receipt( &self, diff --git a/locks-service/src/application/ports/lock_policy.rs b/locks-service/src/application/ports/lock_policy.rs index 03a2438..f8f1f5a 100644 --- a/locks-service/src/application/ports/lock_policy.rs +++ b/locks-service/src/application/ports/lock_policy.rs @@ -25,6 +25,14 @@ pub trait ContentLockRepository: Send + Sync { creator: &CreatorPubky, content_lock_path: &ContentLockPath, ) -> Result, ApplicationError>; + + /// Deletes the public content lock at the canonical creator-owned path. + /// Returns true when a record existed and false when already absent. + async fn delete_content_lock( + &self, + creator: &CreatorPubky, + content_lock_path: &ContentLockPath, + ) -> Result; } /// Repository for creator-owned Lock Service Pointer config objects. diff --git a/locks-service/src/application/use_cases/complete_verification_task.rs b/locks-service/src/application/use_cases/complete_verification_task.rs index fed03ae..7d2c4a7 100644 --- a/locks-service/src/application/use_cases/complete_verification_task.rs +++ b/locks-service/src/application/use_cases/complete_verification_task.rs @@ -1236,6 +1236,14 @@ mod tests { ) -> Result, ApplicationError> { Ok(self.content_lock.clone()) } + + async fn delete_content_lock( + &self, + _creator: &CreatorPubky, + _content_lock_path: &ContentLockPath, + ) -> Result { + unreachable!("completion must not delete content locks") + } } #[derive(Default)] diff --git a/locks-service/src/application/use_cases/create_content_lock.rs b/locks-service/src/application/use_cases/create_content_lock.rs index 4f78029..36526ee 100644 --- a/locks-service/src/application/use_cases/create_content_lock.rs +++ b/locks-service/src/application/use_cases/create_content_lock.rs @@ -5,10 +5,12 @@ use locks_core::lock_policy::{ AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, Criterion, GuardedResource, LockLogic, LockServerConfig, SecondaryGuardedResource, }; +use uuid::Uuid; use crate::application::errors::ApplicationError; use crate::application::ports::{ - Clock, ContentLockOwnershipRepository, ContentLockRepository, GuardedResourceRepository, + Clock, ContentLockDeletionRepository, ContentLockOwnershipRepository, ContentLockRepository, + GuardedResourceRepository, }; /// Request to create a local content lock for an already-registered guarded resource. @@ -44,6 +46,7 @@ pub struct CreatedContentLock { /// Creates local content locks after verifying current guarded resource metadata. pub struct CreateContentLockUseCase<'a> { content_locks: &'a dyn ContentLockRepository, + content_lock_deletions: &'a dyn ContentLockDeletionRepository, content_lock_ownership: &'a dyn ContentLockOwnershipRepository, guarded_resources: &'a dyn GuardedResourceRepository, clock: &'a dyn Clock, @@ -53,12 +56,14 @@ impl<'a> CreateContentLockUseCase<'a> { /// Creates a content-lock use case from its application ports. pub fn new( content_locks: &'a dyn ContentLockRepository, + content_lock_deletions: &'a dyn ContentLockDeletionRepository, content_lock_ownership: &'a dyn ContentLockOwnershipRepository, guarded_resources: &'a dyn GuardedResourceRepository, clock: &'a dyn Clock, ) -> Self { Self { content_locks, + content_lock_deletions, content_lock_ownership, guarded_resources, clock, @@ -129,6 +134,19 @@ impl<'a> CreateContentLockUseCase<'a> { .reserve_paths(&request.creator, &guarded_paths, &lock_id) .await?; + let publication_token = Uuid::new_v4(); + if let Err(error) = self + .content_lock_deletions + .begin_publication(&request.creator, &lock_id, publication_token) + .await + { + let _ = self + .content_lock_ownership + .compensate_reserved_paths(&request.creator, &guarded_paths, &lock_id) + .await; + return Err(error); + } + if let Err(error) = self .content_locks .upsert_content_lock( @@ -138,16 +156,54 @@ impl<'a> CreateContentLockUseCase<'a> { ) .await { - let _ = self - .content_lock_ownership - .compensate_reserved_paths(&request.creator, &guarded_paths, &lock_id) - .await; + match self + .content_locks + .get_content_lock(&request.creator, &content_lock_path) + .await + { + Ok(Some(published)) if published == content_lock => { + if self + .content_lock_ownership + .mark_paths_published(&request.creator, &guarded_paths, &lock_id) + .await + .is_ok() + { + let _ = self + .content_lock_deletions + .finish_publication(&request.creator, &lock_id, publication_token) + .await; + } + } + Ok(None) => { + if self + .content_lock_ownership + .compensate_reserved_paths(&request.creator, &guarded_paths, &lock_id) + .await + .is_ok() + { + let _ = self + .content_lock_deletions + .abandon_publication(&request.creator, &lock_id, publication_token) + .await; + } + } + Ok(Some(_)) | Err(_) => {} + } return Err(error); } self.content_lock_ownership .mark_paths_published(&request.creator, &guarded_paths, &lock_id) .await?; + if !self + .content_lock_deletions + .finish_publication(&request.creator, &lock_id, publication_token) + .await? + { + return Err(ApplicationError::Storage { + message: "content lock publication intent was lost".to_owned(), + }); + } Ok(CreatedContentLock { lock_id, @@ -203,10 +259,14 @@ mod tests { use locks_core::lock_policy::VerifierType; use super::*; - use crate::application::models::GuardedResourceRecord; + use crate::application::models::{ + ContentLockOwnershipStatus, GuardedResourceRecord, PrepareForceDeletionResult, + }; use crate::application::ports::{ - Clock, ContentLockOwnershipRepository, ContentLockRepository, GuardedResourceRepository, + Clock, ContentLockDeletionRepository, ContentLockOwnershipRepository, + ContentLockRepository, GuardedResourceRepository, }; + use crate::infrastructure::memory::content_lock_deletions::InMemoryContentLockDeletionRepository; use crate::infrastructure::memory::content_lock_ownership::InMemoryContentLockOwnershipRepository; use crate::infrastructure::memory::content_locks::InMemoryContentLockRepository; use crate::infrastructure::memory::guarded_resources::InMemoryGuardedResourceRepository; @@ -258,6 +318,42 @@ mod tests { assert_eq!(ownership.status.as_str(), "published"); } + #[tokio::test] + async fn permanent_force_receipt_blocks_canonical_lock_republication() { + let fixture = Fixture::seeded().await; + let request = content_lock_request(registered_guarded_resource()); + let content_lock = ContentLock { + version: CONTENT_LOCK_VERSION, + creator: request.creator.clone(), + primary_resource: request.primary_resource.clone(), + secondary_resources: request.secondary_resources.clone(), + criteria: request.criteria.clone(), + lock_logic: request.lock_logic.clone(), + access_policy: request.access_policy.clone(), + lock_server: request.lock_server.clone(), + created_at: fixture.clock.now(), + }; + let lock_id = content_lock.lock_id().unwrap(); + fixture + .content_lock_deletions + .prepare_force_deletion(&request.creator, &lock_id, fixture.clock.now()) + .await + .unwrap(); + + let result = fixture.use_case().execute(request).await; + + assert_eq!(result, Err(ApplicationError::ContentLockDeletionInProgress)); + assert_eq!(fixture.content_locks_len().await, 0); + assert_eq!( + fixture + .content_lock_ownership + .get_path_ownership(&creator(), "/priv/locks.app/content/hello.txt") + .await + .unwrap(), + None + ); + } + #[tokio::test] async fn create_content_lock_rejects_missing_guarded_resource() { let fixture = Fixture::empty(); @@ -422,6 +518,7 @@ mod tests { let fixture = Fixture::seeded().await; let use_case = CreateContentLockUseCase::new( &FailingContentLockRepository, + &fixture.content_lock_deletions, &fixture.content_lock_ownership, &fixture.guarded_resources, &fixture.clock, @@ -445,6 +542,272 @@ mod tests { ); } + #[tokio::test] + async fn ambiguous_publication_error_reconciles_committed_lock_without_releasing_ownership() { + let fixture = Fixture::seeded().await; + let content_locks = AmbiguousContentLockRepository::default(); + let use_case = CreateContentLockUseCase::new( + &content_locks, + &fixture.content_lock_deletions, + &fixture.content_lock_ownership, + &fixture.guarded_resources, + &fixture.clock, + ); + let request = content_lock_request(registered_guarded_resource()); + let expected = ContentLock { + version: CONTENT_LOCK_VERSION, + creator: request.creator.clone(), + primary_resource: request.primary_resource.clone(), + secondary_resources: request.secondary_resources.clone(), + criteria: request.criteria.clone(), + lock_logic: request.lock_logic.clone(), + access_policy: request.access_policy.clone(), + lock_server: request.lock_server.clone(), + created_at: fixture.clock.now(), + }; + let lock_id = expected.lock_id().unwrap(); + let path = expected.content_lock_path().unwrap(); + + let result = use_case.execute(request).await; + + assert!(matches!( + result, + Err(ApplicationError::Storage { ref message }) if message == "publication response lost" + )); + assert_eq!( + content_locks + .get_content_lock(&creator(), &path) + .await + .unwrap(), + Some(expected) + ); + assert_eq!( + fixture + .content_lock_ownership + .get_path_ownership(&creator(), "/priv/locks.app/content/hello.txt") + .await + .unwrap() + .unwrap() + .status, + ContentLockOwnershipStatus::Published + ); + assert!( + !fixture + .content_lock_deletions + .publication_in_progress(&creator(), &lock_id) + .await + .unwrap() + ); + } + + #[tokio::test] + async fn unreconciled_publication_error_retains_reserved_ownership_and_deletion_fence() { + let fixture = Fixture::seeded().await; + let content_locks = UnreconciledContentLockRepository; + let use_case = CreateContentLockUseCase::new( + &content_locks, + &fixture.content_lock_deletions, + &fixture.content_lock_ownership, + &fixture.guarded_resources, + &fixture.clock, + ); + let request = content_lock_request(registered_guarded_resource()); + let lock_id = ContentLock { + version: CONTENT_LOCK_VERSION, + creator: request.creator.clone(), + primary_resource: request.primary_resource.clone(), + secondary_resources: request.secondary_resources.clone(), + criteria: request.criteria.clone(), + lock_logic: request.lock_logic.clone(), + access_policy: request.access_policy.clone(), + lock_server: request.lock_server.clone(), + created_at: fixture.clock.now(), + } + .lock_id() + .unwrap(); + + let result = use_case.execute(request).await; + + assert!(matches!( + result, + Err(ApplicationError::Storage { ref message }) if message == "publication response lost" + )); + assert_eq!( + fixture + .content_lock_ownership + .get_path_ownership(&creator(), "/priv/locks.app/content/hello.txt") + .await + .unwrap() + .unwrap() + .status, + ContentLockOwnershipStatus::Reserved + ); + assert!( + fixture + .content_lock_deletions + .publication_in_progress(&creator(), &lock_id) + .await + .unwrap() + ); + assert_eq!( + fixture + .content_lock_deletions + .prepare_force_deletion(&creator(), &lock_id, fixture.clock.now()) + .await + .unwrap(), + PrepareForceDeletionResult::PublicationInProgress + ); + } + + #[tokio::test] + async fn publication_intent_fences_force_during_external_upsert() { + let fixture = Fixture::seeded().await; + let probe = PublicationRaceProbe { + deletions: &fixture.content_lock_deletions, + now: fixture.clock.now(), + }; + let use_case = CreateContentLockUseCase::new( + &probe, + &fixture.content_lock_deletions, + &fixture.content_lock_ownership, + &fixture.guarded_resources, + &fixture.clock, + ); + + let created = use_case + .execute(content_lock_request(registered_guarded_resource())) + .await + .unwrap(); + + assert_eq!( + fixture + .content_lock_deletions + .prepare_force_deletion(&creator(), &created.lock_id, fixture.clock.now()) + .await + .unwrap(), + PrepareForceDeletionResult::Synchronous(None) + ); + } + + struct PublicationRaceProbe<'a> { + deletions: &'a InMemoryContentLockDeletionRepository, + now: OffsetDateTime, + } + + #[async_trait] + impl ContentLockRepository for PublicationRaceProbe<'_> { + async fn upsert_content_lock( + &self, + creator: CreatorPubky, + _path: ContentLockPath, + content_lock: ContentLock, + ) -> Result<(), ApplicationError> { + let lock_id = content_lock.lock_id().unwrap(); + assert_eq!( + self.deletions + .prepare_force_deletion(&creator, &lock_id, self.now) + .await?, + PrepareForceDeletionResult::PublicationInProgress + ); + assert!(!self.deletions.has_force_receipt(&creator, &lock_id).await?); + Ok(()) + } + + async fn get_content_lock( + &self, + _creator: &CreatorPubky, + _path: &ContentLockPath, + ) -> Result, ApplicationError> { + Ok(None) + } + + async fn delete_content_lock( + &self, + _creator: &CreatorPubky, + _path: &ContentLockPath, + ) -> Result { + unreachable!("creation must not delete content locks") + } + } + + #[derive(Default)] + struct AmbiguousContentLockRepository { + published: tokio::sync::RwLock>, + } + + #[async_trait] + impl ContentLockRepository for AmbiguousContentLockRepository { + async fn upsert_content_lock( + &self, + creator: CreatorPubky, + path: ContentLockPath, + content_lock: ContentLock, + ) -> Result<(), ApplicationError> { + *self.published.write().await = Some((creator, path, content_lock)); + Err(ApplicationError::Storage { + message: "publication response lost".to_owned(), + }) + } + + async fn get_content_lock( + &self, + creator: &CreatorPubky, + path: &ContentLockPath, + ) -> Result, ApplicationError> { + Ok(self + .published + .read() + .await + .as_ref() + .filter(|(stored_creator, stored_path, _)| { + stored_creator == creator && stored_path == path + }) + .map(|(_, _, content_lock)| content_lock.clone())) + } + + async fn delete_content_lock( + &self, + _creator: &CreatorPubky, + _path: &ContentLockPath, + ) -> Result { + unreachable!("creation must not delete content locks") + } + } + + struct UnreconciledContentLockRepository; + + #[async_trait] + impl ContentLockRepository for UnreconciledContentLockRepository { + async fn upsert_content_lock( + &self, + _creator: CreatorPubky, + _path: ContentLockPath, + _content_lock: ContentLock, + ) -> Result<(), ApplicationError> { + Err(ApplicationError::Storage { + message: "publication response lost".to_owned(), + }) + } + + async fn get_content_lock( + &self, + _creator: &CreatorPubky, + _path: &ContentLockPath, + ) -> Result, ApplicationError> { + Err(ApplicationError::Storage { + message: "publication reconciliation failed".to_owned(), + }) + } + + async fn delete_content_lock( + &self, + _creator: &CreatorPubky, + _path: &ContentLockPath, + ) -> Result { + unreachable!("creation must not delete content locks") + } + } + struct FailingContentLockRepository; #[async_trait] @@ -467,10 +830,19 @@ mod tests { ) -> Result, ApplicationError> { Ok(None) } + + async fn delete_content_lock( + &self, + _creator: &CreatorPubky, + _path: &ContentLockPath, + ) -> Result { + unreachable!("creation must not delete content locks") + } } struct Fixture { content_locks: InMemoryContentLockRepository, + content_lock_deletions: InMemoryContentLockDeletionRepository, content_lock_ownership: InMemoryContentLockOwnershipRepository, guarded_resources: InMemoryGuardedResourceRepository, clock: FixedClock, @@ -480,6 +852,7 @@ mod tests { fn empty() -> Self { Self { content_locks: InMemoryContentLockRepository::new(), + content_lock_deletions: InMemoryContentLockDeletionRepository::new(), content_lock_ownership: InMemoryContentLockOwnershipRepository::new(), guarded_resources: InMemoryGuardedResourceRepository::new(), clock: FixedClock(datetime!(2026-06-03 12:00:00 UTC)), @@ -507,6 +880,7 @@ mod tests { fn use_case(&self) -> CreateContentLockUseCase<'_> { CreateContentLockUseCase::new( &self.content_locks, + &self.content_lock_deletions, &self.content_lock_ownership, &self.guarded_resources, &self.clock, diff --git a/locks-service/src/application/use_cases/credential_flow_tests.rs b/locks-service/src/application/use_cases/credential_flow_tests.rs index eedda7b..99d3bc0 100644 --- a/locks-service/src/application/use_cases/credential_flow_tests.rs +++ b/locks-service/src/application/use_cases/credential_flow_tests.rs @@ -502,6 +502,14 @@ impl ContentLockRepository for FakeContentLocks { ) -> Result, ApplicationError> { Ok(self.content_lock.lock().unwrap().clone()) } + + async fn delete_content_lock( + &self, + _creator: &CreatorPubky, + _content_lock_path: &ContentLockPath, + ) -> Result { + unreachable!("credential flow must not delete content locks") + } } struct FakeEntitlements { diff --git a/locks-service/src/application/use_cases/validate_paykit_payment_submission.rs b/locks-service/src/application/use_cases/validate_paykit_payment_submission.rs index 73e2699..a2cfbc1 100644 --- a/locks-service/src/application/use_cases/validate_paykit_payment_submission.rs +++ b/locks-service/src/application/use_cases/validate_paykit_payment_submission.rs @@ -245,6 +245,14 @@ mod tests { ) -> Result, ApplicationError> { Ok(self.0.clone()) } + + async fn delete_content_lock( + &self, + _creator: &CreatorPubky, + _content_lock_path: &ContentLockPath, + ) -> Result { + unreachable!("validation must not delete content locks") + } } fn content_lock() -> ContentLock { diff --git a/locks-service/src/infrastructure/memory/content_lock_deletions.rs b/locks-service/src/infrastructure/memory/content_lock_deletions.rs index 238ee31..8efcd24 100644 --- a/locks-service/src/infrastructure/memory/content_lock_deletions.rs +++ b/locks-service/src/infrastructure/memory/content_lock_deletions.rs @@ -10,7 +10,7 @@ use crate::application::{ errors::ApplicationError, models::{ ClaimedContentLockDeletionJob, ContentLockDeletionFailureCode, ContentLockDeletionJob, - ContentLockDeletionPhase, ContentLockDeletionState, + ContentLockDeletionPhase, ContentLockDeletionState, PrepareForceDeletionResult, }, ports::ContentLockDeletionRepository, }; @@ -30,6 +30,7 @@ struct StoredJob { pub struct InMemoryContentLockDeletionRepository { jobs: RwLock>, force_receipts: RwLock>, + publication_intents: RwLock>, } impl InMemoryContentLockDeletionRepository { @@ -40,11 +41,80 @@ impl InMemoryContentLockDeletionRepository { #[async_trait] impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { + async fn begin_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result<(), ApplicationError> { + let key = (creator.clone(), lock_id.clone()); + let mut intents = self.publication_intents.write().await; + let jobs = self.jobs.read().await; + let receipts = self.force_receipts.read().await; + if jobs.contains_key(&key) || receipts.contains(&key) { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + if intents.contains_key(&key) { + return Err(ApplicationError::ContentLockPathConflict { + guarded_path: "content lock publication in progress".to_owned(), + }); + } + intents.insert(key, publication_token); + Ok(()) + } + + async fn finish_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result { + remove_publication_intent( + &self.publication_intents, + creator, + lock_id, + publication_token, + ) + .await + } + + async fn abandon_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result { + remove_publication_intent( + &self.publication_intents, + creator, + lock_id, + publication_token, + ) + .await + } + + async fn publication_in_progress( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result { + Ok(self + .publication_intents + .read() + .await + .contains_key(&(creator.clone(), lock_id.clone()))) + } + async fn insert_job(&self, job: ContentLockDeletionJob) -> Result<(), ApplicationError> { job.validate_frozen_identity()?; job.validate_state(false)?; let key = (job.creator.clone(), job.lock_id.clone()); + let intents = self.publication_intents.read().await; let mut jobs = self.jobs.write().await; + let receipts = self.force_receipts.read().await; + if intents.contains_key(&key) || receipts.contains(&key) { + return Err(ApplicationError::ContentLockDeletionInProgress); + } if jobs.contains_key(&key) || jobs.values().any(|stored| stored.job.job_id == job.job_id) { return Err(ApplicationError::DuplicateRecord { record: "content_lock_deletion_job", @@ -188,34 +258,66 @@ impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { Ok(Some(stored.job.clone())) } - async fn request_force( + async fn resume_failed_job( &self, creator: &CreatorPubky, lock_id: &LockId, - requested_at: OffsetDateTime, - ) -> Result { + _resumed_at: OffsetDateTime, + ) -> Result, ApplicationError> { let mut jobs = self.jobs.write().await; + let receipts = self.force_receipts.read().await; + if receipts.contains(&(creator.clone(), lock_id.clone())) { + return Ok(None); + } let Some(stored) = jobs.get_mut(&(creator.clone(), lock_id.clone())) else { - return Ok(false); + return Ok(None); }; - if stored.job.force_requested_at.is_some() { - return Ok(false); + if stored.job.state == ContentLockDeletionState::Failed { + stored.job.state = ContentLockDeletionState::Queued; + stored.job.attempt_count = 0; + stored.job.next_attempt_at = None; + stored.job.failure_code = None; + clear_claim(stored); } - stored.job.force_requested_at = Some(requested_at); - Ok(true) + Ok(Some(stored.job.clone())) } - async fn record_force_receipt( + async fn prepare_force_deletion( &self, creator: &CreatorPubky, lock_id: &LockId, - _forced_at: OffsetDateTime, - ) -> Result<(), ApplicationError> { - self.force_receipts - .write() - .await - .insert((creator.clone(), lock_id.clone())); - Ok(()) + forced_at: OffsetDateTime, + ) -> Result { + let key = (creator.clone(), lock_id.clone()); + let intents = self.publication_intents.read().await; + if intents.contains_key(&key) { + return Ok(PrepareForceDeletionResult::PublicationInProgress); + } + let mut jobs = self.jobs.write().await; + let mut receipts = self.force_receipts.write().await; + if receipts.contains(&key) { + return Ok(PrepareForceDeletionResult::Synchronous( + jobs.get(&key).map(|stored| stored.job.clone()), + )); + } + if let Some(stored) = jobs.get_mut(&key) { + if matches!( + stored.job.state, + ContentLockDeletionState::Queued | ContentLockDeletionState::Running + ) { + stored.job.force_requested_at.get_or_insert(forced_at); + stored.job.state = ContentLockDeletionState::Queued; + stored.job.next_attempt_at = None; + clear_claim(stored); + return Ok(PrepareForceDeletionResult::Active(stored.job.clone())); + } + let job = stored.job.clone(); + jobs.remove(&key); + receipts.insert(key); + return Ok(PrepareForceDeletionResult::Synchronous(Some(job))); + } + receipts.insert(key); + Ok(PrepareForceDeletionResult::Synchronous(None)) } async fn has_force_receipt( @@ -231,6 +333,21 @@ impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { } } +async fn remove_publication_intent( + intents: &RwLock>, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, +) -> Result { + let key = (creator.clone(), lock_id.clone()); + let mut intents = intents.write().await; + if intents.get(&key) != Some(&publication_token) { + return Ok(false); + } + intents.remove(&key); + Ok(true) +} + fn is_claimable(stored: &StoredJob, now: OffsetDateTime) -> bool { match stored.job.state { ContentLockDeletionState::Queued => stored diff --git a/locks-service/src/infrastructure/memory/content_locks.rs b/locks-service/src/infrastructure/memory/content_locks.rs index bbe9229..5db6ced 100644 --- a/locks-service/src/infrastructure/memory/content_locks.rs +++ b/locks-service/src/infrastructure/memory/content_locks.rs @@ -51,6 +51,19 @@ impl ContentLockRepository for InMemoryContentLockRepository { .get(&(creator.clone(), content_lock_path.clone())) .cloned()) } + + async fn delete_content_lock( + &self, + creator: &CreatorPubky, + content_lock_path: &ContentLockPath, + ) -> Result { + Ok(self + .records + .write() + .await + .remove(&(creator.clone(), content_lock_path.clone())) + .is_some()) + } } #[cfg(test)] diff --git a/locks-service/src/infrastructure/postgres/content_lock_deletions.rs b/locks-service/src/infrastructure/postgres/content_lock_deletions.rs index 11412ca..1d39261 100644 --- a/locks-service/src/infrastructure/postgres/content_lock_deletions.rs +++ b/locks-service/src/infrastructure/postgres/content_lock_deletions.rs @@ -13,7 +13,7 @@ use crate::application::{ errors::ApplicationError, models::{ ClaimedContentLockDeletionJob, ContentLockDeletionFailureCode, ContentLockDeletionJob, - ContentLockDeletionPhase, ContentLockDeletionState, + ContentLockDeletionPhase, ContentLockDeletionState, PrepareForceDeletionResult, }, ports::ContentLockDeletionRepository, }; @@ -53,6 +53,66 @@ impl PostgresContentLockDeletionRepository { #[async_trait] impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { + async fn begin_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result<(), ApplicationError> { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, creator, lock_id).await?; + let deletion_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM content_lock_force_deletion_receipts WHERE creator = $1 AND lock_id = $2) + OR EXISTS (SELECT 1 FROM content_lock_deletion_jobs WHERE creator = $1 AND lock_id = $2)", + ) + .bind(creator.to_string()).bind(lock_id.to_string()) + .fetch_one(&mut *transaction).await.map_err(storage_error)?; + if deletion_exists { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + sqlx::query("INSERT INTO content_lock_publication_intents (creator, lock_id, publication_token) VALUES ($1, $2, $3)") + .bind(creator.to_string()).bind(lock_id.to_string()).bind(publication_token) + .execute(&mut *transaction).await.map_err(map_publication_insert_error)?; + transaction.commit().await.map_err(storage_error) + } + + async fn finish_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result { + delete_publication_intent(&self.pool, creator, lock_id, publication_token).await + } + + async fn abandon_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result { + delete_publication_intent(&self.pool, creator, lock_id, publication_token).await + } + + async fn publication_in_progress( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, creator, lock_id).await?; + let exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM content_lock_publication_intents WHERE creator = $1 AND lock_id = $2)", + ) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(exists) + } + async fn insert_job(&self, job: ContentLockDeletionJob) -> Result<(), ApplicationError> { job.validate_frozen_identity()?; job.validate_state(false)?; @@ -64,6 +124,20 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { ); let mut transaction = self.pool.begin().await.map_err(storage_error)?; lock_proof_admission(&mut transaction, &job.creator, &job.lock_id).await?; + let deletion_cutoff_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM content_lock_force_deletion_receipts + WHERE creator = $1 AND lock_id = $2) + OR EXISTS (SELECT 1 FROM content_lock_publication_intents + WHERE creator = $1 AND lock_id = $2)", + ) + .bind(job.creator.to_string()) + .bind(job.lock_id.to_string()) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if deletion_cutoff_exists { + return Err(ApplicationError::ContentLockDeletionInProgress); + } sqlx::query( "INSERT INTO content_lock_deletion_jobs (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, state, phase, @@ -277,31 +351,129 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { ) } - async fn request_force( + async fn resume_failed_job( &self, creator: &CreatorPubky, lock_id: &LockId, - requested_at: OffsetDateTime, - ) -> Result { - let result = sqlx::query( - "UPDATE content_lock_deletion_jobs SET force_requested_at = $3, updated_at = $3 - WHERE creator = $1 AND lock_id = $2 AND force_requested_at IS NULL", + resumed_at: OffsetDateTime, + ) -> Result, ApplicationError> { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, creator, lock_id).await?; + let receipt_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM content_lock_force_deletion_receipts + WHERE creator = $1 AND lock_id = $2)", ) .bind(creator.to_string()) .bind(lock_id.to_string()) - .bind(requested_at) - .execute(&self.pool) + .fetch_one(&mut *transaction) .await .map_err(storage_error)?; - Ok(result.rows_affected() == 1) + if receipt_exists { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let sql = format!( + "UPDATE content_lock_deletion_jobs + SET state = 'queued', attempt_count = 0, next_attempt_at = NULL, + failure_code = NULL, claimed_by = NULL, claim_token = NULL, + claim_expires_at = NULL, updated_at = $3 + WHERE creator = $1 AND lock_id = $2 AND state = 'failed' + RETURNING {ROW_COLUMNS}" + ); + let resumed = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .bind(resumed_at) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + let current = if resumed.is_some() { + resumed + } else { + let sql = format!( + "SELECT {ROW_COLUMNS} FROM content_lock_deletion_jobs + WHERE creator = $1 AND lock_id = $2" + ); + sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)? + }; + transaction.commit().await.map_err(storage_error)?; + fetch_optional_job(current) } - async fn record_force_receipt( + async fn prepare_force_deletion( &self, creator: &CreatorPubky, lock_id: &LockId, forced_at: OffsetDateTime, - ) -> Result<(), ApplicationError> { + ) -> Result { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, creator, lock_id).await?; + let publication_in_progress = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM content_lock_publication_intents WHERE creator = $1 AND lock_id = $2)", + ) + .bind(creator.to_string()).bind(lock_id.to_string()) + .fetch_one(&mut *transaction).await.map_err(storage_error)?; + if publication_in_progress { + transaction.commit().await.map_err(storage_error)?; + return Ok(PrepareForceDeletionResult::PublicationInProgress); + } + let sql = format!( + "SELECT {ROW_COLUMNS} FROM content_lock_deletion_jobs + WHERE creator = $1 AND lock_id = $2 FOR UPDATE" + ); + let existing = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + if let Some(row) = existing { + let job = row_to_job(row)?; + if matches!( + job.state, + ContentLockDeletionState::Queued | ContentLockDeletionState::Running + ) { + let sql = format!( + "UPDATE content_lock_deletion_jobs + SET force_requested_at = COALESCE(force_requested_at, $3), + state = 'queued', next_attempt_at = NULL, + claimed_by = NULL, claim_token = NULL, claim_expires_at = NULL, + updated_at = $3 + WHERE creator = $1 AND lock_id = $2 RETURNING {ROW_COLUMNS}" + ); + let active = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .bind(forced_at) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + return Ok(PrepareForceDeletionResult::Active(row_to_job(active)?)); + } + sqlx::query( + "INSERT INTO content_lock_force_deletion_receipts (creator, lock_id, forced_at) + VALUES ($1, $2, $3) ON CONFLICT (creator, lock_id) DO NOTHING", + ) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .bind(forced_at) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + sqlx::query("DELETE FROM content_lock_deletion_jobs WHERE job_id = $1") + .bind(job.job_id) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + return Ok(PrepareForceDeletionResult::Synchronous(Some(job))); + } sqlx::query( "INSERT INTO content_lock_force_deletion_receipts (creator, lock_id, forced_at) VALUES ($1, $2, $3) ON CONFLICT (creator, lock_id) DO NOTHING", @@ -309,10 +481,11 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { .bind(creator.to_string()) .bind(lock_id.to_string()) .bind(forced_at) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(storage_error)?; - Ok(()) + transaction.commit().await.map_err(storage_error)?; + Ok(PrepareForceDeletionResult::Synchronous(None)) } async fn has_force_receipt( @@ -332,6 +505,32 @@ impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { } } +async fn delete_publication_intent( + pool: &PgPool, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, +) -> Result { + let mut transaction = pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, creator, lock_id).await?; + let result = sqlx::query("DELETE FROM content_lock_publication_intents WHERE creator = $1 AND lock_id = $2 AND publication_token = $3") + .bind(creator.to_string()).bind(lock_id.to_string()).bind(publication_token) + .execute(&mut *transaction).await.map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(result.rows_affected() == 1) +} + +fn map_publication_insert_error(error: sqlx::Error) -> ApplicationError { + if let sqlx::Error::Database(database_error) = &error + && database_error.is_unique_violation() + { + return ApplicationError::ContentLockPathConflict { + guarded_path: "content lock publication in progress".to_owned(), + }; + } + storage_error(error) +} + async fn load_owned_claim( transaction: &mut Transaction<'_, Postgres>, job_id: Uuid, @@ -492,7 +691,8 @@ mod tests { errors::ApplicationError, models::{ ContentLockDeletionFailureCode, ContentLockDeletionJob, ContentLockDeletionPhase, - ContentLockDeletionState, VerificationTaskRecord, VerificationTaskStatus, + ContentLockDeletionState, PrepareForceDeletionResult, VerificationTaskRecord, + VerificationTaskStatus, }, ports::{ContentLockDeletionRepository, VerificationTaskRepository}, }, @@ -582,6 +782,95 @@ mod tests { } } + #[tokio::test] + async fn concurrent_graceful_start_and_force_prepare_leave_exactly_one_durable_mode() { + for _ in 0..20 { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + + let (graceful, force) = tokio::join!( + repository.insert_job(job.clone()), + repository.prepare_force_deletion(&job.creator, &job.lock_id, NOW) + ); + let persisted_job = repository + .get_job(&job.creator, &job.lock_id) + .await + .unwrap(); + let receipt = repository + .has_force_receipt(&job.creator, &job.lock_id) + .await + .unwrap(); + + match (graceful, force) { + (Ok(()), Ok(PrepareForceDeletionResult::Active(active))) => { + assert_eq!(active.job_id, job.job_id); + assert!(active.force_requested_at.is_some()); + assert_eq!(persisted_job, Some(active)); + assert!(!receipt); + } + ( + Err(ApplicationError::ContentLockDeletionInProgress), + Ok(PrepareForceDeletionResult::Synchronous(None)), + ) => { + assert!(persisted_job.is_none()); + assert!(receipt); + } + other => panic!("unexpected graceful/force race result: {other:?}"), + } + + database.cleanup().await; + } + } + + #[tokio::test] + async fn publication_intent_and_force_receipt_have_one_serialized_cutoff_order() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let lock = content_lock(); + let lock_id = lock.lock_id().unwrap(); + let token = Uuid::new_v4(); + + repository + .begin_publication(&lock.creator, &lock_id, token) + .await + .unwrap(); + assert_eq!( + repository + .prepare_force_deletion(&lock.creator, &lock_id, NOW) + .await + .unwrap(), + PrepareForceDeletionResult::PublicationInProgress + ); + assert!( + !repository + .has_force_receipt(&lock.creator, &lock_id) + .await + .unwrap() + ); + assert!( + repository + .finish_publication(&lock.creator, &lock_id, token) + .await + .unwrap() + ); + assert_eq!( + repository + .prepare_force_deletion(&lock.creator, &lock_id, NOW) + .await + .unwrap(), + PrepareForceDeletionResult::Synchronous(None) + ); + assert_eq!( + repository + .begin_publication(&lock.creator, &lock_id, Uuid::new_v4()) + .await, + Err(ApplicationError::ContentLockDeletionInProgress) + ); + + database.cleanup().await; + } + #[tokio::test] async fn durable_paykit_reservation_commits_before_deletion_and_is_snapshotted() { use crate::infrastructure::postgres::PostgresPaykitTaskAdmissionRepository; @@ -860,37 +1149,33 @@ mod tests { Some(ContentLockDeletionFailureCode::TombstoneMissing) ); - assert!( + assert!(matches!( reopened - .request_force(&job.creator, &job.lock_id, NOW) + .prepare_force_deletion(&job.creator, &job.lock_id, NOW) .await - .unwrap() - ); + .unwrap(), + PrepareForceDeletionResult::Synchronous(Some(_)) + )); + assert!(matches!( + reopened + .prepare_force_deletion(&job.creator, &job.lock_id, NOW) + .await + .unwrap(), + PrepareForceDeletionResult::Synchronous(None) + )); assert!( - !reopened - .request_force(&job.creator, &job.lock_id, NOW) + reopened + .has_force_receipt(&job.creator, &job.lock_id) .await .unwrap() ); - reopened - .record_force_receipt(&job.creator, &job.lock_id, NOW) - .await - .unwrap(); - reopened - .record_force_receipt(&job.creator, &job.lock_id, NOW) - .await - .unwrap(); assert!( reopened - .has_force_receipt(&job.creator, &job.lock_id) + .get_job(&job.creator, &job.lock_id) .await .unwrap() + .is_none() ); - sqlx::query("DELETE FROM content_lock_deletion_jobs WHERE job_id = $1") - .bind(job.job_id) - .execute(database.pool()) - .await - .unwrap(); assert!( reopened .has_force_receipt(&job.creator, &job.lock_id) @@ -920,6 +1205,80 @@ mod tests { database.cleanup().await; } + #[tokio::test] + async fn force_escalation_invalidates_the_active_claim_and_requeues_for_force_processing() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + let claimed = repository + .claim_next("graceful-worker", NOW, NOW + time::Duration::minutes(1)) + .await + .unwrap() + .unwrap(); + + let escalated = repository + .prepare_force_deletion(&job.creator, &job.lock_id, NOW) + .await + .unwrap(); + let PrepareForceDeletionResult::Active(escalated) = escalated else { + panic!("active job must be escalated asynchronously"); + }; + assert_eq!(escalated.state, ContentLockDeletionState::Queued); + assert!(escalated.force_requested_at.is_some()); + + assert_eq!( + repository + .schedule_retry( + job.job_id, + "graceful-worker", + claimed.claim_token, + NOW, + NOW + time::Duration::seconds(1), + ) + .await + .unwrap(), + None + ); + assert_eq!( + repository + .advance_phase( + job.job_id, + "graceful-worker", + claimed.claim_token, + NOW, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap(), + None + ); + assert_eq!( + repository + .finish( + job.job_id, + "graceful-worker", + claimed.claim_token, + NOW, + None, + ) + .await + .unwrap(), + None + ); + + let force_claim = repository + .claim_next("force-worker", NOW, NOW + time::Duration::minutes(1)) + .await + .unwrap() + .unwrap(); + assert_eq!(force_claim.job.job_id, job.job_id); + assert!(force_claim.job.force_requested_at.is_some()); + assert_ne!(force_claim.claim_token, claimed.claim_token); + + database.cleanup().await; + } + #[tokio::test] async fn read_rejects_corrupt_frozen_manifest_identity() { let database = TestDatabase::create().await; diff --git a/locks-service/src/infrastructure/postgres/migrations.rs b/locks-service/src/infrastructure/postgres/migrations.rs index 98b492a..db6b1b7 100644 --- a/locks-service/src/infrastructure/postgres/migrations.rs +++ b/locks-service/src/infrastructure/postgres/migrations.rs @@ -52,6 +52,7 @@ mod tests { assert_table_exists(&mut connection, "content_lock_ownership").await; assert_table_exists(&mut connection, "content_lock_deletion_jobs").await; assert_table_exists(&mut connection, "content_lock_force_deletion_receipts").await; + assert_table_exists(&mut connection, "content_lock_publication_intents").await; assert_table_exists(&mut connection, "content_lock_deletion_task_snapshot").await; assert_table_exists(&mut connection, "paykit_task_admissions").await; assert_column_exists(&mut connection, "verification_tasks", "creator").await; diff --git a/locks-service/src/infrastructure/pubky/content_locks.rs b/locks-service/src/infrastructure/pubky/content_locks.rs index 89d1a94..ebe0819 100644 --- a/locks-service/src/infrastructure/pubky/content_locks.rs +++ b/locks-service/src/infrastructure/pubky/content_locks.rs @@ -61,6 +61,21 @@ where }) .transpose() } + + async fn delete_content_lock( + &self, + creator: &CreatorPubky, + content_lock_path: &ContentLockPath, + ) -> Result { + let path = content_lock_path.to_string(); + let existed = self + .client + .get_json_value_as_creator(creator, &path) + .await? + .is_some(); + self.client.delete_as_creator(creator, &path).await?; + Ok(existed) + } } #[cfg(test)] @@ -144,6 +159,32 @@ mod tests { assert_eq!(loaded, None); } + #[tokio::test] + async fn delete_content_lock_reads_and_deletes_the_exact_canonical_path() { + let requested_path = content_lock(900).content_lock_path().unwrap(); + let repository = PubkyContentLockRepository::new( + FakeStorageClient::default().with_json_read(Some(json!({}))), + ); + + assert!( + repository + .delete_content_lock(&creator(), &requested_path) + .await + .unwrap() + ); + assert_eq!( + repository.client().operations(), + vec![ + format!( + "get_json pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy {requested_path}" + ), + format!( + "delete pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy {requested_path}" + ) + ] + ); + } + #[tokio::test] async fn storage_errors_are_propagated() { let repository = PubkyContentLockRepository::new(FakeStorageClient::default().with_error( @@ -253,10 +294,15 @@ mod tests { async fn delete_as_creator( &self, - _creator: &CreatorPubky, - _path: &str, + creator: &CreatorPubky, + path: &str, ) -> Result<(), ApplicationError> { - unimplemented!("not needed by content lock repository tests") + self.maybe_error()?; + self.operations + .lock() + .unwrap() + .push(format!("delete {creator} {path}")); + Ok(()) } } diff --git a/locks-service/tests/content_lock_deletions.rs b/locks-service/tests/content_lock_deletions.rs index 91a64a4..e302635 100644 --- a/locks-service/tests/content_lock_deletions.rs +++ b/locks-service/tests/content_lock_deletions.rs @@ -11,7 +11,7 @@ use locks_service::{ application::{ models::{ ContentLockDeletionFailureCode, ContentLockDeletionJob, ContentLockDeletionPhase, - ContentLockDeletionState, + ContentLockDeletionState, PrepareForceDeletionResult, }, ports::ContentLockDeletionRepository, }, @@ -224,28 +224,15 @@ async fn retry_due_time_and_force_receipts_are_durable_repository_facts() { .is_some() ); - assert!( + assert!(matches!( repository - .request_force(&job.creator, &job.lock_id, NOW) + .prepare_force_deletion(&job.creator, &job.lock_id, NOW) .await - .unwrap() - ); + .unwrap(), + PrepareForceDeletionResult::Active(_) + )); assert!( !repository - .request_force(&job.creator, &job.lock_id, NOW) - .await - .unwrap() - ); - repository - .record_force_receipt(&job.creator, &job.lock_id, NOW) - .await - .unwrap(); - repository - .record_force_receipt(&job.creator, &job.lock_id, NOW) - .await - .unwrap(); - assert!( - repository .has_force_receipt(&job.creator, &job.lock_id) .await .unwrap()