diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 8c3d582bbb..fc47ecb3da 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,4 +1,4 @@ { - ".": "9.3.0", + ".": "9.6.1", "packages/adcp-reporting-storage": "0.1.0-beta.4" } diff --git a/.traycer/environment.json b/.traycer/environment.json new file mode 100644 index 0000000000..c7e50a3c3f --- /dev/null +++ b/.traycer/environment.json @@ -0,0 +1,15 @@ +{ + "setup": { + "default": "ROOT=\"$(cd \"$(git rev-parse --git-common-dir)/..\" && pwd)\" || exit 1\nfor f in \"$ROOT\"/.env*; do n=\"${f##*/}\"; [ -f \"$f\" ] && [ ! -e \"$n\" ] && git check-ignore -q \"$n\" && cp \"$f\" \"$n\"; done; true || exit 1\nuv sync --all-extras || exit 1\nmake bootstrap || exit 1", + "macos": null, + "linux": null, + "windows": null + }, + "teardown": { + "default": "", + "macos": null, + "linux": null, + "windows": null + }, + "updatedAt": 1793934266991 +} diff --git a/AGENTS.md b/AGENTS.md index c552274c42..0cf47de627 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -272,6 +272,19 @@ from adcp.testing import build_asgi_app, build_test_client, make_request_context ## Property Lists and Identifier Resolution +For publisher manifest sync, resolve the agent's selectors with +`get_properties_by_agent` or `resolve_properties_for_agent`, then call +`properties_for_publisher_domain(properties, publisher_domain)` to attribute +inventory. It keeps publisher subdomains and wildcard domain inventory, and +excludes domainless inventory (bundle-only apps, venues) unless +`include_domainless=True`. Permissive resolution requires `publisher_domain=` +from the trusted fetch origin (`allow_any_domain=True` skips the gate for +diagnostics). Use `await verify_cross_domain_properties(properties, +source_domain=..., agent_url=...)` before trusting claims about another +publisher's domain; its `CrossDomainVerification` result separates +`verified`, `unverified` and `domainless` properties. All three helpers are +exported from `adcp`. See `docs/adagents-discovery.md`. + Import `MCPPropertyListFetcher`, `ProductPropertyPolicy`, and `check_property_list_reference` from `adcp`. The default seller fetcher resolves lists with MCP `get_property_list`, follows pagination, and caches complete diff --git a/CHANGELOG.md b/CHANGELOG.md index 813d079e60..5046841399 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -236,6 +236,75 @@ change public consumer defaults. See [format-kind migration](docs/canonical-format-kinds-migration.md). +## [9.6.1](https://github.com/adcontextprotocol/adcp-client-python/compare/v9.6.0...v9.6.1) (2026-11-03) + + +### Bug Fixes + +* **reporting:** isolate handler registrations for each installation ([#1847](https://github.com/adcontextprotocol/adcp-client-python/issues/1847)) ([86302d0](https://github.com/adcontextprotocol/adcp-client-python/commit/86302d09404067410a60d3f9d0f850790038a36f)) + +## [9.6.0](https://github.com/adcontextprotocol/adcp-client-python/compare/v9.5.1...v9.6.0) (2026-11-03) + + +### Features + +* **reporting-storage:** accept 255 character row binding IDs and credential refs ([#1828](https://github.com/adcontextprotocol/adcp-client-python/issues/1828)) ([6bb34df](https://github.com/adcontextprotocol/adcp-client-python/commit/6bb34df00821e11f03aed2a018a0c5a6ccab3cdf)) +* **reporting-storage:** collect orphaned row objects by listing with a clone fence ([#1835](https://github.com/adcontextprotocol/adcp-client-python/issues/1835)) ([e004f1b](https://github.com/adcontextprotocol/adcp-client-python/commit/e004f1bc55eff3bc32065a9c785f451af3813f79)) +* **reporting-storage:** order the account change feed by xid and seq ([#1827](https://github.com/adcontextprotocol/adcp-client-python/issues/1827)) ([4449bc9](https://github.com/adcontextprotocol/adcp-client-python/commit/4449bc94cdd47900f39e15f5e99a35a9699d0c30)) + + +### Bug Fixes + +* **server:** preserve valid creatives in lenient legacy batches ([#1841](https://github.com/adcontextprotocol/adcp-client-python/issues/1841)) ([2bd7751](https://github.com/adcontextprotocol/adcp-client-python/commit/2bd7751cb1312efda5aefc4a11fecdfab0933c9e)) + + +### Performance Improvements + +* **reporting-storage:** drop the per-object advisory lock where nothing pairs on it ([#1836](https://github.com/adcontextprotocol/adcp-client-python/issues/1836)) ([536b8f8](https://github.com/adcontextprotocol/adcp-client-python/commit/536b8f870f518e7ceab3c7a1d64aa0b229b75e8e)) + +## [9.5.1](https://github.com/adcontextprotocol/adcp-client-python/compare/v9.5.0...v9.5.1) (2026-11-03) + + +### Bug Fixes + +* **ci:** check PR titles without stripping a trailing issue reference ([#1819](https://github.com/adcontextprotocol/adcp-client-python/issues/1819)) ([495593e](https://github.com/adcontextprotocol/adcp-client-python/commit/495593ec319bca75b22174c582fa5417f9730860)) + +## [9.5.0](https://github.com/adcontextprotocol/adcp-client-python/compare/v9.4.0...v9.5.0) (2026-11-02) + + +### Features + +* **decisioning:** let a raised AdcpError reject a request with every error ([#1780](https://github.com/adcontextprotocol/adcp-client-python/issues/1780)) ([#1810](https://github.com/adcontextprotocol/adcp-client-python/issues/1810)) ([272fced](https://github.com/adcontextprotocol/adcp-client-python/commit/272fced52d57632dedf32f2c175cf294100bff63)) + + +### Bug Fixes + +* **canonical-formats:** check narrowing on seller-asserted references ([#1758](https://github.com/adcontextprotocol/adcp-client-python/issues/1758)) ([617762e](https://github.com/adcontextprotocol/adcp-client-python/commit/617762e94371b5817f400481bbbc866c760a6a1a)) +* **canonical-formats:** read versions from typed legacy formats ([#1757](https://github.com/adcontextprotocol/adcp-client-python/issues/1757)) ([6714203](https://github.com/adcontextprotocol/adcp-client-python/commit/67142036f40e6a68fd9ca3d5c51acaa7bfe402fb)) +* **server:** return task and workflow handoffs uncached from IdempotencyStore.wrap ([#1795](https://github.com/adcontextprotocol/adcp-client-python/issues/1795)) ([126f905](https://github.com/adcontextprotocol/adcp-client-python/commit/126f9052ab79399f1e7bab8a53f34e24999ef95e)), closes [#1781](https://github.com/adcontextprotocol/adcp-client-python/issues/1781) +* **types:** make the domain package roots lazy so a types package imports first in any order ([#1798](https://github.com/adcontextprotocol/adcp-client-python/issues/1798)) ([7447d00](https://github.com/adcontextprotocol/adcp-client-python/commit/7447d00ceba97490109c46f0cf2f0ca9d6f8e5f3)) + +## [9.4.0](https://github.com/adcontextprotocol/adcp-client-python/compare/v9.3.1...v9.4.0) (2026-10-31) + + +### Features + +* **adagents:** attribute and verify publisher property claims ([#1719](https://github.com/adcontextprotocol/adcp-client-python/issues/1719)) ([0c0ef92](https://github.com/adcontextprotocol/adcp-client-python/commit/0c0ef92515ec5dfe9d1f045b976194d3d245d4e5)) +* **server:** seller helper for wholesale feed versions ([#1718](https://github.com/adcontextprotocol/adcp-client-python/issues/1718)) ([7088ebf](https://github.com/adcontextprotocol/adcp-client-python/commit/7088ebf909852134521ba0ed4796b6f2896a4ad7)), closes [#1676](https://github.com/adcontextprotocol/adcp-client-python/issues/1676) +* **signing:** accept any well-formed adcp_use purpose in keygen ([#1710](https://github.com/adcontextprotocol/adcp-client-python/issues/1710)) ([787cfb2](https://github.com/adcontextprotocol/adcp-client-python/commit/787cfb215cf9207bfef8c28cc34ff0b737cea39b)) + + +### Bug Fixes + +* **types:** defer the import-time model rebuilds that cost adcp.client 560 MiB RSS ([#1704](https://github.com/adcontextprotocol/adcp-client-python/issues/1704)) ([aac52ca](https://github.com/adcontextprotocol/adcp-client-python/commit/aac52cac66ecae2d1c2c4e11be23294335a3edd8)), closes [#1694](https://github.com/adcontextprotocol/adcp-client-python/issues/1694) + +## [9.3.1](https://github.com/adcontextprotocol/adcp-client-python/compare/v9.3.0...v9.3.1) (2026-10-29) + + +### Bug Fixes + +* **server:** compact a pinned seller's tools/list so an SDK 10 buyer can call it ([#1702](https://github.com/adcontextprotocol/adcp-client-python/issues/1702)) ([cc11643](https://github.com/adcontextprotocol/adcp-client-python/commit/cc11643352a078b977449f44dcd2d5687ad29bf6)), closes [#1692](https://github.com/adcontextprotocol/adcp-client-python/issues/1692) + ## [9.3.0](https://github.com/adcontextprotocol/adcp-client-python/compare/v9.2.0...v9.3.0) (2026-10-27) diff --git a/docs/reporting-ledger-migration.md b/docs/reporting-ledger-migration.md index 36578c91aa..b1267ac461 100644 --- a/docs/reporting-ledger-migration.md +++ b/docs/reporting-ledger-migration.md @@ -368,3 +368,35 @@ seeds ledgers with the actual published wheel pairs (`adcp==9.3.1` with `adcp-reporting-storage==0.1.0b4`, `adcp==10.0.0b2` with `0.2.0b0`) and proves the retained evidence reads back unchanged, a restart rebinds the stored generations, and the upgraded catalog is identical to a fresh migration. + +## Upgrading a ledger that adcp-reporting-storage 0.1.0b5 wrote + +0.1.0b5 is the SDK 9 storage release that orders the account change feed by +`(xid, seq)`, accepts 255-character row binding IDs and credential refs, and +adds the opt-in orphaned-object collector. Its ledger upgrades to this release +through the same two entry points and the same numbered migrations as a +0.1.0b4 ledger (above): this release adds no migration for them. Their DDL is +in the files `create_schema()` re-applies on every start when the option is +on (`reporting_changes.sql` with `change_feed=True`, +`reporting_row_storage.sql` with `row_storage`), as on the SDK 9 line, and +`python -m adcp migrate` never creates those tables. A ledger created before +0.1.0b5 gains the account horizon's xid columns, the named feed-consumer and +row-binding constraints, the `(account_id, xid, seq)` index and the orphan +collector fence table the first time this release starts with those options; +see [host change feed and consumers](reporting-row-storage.md#host-change-feed-and-consumers) +for creating that index concurrently on a large changes table. + +adcp-reporting-storage 0.2.0b0 to 0.2.0b2 predate the `(xid, seq)` account +feed. On a 0.1.0b5 ledger they refuse an account cursor that carries an +`xid`, refuse a row binding ID over 128 characters, and prune the change feed +without advancing the account horizon's xid columns. A cursor 0.1.0b5 handed +out could then pass the account horizon check and skip what 0.2.x pruned. So +the first time this release starts with `change_feed=True` on a 0.1.0b5 ledger +that a 0.2 release has already upgraded, it raises each account horizon to the +deployment horizon, or to just below the account's oldest retained change when +that is lower. Those cursors then fail closed with `CURSOR_EXPIRED`; a +consumer starting over still reads every retained change. It runs once (a +comment on `adcp_reporting_change_account_horizon` records it) and changes +nothing on a ledger upgraded straight from 0.1.0b5. As for the `(xid, seq)` +upgrade on SDK 9, stop every older host's change maintenance before this +release starts. diff --git a/docs/reporting-release-notes.md b/docs/reporting-release-notes.md index 9cd6bcdf80..e25641324d 100644 --- a/docs/reporting-release-notes.md +++ b/docs/reporting-release-notes.md @@ -76,6 +76,23 @@ restart across the upgrade does not turn a replay into a conflict. This is part the shared layout in adcontextprotocol/adcp#7996 (Stage 2, tracked in [#1457](https://github.com/adcontextprotocol/adcp-client-python/issues/1457)). +## Account change feed orders by `(xid, seq)` + +The account change feed (`changes_after(account_id=...)`) now orders by `(xid, seq)` behind +`xid < pg_snapshot_xmin(pg_current_snapshot())`, as the deployment-wide feed already did, matching the JavaScript SDK +([adcp-client#3289](https://github.com/adcontextprotocol/adcp-client/pull/3289)). It replaces the page truncation at +the first in-flight change from #1483, and no longer depends on the account lock for its order: a lower `seq` that +commits late is returned on a later page, never skipped. Account cursors now carry `xid`. A seq-only account cursor +saved by an earlier release is still accepted and mapped to a position that skips nothing, at the cost of redelivering +a few changes, and it still expires exactly (`pruned_through_seq` is kept). `reporting_changes.sql` is re-applied by +`create_schema` and is idempotent: it adds the account horizon's xid columns, replaces the consumer check constraint, +and creates `adcp_reporting_changes_account_xid`. Create that index with `CREATE INDEX CONCURRENTLY` first on a large +changes table, stop older hosts' change maintenance before the new reader runs, and do not roll back after a consumer +saved a new-format account cursor. Not affected: deployment-wide cursors, the buyer-facing `changes_after` projection and +anyone who never enabled `change_feed`. See +[host change feed and consumers](reporting-row-storage.md#host-change-feed-and-consumers) +([#1825](https://github.com/adcontextprotocol/adcp-client-python/issues/1825)). + ## Feed consumer checkpoints are forward-only `save_feed_consumer_cursor` used to replace the saved position unconditionally. It now ignores a cursor behind the @@ -94,6 +111,56 @@ Reads handle both forms. `migrate_inline_rows` moves legacy rows with verificati rows do not reproduce its binding, and `retire_legacy_inline_rows` then renames the legacy table so older binaries fail closed. See [row storage](reporting-row-storage.md). +## The per-object lock is kept only where it pairs + +Retirement, the intent sweep and a commit that adopts an object its own intent recorded no longer take the per-object +advisory lock, and no longer hold a transaction or pooled connection across the provider delete. The lock stays in the +two places that pair with each other: the orphan collector's delete, which locks every binding's key for the object, and +a commit that adopts an object its intent did not record. The lock text is unchanged, so a Python deleter and a +JavaScript adopter still agree, and rolling upgrades between a version that locks everywhere and this one are safe (every +pair that needs exclusion keeps the lock on both sides). A commit now closes its intent with `RETURNING created_objects` +to know which adopted objects it recorded. No schema or wire change. See +[the object lock contract](reporting-row-storage.md#the-object-lock-contract) +([#1492](https://github.com/adcontextprotocol/adcp-client-python/issues/1492), adcp-client#3606). + +## Opt-in orphaned-object collector, fenced against clones + +`PgReportingLedgerStore(orphan_collection=ReportingOrphanCollectionOptionsV1(...))` and +`collect_orphaned_row_objects` list each object binding under its owned prefix and delete the objects no ledger record +or open upload names, which the write-intent sweep cannot see (an upload that succeeded before its intent recorded it, +intents for a binding no host configures, intents lost to a restore). It mirrors the JavaScript SDK +([adcp-client#3291](https://github.com/adcontextprotocol/adcp-client/pull/3291) and the clone fence in +[#3593](https://github.com/adcontextprotocol/adcp-client/pull/3593)). Nothing changes unless you opt in. An object is +deleted only when it is a well-formed row key under `{prefix}/{namespace_key}/`, older than `min_age_seconds` (default +7 days, minimum 1 day), named by no record or upload at any version under any binding, and unreferenced under the +per-object lock of every binding. A real pass first claims the namespace in a create-only fence in the bucket and +refuses to delete while another database holds a newer claim, so a clone or restore of the ledger database that shares +its namespace key and its opt-in cannot delete the live database's newer objects; `take_over_orphan_collector_fence` +is the deliberate step after a restore. + +Providers gain an optional `list`. The provider conformance suite checks it when present, so its result now ends with a +`list` entry for providers that implement it; compare accordingly. A provider without `list` keeps working and is +reported `unsupported`. `create_schema` adds the shared `adcp_reporting_orphan_collector_fences` table, additively and +outside the required-schema manifest. The production worker reports `fenced` bindings as stalled work and `unsupported` +bindings as pending work. **Roll out:** start with `dry_run=True`, claim from the live database first, and never enable +the collector on a copy of the deployment. See [the listing collector](reporting-row-storage.md#orphaned-objects-the-opt-in-listing-collector) +([#1829](https://github.com/adcontextprotocol/adcp-client-python/issues/1829)). + +## Row binding IDs and credential references accept 255 characters + +`row_binding_id` and a binding's `credential_ref` were capped at 128 characters. Every comparable AdCP identifier +(`reporting_revision_id`, `resource_ref`, `destination_ref`) allows 255, so an ID derived from an AdCP destination +reference could be refused for no protocol reason. Both now accept 1 to 255 characters of `A-Za-z0-9_.:-`, matching the +JavaScript SDK ([adcp-client#3595](https://github.com/adcontextprotocol/adcp-client/pull/3595)). The configuration +check, the `adcp_reporting_row_bindings` `CHECK` and the provider `clients` map share the bound, which is exported as +`REPORTING_ROW_BINDING_ID_MAX_LENGTH` and `REPORTING_ROW_CREDENTIAL_REF_MAX_LENGTH`. A malformed value that the old +check let through (a trailing newline) is now refused too. `create_schema` replaces the original unnamed 128-character +`CHECK` with `adcp_reporting_row_bindings_row_binding_id_bounds` in place, idempotently, with no data migration; stop +older writers first, because an older binary requires the original constraint and fails closed against an upgraded +database. Nothing that was valid before is refused. See +[stored names follow AdCP](reporting-row-storage.md#stored-names-follow-adcp) +([#1498](https://github.com/adcontextprotocol/adcp-client-python/issues/1498)). + ## Row storage field alignment (breaking for row storage adopters) Row storage stores every value in the format AdCP defines for it, matching the JavaScript SDK byte for byte: diff --git a/docs/reporting-row-storage.md b/docs/reporting-row-storage.md index 7c8d3ec48e..5095fd3023 100644 --- a/docs/reporting-row-storage.md +++ b/docs/reporting-row-storage.md @@ -164,6 +164,13 @@ await store.create_schema() fails with `UNSAFE_BINDING`, never `INVALID_INPUT`, which would read as bad rows), is not part of the binding identity, and is still used for reads and retirement. Rows already written stay readable at their recorded `object_ref`; reads never re-render a key. Keep a retired binding configured while revisions still reference it. +- **Identifiers you choose are AdCP-bounded.** A binding ID and a binding's `credential_ref` (and the keys of a + provider's `clients` map) are 1 to 255 characters of `A-Za-z0-9_.:-`, the bound `reporting_revision_id`, + `resource_ref` and `destination_ref` use (`REPORTING_ROW_BINDING_ID_MAX_LENGTH` and + `REPORTING_ROW_CREDENTIAL_REF_MAX_LENGTH` in `adcp_reporting_storage.ledger.row_storage_object`). They were capped at + 128, so an ID derived from an AdCP destination reference could be refused for no protocol reason. A longer or + malformed value fails `INVALID_INPUT` at configuration with the bound named (a long ID is truncated in the message, + never echoed whole). The `adcp_reporting_row_bindings` `CHECK` is the same bound. - **`{period_date}` is AdCP's `YYYY-MM-DD`**: the calendar date of the period start in the period's `source_timezone`, as one path segment (a period starting `2026-10-01T00:00:00Z` in `America/Los_Angeles` is `2026-09-30`). It fails closed (`INVALID_INPUT`) on a missing, unknown or non-IANA timezone and never assumes UTC; @@ -320,6 +327,141 @@ stays as a small marker that makes reads answer as expired; the records are alre whose horizon excludes periods past retention. Versioned buckets, soft delete and retention locks are allowed as backstops, but a `pruned` period is not "erased" while they keep the old bytes. +## The object lock contract + +The per-object advisory lock is kept only where nothing else orders the two sides, in the contract the JavaScript SDK +shares ([adcp-client#3606](https://github.com/adcontextprotocol/adcp-client/pull/3606)). The lock text is +`adcp-reporting-object:` + binding ID + `:` + object ref, passed to `pg_advisory_xact_lock(hashtextextended(text, 0))`, +so a Python deleter and a JavaScript adopter agree in one database. + +1. **No lock:** retirement, the intent sweep and a commit that adopts an object its own intent recorded. Per object, one + statement checks that no `live` or `unavailable` row set outside the operation names the same binding, `object_ref` + and `native_version_ref`, and no `open` intent in the binding lists it in `created_objects` (both the current and the + pre-alignment `objectKey` / `nativeVersion` spellings are read); then the exact version is deleted. No transaction or + pooled connection is held across the provider delete. +2. **A commit** takes the account lock and closes its intent with `DELETE ... WHERE owner_token = ... AND state = + 'open' RETURNING created_objects` (no row fails `STATE_UNAVAILABLE`). For each adopted object whose + `(object_ref, native_version_ref)` is **not** in the returned list it takes its binding's object lock, in object-ref + order. It then proves every adopted object with a version-pinned `stat` (a full read when the provider has none) + before inserting the chunk rows. +3. **The orphan collector** locks every binding's key for the object (configured plus registered, sorted by binding + ID), repeats the reference check across all bindings under those locks, then deletes. It is the only locked delete. +4. Lock order: the account lock, then object locks. Native versions must be write-unique (the conformance suite's + `write-unique-version`). + +Why this is safe without the lock: the account lock orders commits, retirement phase 1, intent takeover and the sweep +claim; `pruning` is committed under it before any delete, and `pruning` row sets neither count as references nor can be +shared; an intent moves `open -> sweeping` against the commit's `DELETE ... WHERE state = 'open'`, in the same transaction +as the chunk insert, so a reader sees the open intent or the chunk rows, never neither; deletes name an exact native +version; and every writable key includes `{revision_key}`, so one row set alone can name it. **Precondition (documented, +not enforced):** two writable bindings must not share one bucket and prefix, because the reference check is per binding +outside the collector. + +## Orphaned objects: the opt-in listing collector + +The intent sweep deletes only what an intent lists, so it cannot see three cases: an upload whose `PUT` succeeded before +the intent recorded it, intents for a binding no running host configures, and intents lost to a database restore. +`collect_orphaned_row_objects` covers them by listing, and it is **off unless you ask for it**. It mirrors the +JavaScript SDK's collector and fence ([adcp-client#3291](https://github.com/adcontextprotocol/adcp-client/pull/3291), +[#3593](https://github.com/adcontextprotocol/adcp-client/pull/3593)); the design record is +`specs/shared-sdk-persistence-proposal.md` section 4.2. + +```python +from adcp_reporting_storage import ReportingOrphanCollectionOptionsV1 + +store = PgReportingLedgerStore( + pool=pool, + row_storage=options, + # Start with dry_run=True and read the candidates in the task result. A real pass needs the + # acknowledgement below. + orphan_collection=ReportingOrphanCollectionOptionsV1(acknowledge_exclusive_namespace=True), +) +# or, from your own scheduler: +result = await store.collect_orphaned_row_objects(dry_run=True) +``` + +It lists each configured object binding under `{prefix}/{namespace_key}/` and deletes an object only when **all** of +these hold: + +1. Its key is a well-formed row object (`.jsonl` or `.jsonl.gz`, the grammar `render_reporting_row_object_key_v1` + produces, no `.` or `..` segment) under that prefix. Anything else, or anything a provider returns outside the + prefix, is dropped and counted as `foreign`; `delete_objects` independently refuses keys outside the owned prefix. +2. It was written more than `min_age_seconds` ago (default 7 days, **minimum 1 day**, against a write deadline of at + most 10 minutes). An unreadable or non-RFC 3339 write time counts as young. Providers report the later of creation + and modification time. +3. A version- and binding-blind pre-check finds no chunk of a not-yet-`pruned` row set and no write intent (any state, + old or new JSON keys) naming the key. Two bindings can share a bucket and prefix (a retired `read_only` binding kept + next to its successor), so the check is not scoped to the collecting binding. +4. Under the per-object advisory lock of **every** binding, taken in a fixed order so the delete is serialized with an + adoption through any of them (see [the object lock contract](#the-object-lock-contract)), no `live` or `unavailable` row set references that exact version and no open intent + created it. The provider then deletes that exact version. + +`read_only` bindings are skipped unless `include_read_only`. `dry_run` deletes nothing and reports up to 1000 +`candidates`. Deleting requires `acknowledge_exclusive_namespace=True` (a dry run does not). `deleted` counts delete +requests that found nothing referenced; an object another host removed first still counts. A delete that fails is +counted and the pass moves on, and is retried on the next full listing; a listing failure retries the same page. + +**Providers.** A provider gains an optional `list` (`ReportingRowObjectListInputV1` in, a page of +`ReportingRowObjectListEntryV1` out: `object_ref`, `native_version_ref`, `size_bytes`, `created_at`), implemented by the +filesystem, GCS, S3 and Azure Blob providers (on versioned stores it lists noncurrent versions too and never delete +markers). `run_reporting_row_object_provider_conformance_v1` checks it when present and then returns a `list` entry, so +code that compares the whole list must expect it. A provider without `list` keeps working; its binding is reported +`unsupported`. `list` must be read-after-write consistent for keys `put_if_absent` returned: the fence below reads the +newest claim by listing. The filesystem provider's `list` reads and sorts each directory in full and neither lists nor follows symlinks, so it is +for tests and single hosts. + +**Budget and resumability.** `limit` bounds the objects examined per binding per pass; a page cap bounds empty pages; a +repeated page token or an oversized page fails the binding instead of looping; a failure on the first page of a resumed +cursor drops the cursor so it cannot wedge. Pass `result.cursors` back as `cursors` and `result.completed` as +`skip_bindings`. As a store option, `interval_seconds` (default one day) spaces full listings, a listing that does not +finish within `limit` continues on every pass, and the position lives in the process, so a restart begins a new +listing. + +### The clone fence + +A clone or point-in-time restore of the ledger database shares the live database's installation identity, namespace +key and configuration, **including `acknowledge_exclusive_namespace`**, but sees an older set of references, so its +collector would delete the live database's newer objects once they pass the age threshold. A real pass is therefore +fenced: + +- **Fence.** `{prefix}/{namespace_key}/_collector-fence/{epoch:016d}.json`: one create-only object per epoch, + `{v, namespace_key, epoch, token, previous_token, claimed_at}`, the bytes the JavaScript SDK writes. The highest epoch + is the current claim. The database records its last `(epoch, token, previous_token)` in + `adcp_reporting_orphan_collector_fences`, keyed by a hash of provider, location, prefix and namespace key, so + bindings that reach the same bucket and prefix share one fence. +- **Claim.** A real pass takes a session advisory lock, proceeds only if the bucket's highest epoch is exactly the + recorded claim, commits the intent for `epoch + 1` under a fresh token, writes that object create-only, verifies the + token at the key and prunes all but the predecessor. It holds the lock through the write, so a sibling host of the + same database cannot interleave, and it does this before the first delete. A claim holds one pool connection across + the object-store reads and write. +- **Check.** The fence is read again before the first delete of every page and every 25 deletes within one. Any + mismatch stops the pass; `deleted` keeps what went before. +- **Refusal.** A refused binding has `status == "fenced"` and `fence` (`cause`, `detail`, `fence_epoch`, + `recorded_epoch`); the result lists it in `fenced`. Causes: `claimed_elsewhere`, `fence_advanced`, `token_mismatch`, + `fence_missing`, `fence_regressed`, `fence_unreadable`, `claim_lost`, `migration_required`. Everything unexpected + fails closed, including a fence-directory listing in which the provider left out an entry it could not address (it could be + the newest claim). A provider or database error is `failed`, not `fenced`. `dry_run` reads the fence without claiming it + and reports fenced bindings too. +- **Crash safety.** The intent commits before the object is written. A crash between the two leaves the database one + epoch ahead of the bucket with the matching previous token; the next pass finishes the claim under a **fresh** token, + so a copy taken in that window cannot adopt the same token as the original. +- **Takeover.** After a restore, or when the previous owner is gone, `await store.take_over_orphan_collector_fence( + acknowledge_previous_owner_gone=True)` claims `max(epoch) + 1` and supersedes every older object and version, + including a damaged chain. It refuses without the acknowledgement. Stop the previous owner's collector first: a pass + already running notices the new epoch at its next check, at most 25 deletes later. + +Known limits. **First claim wins:** nothing in a database distinguishes a copy from the original until one of them +claims, so a clone that collects before the original has run a pass since the copy claims first and the original is +fenced (and reported). The clone still cannot delete an object the original wrote after the copy until it is +`min_age_seconds` old. **Claim from the live database first** when you roll this out: one real pass, or a takeover. +The fence needs write access to each collected binding (a `read_only` binding collected with `include_read_only` +included), and a lifecycle rule must not expire `_collector-fence/`. + +**Rollout.** The fence table is created by `reporting_row_storage.sql` (`create_schema`), outside the required-schema +manifest, so a deployment that never enables the collector needs nothing from it. Until it exists a pass reports every +binding `fenced` with `migration_required`, a dry run included, and deletes nothing. Python supports one writer SDK per +ledger database; if a JavaScript deployment shares the database, claim from one SDK only. + ## Stored names follow AdCP Stored values use the names and bounds AdCP already defines (`reporting-file-manifest.json`, @@ -334,6 +476,7 @@ by an earlier release in place, idempotently, under both SDKs' migration advisor | `adcp_reporting_row_chunks.byte_count` | `canonical_size_bytes` | uncompressed canonical bytes; the digest-pinned manifest value | | `adcp_reporting_row_sets.canonical_byte_count` | `content_size_bytes` | size of the content `content_sha256` covers (a revision's includes the envelope, so it is not the sum of its chunks) | | `adcp_reporting_row_sets.encoding` | `format` = `jsonl`, `algorithm` = `adcp_jcs_rows_v1` | `adcp_jcs_rows_v1` names the per-row RFC 8785 canonicalization | +| `adcp_reporting_row_bindings.row_binding_id` `CHECK` `{1,128}` (unnamed) | `{1,255}`, named `adcp_reporting_row_bindings_row_binding_id_bounds` | AdCP's identifier bound; `credential_ref` is bounded the same in code (it lives in `operational_config`) | In Python: `ReportingRowChunkLocatorV1` has `object_ref`, `native_version_ref` and `size_bytes`; provider inputs take `object_ref` and `native_version_ref`; `ReportingRowBindingSelectionInputV1.content_size_bytes` replaces @@ -343,6 +486,15 @@ change, and no digest changed: the format and algorithm tokens feed none. Open w `{"object_ref", "native_version_ref"}`; the sweep and the reference check also read the earlier `objectKey` / `nativeVersion` spelling, for a writer still running during a rolling upgrade. +Widening `row_binding_id` drops the original unnamed `{1,128}` `CHECK` (found through `pg_get_constraintdef`) and adds +the named `{1,255}` one, only when the database still has the old one, so a current schema changes nothing and no data +migrates; the two tables that reference the ID hold it by foreign key only. A host-customised constraint of another +shape is left alone. The startup catalog check requires the new constraint, so an application upgraded before +`create_schema` runs fails closed with `STATE_UNAVAILABLE` naming `adcp_reporting_row_bindings_row_binding_id_bounds`, +and an older binary fails closed against an upgraded database (it requires the original constraint name): stop older +writers first. The JavaScript SDK makes the same change with the same constraint name +([adcp-client#3595](https://github.com/adcontextprotocol/adcp-client/pull/3595)). + The migration does not rewrite recorded object keys. Objects written before the upgrade keep their recorded `object_ref` (rendered under the old UTC `yyyy/MM/dd` layout), so reads, retention and deletes keep working; new writes render the new key. It refuses, naming the count and rolling back, a database that already holds a key or version @@ -386,7 +538,7 @@ missing. The JavaScript SDK has no replay of `python-written.json`; the live tes `PgReportingLedgerStore(pool=pool, change_feed=True)` also appends one row to the shared `adcp_reporting_changes` table (PostgreSQL 13 or later, `xid8`) for every obligation, revision and adjustment, in the transaction that writes -the record and after the account lock. The JavaScript SDK writes the same table. It is separate from +the record. The JavaScript SDK writes the same table. It is separate from `reporting_ledger_changes`, the buyer-facing feed behind `changes_checkpoint` and `changes_after`, until the shared Core layout. @@ -398,18 +550,44 @@ await store.save_feed_consumer_cursor("warehouse-sink", page.cursor) # holds ba await store.changes_after(account_id="acct_a", cursor=page.cursor) ``` -- **Account feeds** order by `seq` and truncate each page before the first change whose transaction is not below - every transaction still in flight (`xid >= pg_snapshot_xmin(pg_current_snapshot())`): a page is the committed prefix - in `seq` order and stops there, and its cursor never moves past a held-back change. `seq` is taken under the account - lock, but ordering does not rely on that lock. Truncation is deliberate: a plain `xid < xmin` *filter* would be - wrong, because `seq` order and `xid` order can differ, and it would hand out a later change while skipping an earlier - one it excluded. `xmin` is cluster-wide, so **any older open transaction anywhere in the database holds account pages - back until it ends, and an empty page can mean "caught up" or "held back"**; poll again rather than concluding - nothing is new. The row scan is still served by the `(account_id, seq)` index. This narrows the window for a second - writer but does not close it for every interleaving: one writer SDK per database remains the contract. -- **The deployment-wide feed** (no `account_id`) orders by `(xid, seq)` and returns only changes whose transaction is - older than every transaction still in flight, so a slow commit is never skipped. A long-running transaction delays - this feed; it never makes it lose a record. +- **Every feed orders by `(xid, seq)`**, account-scoped or deployment-wide (no `account_id`), and returns only changes + whose transaction is older than every transaction still in flight (`xid < pg_snapshot_xmin(pg_current_snapshot())`), + so a slow commit is never skipped. The order does not depend on the account lock, on the order a writer drew `seq` + in, or on which lock key a writer took: a change with a lower `seq` and a higher `xid` that commits late is returned + on a later page, after the cursor. The cost is that `xmin` is cluster-wide, so **any older open transaction anywhere + in the database holds every feed back until it ends, and an empty page can mean "caught up" or "held back"**; poll + again rather than concluding nothing is new, and keep transactions short. The account scan is served by the + `(account_id, xid, seq)` index that `create_schema` creates. One writer SDK per database remains the contract: a + second SDK can write rows the first SDK's lifecycle, retention and idempotency logic does not know about. +- **Account cursors carry `xid`** (`{"v":1,"account":...,"seq":...,"xid":...}`), the same bytes the JavaScript SDK + writes. Store the cursor string opaquely. Cursor `seq` and `xid` are range-checked, so a value the database would + reject raises `ValueError`, never a driver error. +- **Upgrading from account cursors that carry only `seq`.** Account feeds used to order by `seq` and truncate each + page before the first in-flight change. A cursor saved then has no `xid`. `changes_after` accepts it and maps it to + the position `(X, S)`, where `X` is the smaller of the oldest `xid` among the account's changes after `S` and the + reading snapshot's `xmin`. Every change after `S` has an `xid` at or above both (a change that has not committed yet + belongs to a transaction at or above `xmin`), so nothing is skipped. The cost is redelivery: changes the consumer + already read can come back (those with a lower `seq` and a higher `xid`, and, if a long transaction was open at that + moment, every change committed since it began), so treat redelivery as normal, as for any at-least-once consumer. + The returned cursor is in the new format even on an empty page, so a sink saves it with its next batch. A seq-only + cursor raises `CURSOR_EXPIRED` when **any** change with a higher `seq` was pruned: `pruned_through_seq` still records + the highest `seq` pruned, whatever its `xid`, for exactly this check, because a tuple compare alone would miss a + pruned change with a higher `seq` and a lower `xid`. A consumer registered before the upgrade has a NULL `cursor_xid` + and holds retention by `seq` until its next `save_feed_consumer_cursor`; saving a seq-only cursor registers the + mapped position. +- **Rolling out the `(xid, seq)` account feed.** `create_schema` applies `reporting_changes.sql`, which is idempotent: + it adds `pruned_through_xid` and `pruned_through_xid_seq` to `adcp_reporting_change_account_horizon`, replaces the + `adcp_reporting_feed_consumers` check constraint with `adcp_reporting_feed_consumers_cursor_scope` (an account + consumer may now carry an `xid`; a pre-upgrade account row keeps a NULL one), and creates + `adcp_reporting_changes_account_xid`. On a large `adcp_reporting_changes` table create the index first with + `CREATE INDEX CONCURRENTLY IF NOT EXISTS adcp_reporting_changes_account_xid ON adcp_reporting_changes + (account_id, xid, seq)`, because a plain `CREATE INDEX` blocks appends while it builds; the migration then skips it. + The old `(account_id, seq)` index stays, because it serves the seq-only mapping; drop it once no older host runs. + **Do not run an older host's change maintenance (`prune_changes`, `retire_expired_periods`) alongside the new + reader.** The older code holds an account consumer's changes by `seq` alone and records only `pruned_through_seq`, + so against a consumer holding new-format cursors it can prune a change the consumer has not read without advancing + the horizon the new reader checks. Stop the older hosts' maintenance, or the hosts, first. Rolling back after a + consumer saved a new-format account cursor is not supported (the older SDK rejects it): reset those consumers. - A cursor older than the retained history raises `ReportingChangeCursorExpiredError` (`code == "CURSOR_EXPIRED"`). Resynchronize from `list_current_revisions` and resume from the cursor returned with that read. - **A read with no cursor can raise `CURSOR_EXPIRED` too.** `changes_after` reads the prune horizon, reads the page, @@ -418,12 +596,15 @@ await store.changes_after(account_id="acct_a", cursor=page.cursor) The page is rejected instead. Retrying the read starts from the new, later horizon, which is the oldest retained change; a consumer that had been reading from a cursor resynchronizes from `list_current_revisions` as usual. Handle `CURSOR_EXPIRED` on every `changes_after` call, including a first read, by matching `error.code`. -- A registered consumer holds back `prune_changes` until it has passed those changes in its own feed order, up to - `max_feed_hold_days` (default 7) after its last update, so a dead consumer cannot block retention. Schedule - `prune_changes(change_retention_days=30)` next to the other maintenance. Cursors are the same bytes in both SDKs. +- A registered consumer holds back `prune_changes` and `retire_expired_periods` until it has passed those changes in + `(xid, seq)` order (an account consumer only within its account), up to `max_feed_hold_days` (default 7) after its + last update, so a dead consumer cannot block retention. `prune_changes` takes its victims in `(xid, seq)` order and + records a per-account `(xid, seq)` horizon, so a batch cut by its limit never leaves a retained change below the + horizon. Schedule `prune_changes(change_retention_days=30)` next to the other maintenance. Cursors are the same bytes + in both SDKs. - **Checkpoints are forward-only within a feed scope.** `save_feed_consumer_cursor` ignores a cursor behind the saved one, so a delayed or repeated acknowledgment cannot pull the consumer, or its retention hold, backward and make it - reprocess changes. Account consumers compare `seq`; deployment consumers compare `(xid, seq)`. The call returns + reprocess changes. Every consumer compares `(xid, seq)`. The call returns `True` when it wrote the checkpoint and `False` when it ignored it (behind the saved position, or equal with `refresh_unchanged=False`). An equal cursor by default rewrites the checkpoint and renews the hold, which is the heartbeat for an idle consumer. A change of scope (account to deployment, deployment to account, or between @@ -510,6 +691,7 @@ Tombstones live in `adcp_reporting_period_tombstones`, created with row storage | `row_storage` | `prune_expired_row_sets` | 20 periods | | `row_storage` or `change_feed` | `retire_expired_periods` | 20 periods | | `change_feed` | `prune_changes` | 10,000 rows | +| `orphan_collection` | `collect_orphaned_row_objects` | `limit` objects per binding per pass (default 1000), full listings spaced by `interval_seconds` | Each operation is bounded and resumable. A failing operation never stops the worker: the next interval retries it. Only the closed operation name and integer counters are logged, never an exception, provider message or identifier: @@ -517,11 +699,13 @@ Only the closed operation name and integer counters are logged, never an excepti - `REPORTING_MAINTENANCE_FAILED` (warning): the operation raised or reported failed items (a binding that cannot delete, for example). Read `causes` from the result for the stable per-item code. - `REPORTING_MAINTENANCE_PENDING` (info): work was left in place on purpose, with the counters `protected`, `held`, - `deferred` and `skipped_unconfigured`. A retirement that does nothing because every period is held shows up here + `deferred`, `skipped_unconfigured` and `unsupported` (object bindings whose provider cannot list). A retirement that does nothing because every period is held shows up here instead of looking like a healthy idle worker. - `REPORTING_MAINTENANCE_STALLED` (warning): work nobody is reaching, with `overdue` (intents expired for more than `overdue_after_seconds`, default 24 hours, and still unswept: a binding no host configures any more, or one that - keeps failing, so objects leak) and `stuck` (periods with five or more failed retirement attempts). + keeps failing, so objects leak), `stuck` (periods with five or more failed retirement attempts) and `fenced` + (object bindings the orphan collector refused to delete from because another database holds the namespace's claim; + see [the clone fence](#the-clone-fence)). The last pass's counters are also on `ReportingProductionSupport.maintenance_report`, per operation. `prune_expired_row_sets` is scheduled so row bytes are reclaimed even while a period's records stay; its only hold is diff --git a/packages/adcp-reporting-storage/README.md b/packages/adcp-reporting-storage/README.md index 7acce1576f..622ea8cfd2 100644 --- a/packages/adcp-reporting-storage/README.md +++ b/packages/adcp-reporting-storage/README.md @@ -72,7 +72,7 @@ Install `adcp>=10.0.0b7` with this release. | adcp-reporting-storage | storage contract | adcp | |---|---|---| -| 0.1.0b4 (SDK 9 line) | 1 | `>=9.0.0,<10` | +| 0.1.0b4+ (SDK 9 line) | 1 | `>=9.0.0,<10` | | 0.2.0b0 | 1 | `>=9.0.0,<11` (**do not pair with 10.0.0b4+**: 10.0.0b4 and b5 install beside it and fail, b7+ refuse it, [#1763](https://github.com/adcontextprotocol/adcp-client-python/issues/1763)) | | 0.2.0b1+ | 2 | `>=10.0.0b4,<11` | @@ -101,6 +101,13 @@ the upgrade. See [Deploying the upgrade](https://github.com/adcontextprotocol/adcp-client-python/blob/next/docs/reporting-ledger-migration.md#deploying-the-upgrade) for a Recreate or two-phase rollout. +A ledger written by 0.1.0b5 (the `(xid, seq)` account change feed, 255-character +row binding IDs, the orphan collector fence) upgrades the same way, with no +further migration: `create_schema()` re-applies the change-feed and row-storage +DDL when those options are on. See +[Upgrading a ledger that 0.1.0b5 wrote](https://github.com/adcontextprotocol/adcp-client-python/blob/next/docs/reporting-ledger-migration.md#upgrading-a-ledger-that-adcp-reporting-storage-010b5-wrote) +for the account horizon a 0.2.0b0–b2 interlude leaves behind. + ## Versioning 0.x: breaking changes may land in a minor release, each with a migration note in diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/__init__.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/__init__.py index 4333635418..1b8bad7eca 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/__init__.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/__init__.py @@ -61,6 +61,27 @@ from adcp_reporting_storage.ledger.delivery_pg import ( PgReportingReconciliationStore as PgReportingReconciliationStore, ) + from adcp_reporting_storage.ledger.orphan_collector import ( + ReportingOrphanBindingResultV1 as ReportingOrphanBindingResultV1, + ) + from adcp_reporting_storage.ledger.orphan_collector import ( + ReportingOrphanCandidateV1 as ReportingOrphanCandidateV1, + ) + from adcp_reporting_storage.ledger.orphan_collector import ( + ReportingOrphanCollectionOptionsV1 as ReportingOrphanCollectionOptionsV1, + ) + from adcp_reporting_storage.ledger.orphan_collector import ( + ReportingOrphanCollectionResultV1 as ReportingOrphanCollectionResultV1, + ) + from adcp_reporting_storage.ledger.orphan_collector import ( + ReportingOrphanCollectionStateV1 as ReportingOrphanCollectionStateV1, + ) + from adcp_reporting_storage.ledger.orphan_collector import ( + ReportingOrphanFenceTakeoverV1 as ReportingOrphanFenceTakeoverV1, + ) + from adcp_reporting_storage.ledger.orphan_fence import ( + ReportingOrphanFenceRefusalV1 as ReportingOrphanFenceRefusalV1, + ) from adcp_reporting_storage.ledger.period_retirement import ( ReportingPeriodRetirementV1 as ReportingPeriodRetirementV1, ) @@ -275,6 +296,22 @@ "ReportingRowBindingSelectionInputV1", ), "ReportingRowIntentSweepV1": ("ledger.row_retention", "ReportingRowIntentSweepV1"), + "ReportingOrphanBindingResultV1": ("ledger.orphan_collector", "ReportingOrphanBindingResultV1"), + "ReportingOrphanCandidateV1": ("ledger.orphan_collector", "ReportingOrphanCandidateV1"), + "ReportingOrphanCollectionOptionsV1": ( + "ledger.orphan_collector", + "ReportingOrphanCollectionOptionsV1", + ), + "ReportingOrphanCollectionResultV1": ( + "ledger.orphan_collector", + "ReportingOrphanCollectionResultV1", + ), + "ReportingOrphanCollectionStateV1": ( + "ledger.orphan_collector", + "ReportingOrphanCollectionStateV1", + ), + "ReportingOrphanFenceTakeoverV1": ("ledger.orphan_collector", "ReportingOrphanFenceTakeoverV1"), + "ReportingOrphanFenceRefusalV1": ("ledger.orphan_fence", "ReportingOrphanFenceRefusalV1"), "ReportingRowPruneV1": ("ledger.row_retention", "ReportingRowPruneV1"), "ReportingPeriodRetirementV1": ("ledger.period_retirement", "ReportingPeriodRetirementV1"), "ReportingRowMigrationBatchV1": ("ledger.row_migration", "ReportingRowMigrationBatchV1"), diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/change_feed.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/change_feed.py index ac1e7b6ca0..5df003e57d 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/change_feed.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/change_feed.py @@ -2,13 +2,17 @@ ``PgReportingLedgerStore(change_feed=True)`` appends one row to ``adcp_reporting_changes`` per committed obligation, revision, adjustment and retirement, in the transaction that -writes the record and after the account lock, so per-account sequence order is commit -order. The JavaScript SDK writes the same table, so a host reads one feed. - -* **Account feeds** order by ``seq``. -* **The deployment-wide feed** orders by ``(xid, seq)`` and returns only changes whose - transaction precedes every transaction still in flight, so a slow commit is never - skipped. A long-running transaction delays this feed; it never makes it lose a record. +writes the record. The JavaScript SDK writes the same table, so a host reads one feed. + +* **Every feed**, account-scoped or deployment-wide, orders by ``(xid, seq)`` and returns + only changes whose transaction precedes every transaction still in flight + (``xid < pg_snapshot_xmin(pg_current_snapshot())``), so a slow commit is never skipped. + The order does not depend on the account lock, on the order a writer drew ``seq`` in, or + on which lock key a writer took: a change with a lower ``seq`` and a higher ``xid`` that + commits late is returned on a later page, after the cursor. A long-running transaction + delays every feed; it never makes one lose a record. +* **Account cursors carry ``xid``.** A cursor from before that, which carries only ``seq``, + is accepted and mapped to a position that skips nothing (see :func:`changes_after`). * Cursors are opaque, versioned and stable across restarts, processes and SDKs. A cursor older than the retained history raises :class:`ReportingChangeCursorExpiredError`; the consumer then resynchronizes from ``list_current_revisions`` and resumes from the @@ -16,7 +20,8 @@ commits mid-read raises the same error (for a cursorless read too) instead of returning a page that skipped the pruned changes. * A named consumer registered with ``save_feed_consumer_cursor`` holds back change pruning - (and period retirement) until it has passed those changes, up to ``max_feed_hold_days`` + (and period retirement) until it has passed those changes in ``(xid, seq)`` order, up to + ``max_feed_hold_days`` after its last update, so a dead consumer cannot block retention indefinitely. Its checkpoint is forward-only within a feed scope; ``rewind=True`` replays deliberately and ``remove_feed_consumer`` unregisters it. @@ -35,7 +40,7 @@ from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Literal, NamedTuple if TYPE_CHECKING: from adcp_reporting_storage.ledger.pg import PgReportingLedgerStore @@ -49,6 +54,7 @@ "changes_after", "decode_reporting_change_cursor_v1", "encode_reporting_change_cursor_v1", + "is_legacy_account_change_cursor", "prune_changes", "remove_feed_consumer", "save_feed_consumer_cursor", @@ -93,11 +99,34 @@ def __init__(self) -> None: ) +_INT64_MAX = 2**63 - 1 +_UINT64_MAX = 2**64 - 1 + + +def _fits_unsigned(value: object, maximum: int) -> bool: + """A decimal string PostgreSQL's ``bigint`` / ``xid8`` cast would accept. + + A value it would reject must fail as ``ValueError`` here, not as a raw database error + mid-query. ASCII digits only: ``\\d`` also matches other scripts' digits, which ``int`` + accepts and the database cast does not. + """ + return ( + isinstance(value, str) + and re.fullmatch(r"[0-9]{1,20}", value) is not None + and int(value) <= maximum + ) + + def encode_reporting_change_cursor_v1( *, account: str | None, seq: str, xid: str | None = None ) -> str: + """Encode a cursor. Every cursor carries ``xid``; only a seq-only account cursor omits it. + + Omitting ``xid`` for an account is how a cursor from before the account feed ordered by + ``(xid, seq)`` is spelled; :func:`changes_after` maps one to a position that skips nothing. + """ cursor: dict[str, Any] = {"v": 1, "account": account, "seq": seq} - if account is None: + if account is None or xid is not None: cursor["xid"] = xid raw = json.dumps(cursor, separators=(",", ":"), ensure_ascii=False).encode("utf-8") return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") @@ -114,18 +143,26 @@ def decode_reporting_change_cursor_v1(value: str, account: str | None) -> dict[s not isinstance(parsed, dict) or parsed.get("v") != 1 or parsed.get("account") != account - or not isinstance(parsed.get("seq"), str) - or not re.fullmatch(r"\d{1,19}", parsed["seq"]) + or not _fits_unsigned(parsed.get("seq"), _INT64_MAX) or ( - not isinstance(parsed.get("xid"), str) or not re.fullmatch(r"\d{1,20}", parsed["xid"]) - if account is None - else "xid" in parsed + account is None + if "xid" not in parsed + else not _fits_unsigned(parsed["xid"], _UINT64_MAX) ) ): raise ValueError("change cursor does not belong to this feed") return parsed +def is_legacy_account_change_cursor(cursor: dict[str, Any]) -> bool: + """True for an account cursor issued before account feeds ordered by ``(xid, seq)``. + + It carries a ``seq`` and no ``xid``. :func:`changes_after` accepts it and maps it to a + position that cannot skip a change. + """ + return cursor.get("account") is not None and "xid" not in cursor + + def _positive(value: int, name: str, maximum: int) -> None: if type(value) is not int or not 1 <= value <= maximum: raise ValueError(f"{name} must be between 1 and {maximum}") @@ -136,8 +173,31 @@ def _require_feed(store: PgReportingLedgerStore) -> None: raise ValueError("this operation requires the store change_feed option") +def feed_consumer_behind_change(consumer: str, change: str) -> str: + """SQL condition: the registered consumer row has not yet passed the change row. + + In the consumer's own feed order: every consumer reads ``(xid, seq)`` order, an account + consumer only within its account. A row saved before account feeds carried an ``xid`` has + a NULL ``cursor_xid`` and holds by ``seq`` until the consumer saves its next cursor. + Shared by change pruning, period retirement and anything else that honours consumer holds. + """ + return ( + f"(({consumer}.account_id IS NULL OR {consumer}.account_id = {change}.account_id)" + f" AND CASE WHEN {consumer}.cursor_xid IS NULL" + f" THEN {change}.seq > {consumer}.cursor_seq" + f" ELSE ({change}.xid, {change}.seq) > ({consumer}.cursor_xid, {consumer}.cursor_seq)" + " END)" + ) + + def _page_query(*, account: bool) -> str: - """The change page query for an account feed (``seq`` order) or the deployment feed. + """The change page query for an account feed or the deployment feed. + + Both feeds order by ``(xid, seq)`` and stop below the reading snapshot's ``xmin``. Every + change with a higher ``xid`` than a returned one either committed before it or is not + returned yet, so a cursor never passes a change that commits late, with no assumption about + the lock a writer took or the order it drew ``seq`` in. The account scan is served by the + ``(account_id, xid, seq)`` index, in order. Every column is qualified with the table alias, in the projection and in ORDER BY. ``seq::text`` is *named* ``seq``, and PostgreSQL resolves a bare ORDER BY name to the output @@ -150,57 +210,84 @@ def _page_query(*, account: bool) -> str: " to_char(change.recorded_at AT TIME ZONE 'UTC'," ' \'YYYY-MM-DD"T"HH24:MI:SS.US"Z"\')' ) - if account: - return ( - f"SELECT {columns}," # noqa: S608 # nosec B608 - " change.xid >= pg_snapshot_xmin(pg_current_snapshot()) AS in_flight" - " FROM adcp_reporting_changes change" - " WHERE change.account_id = %s AND change.seq > %s::bigint" - " AND (%s::text[] IS NULL OR change.record_kind = ANY(%s::text[]))" - " ORDER BY change.seq LIMIT %s" - ) + scope = " change.account_id = %s AND" if account else "" return ( f"SELECT {columns} FROM adcp_reporting_changes change" # noqa: S608 # nosec B608 - " WHERE (change.xid, change.seq) > (%s::text::xid8, %s::bigint)" + f" WHERE{scope} (change.xid, change.seq) > (%s::text::xid8, %s::bigint)" " AND change.xid < pg_snapshot_xmin(pg_current_snapshot())" " AND (%s::text[] IS NULL OR change.record_kind = ANY(%s::text[]))" " ORDER BY change.xid, change.seq LIMIT %s" ) -async def _read_horizon(connection: Any, account_id: str | None) -> tuple[str, str]: - """The pruned horizon ``(xid, seq)`` in this feed's order; ``xid`` is ``"0"`` for accounts.""" +class _Horizon(NamedTuple): + xid: str + #: The ``seq`` of the highest ``(xid, seq)`` pruned. + seq: str + #: The highest ``seq`` pruned whatever its ``xid``: the exact check for a seq-only cursor. + max_seq: str + + +async def _read_horizon(connection: Any, account_id: str | None) -> _Horizon: + """The pruned horizon in this feed's own order.""" if account_id is not None: row = await ( await connection.execute( - "SELECT pruned_through_seq::text FROM adcp_reporting_change_account_horizon" + "SELECT pruned_through_xid::text, pruned_through_xid_seq::text," + " pruned_through_seq::text FROM adcp_reporting_change_account_horizon" " WHERE account_id = %s", (account_id,), ) ).fetchone() - return "0", (row[0] if row else "0") - row = await ( - await connection.execute( - "SELECT pruned_through_xid::text, pruned_through_seq::text" - " FROM adcp_reporting_change_horizon WHERE singleton" - ) - ).fetchone() - return (row[0], row[1]) if row else ("0", "0") + else: + row = await ( + await connection.execute( + "SELECT pruned_through_xid::text, pruned_through_seq::text," + " pruned_through_seq::text FROM adcp_reporting_change_horizon WHERE singleton" + ) + ).fetchone() + return _Horizon(*row) if row else _Horizon("0", "0", "0") + +def _assert_retained(position: tuple[str | None, str], horizon: _Horizon) -> None: + """Raise unless ``position`` ``(xid, seq)`` is at or past the pruned ``horizon``. -def _assert_retained( - account_id: str | None, position: tuple[str, str], horizon: tuple[str, str] -) -> None: - """Raise unless ``position`` ``(xid, seq)`` is at or past the pruned ``horizon``.""" + A position without an ``xid`` is a seq-only account cursor, checked against the highest + ``seq`` ever pruned: a tuple compare would miss a pruned change with a higher ``seq`` and + a lower ``xid``. + """ + xid, seq = position behind = ( - int(horizon[1]) > int(position[1]) - if account_id is not None - else (int(horizon[0]), int(horizon[1])) > (int(position[0]), int(position[1])) + int(horizon.max_seq) > int(seq) + if xid is None + else (int(horizon.xid), int(horizon.seq)) > (int(xid), int(seq)) ) if behind: raise ReportingChangeCursorExpiredError() +async def _legacy_account_cursor_xid(connection: Any, account_id: str, seq: str) -> str: + """Map a seq-only account cursor to a transaction id that cannot skip a change. + + Every change after the cursor's ``seq`` has an ``xid`` at or above both the oldest such + change that is visible now and the reading snapshot's ``xmin`` (a change that is not + visible yet belongs to a transaction that was in flight, or started after the snapshot, + whose id is at or above ``xmin``), so the position ``(xid, seq)`` below the smaller of the + two hands them all out. + """ + row = await ( + await connection.execute( + "SELECT LEAST(" + " pg_snapshot_xmin(pg_current_snapshot())," + " COALESCE((SELECT change.xid FROM adcp_reporting_changes change" + " WHERE change.account_id = %s AND change.seq > %s::bigint" + " ORDER BY change.xid LIMIT 1), pg_snapshot_xmin(pg_current_snapshot())))::text", + (account_id, seq), + ) + ).fetchone() + return str(row[0]) + + async def changes_after( store: PgReportingLedgerStore, *, @@ -211,6 +298,19 @@ async def changes_after( ) -> ReportingChangesPageV1: """Read the host change feed. Omit ``cursor`` to start at the oldest retained change. + Every feed, account-scoped or deployment-wide, orders by ``(xid, seq)`` and returns only + changes whose transaction precedes every transaction still in flight, so no commit is ever + skipped and the order does not depend on the account lock. An empty page can therefore + mean caught up or held back by an older open transaction (``xmin`` is cluster-wide). + + An account cursor issued before account feeds carried an ``xid`` holds only a ``seq``. It + is accepted and mapped to a position that cannot skip a change: the lowest transaction id + any change after its ``seq`` could have (the smaller of the oldest such change's ``xid`` + and the reading snapshot's ``xmin``). That can hand out a few changes the consumer already + read, those with a lower ``seq`` and a higher ``xid``, so consumers must treat redelivery + as normal. The page returns a cursor in the new format, even when it is empty. A seq-only + cursor raises ``CURSOR_EXPIRED`` when any change with a higher ``seq`` was pruned. + Raises :class:`ReportingChangeCursorExpiredError` (``code == "CURSOR_EXPIRED"``) when the starting position is older than the retained history. A read **without** a ``cursor`` can raise it too: the pruned horizon is read again after the page, so a prune that commits @@ -222,23 +322,31 @@ async def changes_after( if kinds is not None and any(kind not in _KINDS for kind in kinds): raise ValueError("unknown change kind") decoded = decode_reporting_change_cursor_v1(cursor, account_id) if cursor is not None else None + legacy = decoded is not None and is_legacy_account_change_cursor(decoded) selected = list(kinds) if kinds else None async with store._pool.connection() as connection: # The pruned horizon in this feed's own order. A supplied cursor below it may have # missed a pruned change and fails closed; a consumer with no cursor begins at it. - horizon_xid, horizon_seq = await _read_horizon(connection, account_id) + horizon = await _read_horizon(connection, account_id) + checked: tuple[str | None, str] | None = None if decoded is not None: - _assert_retained( - account_id, (decoded.get("xid", "0"), decoded["seq"]), (horizon_xid, horizon_seq) + checked = (decoded.get("xid"), decoded["seq"]) + _assert_retained(checked, horizon) + position_seq = decoded["seq"] + position_xid = ( + await _legacy_account_cursor_xid(connection, account_id, position_seq) + if legacy and account_id is not None + else decoded["xid"] ) - position_seq = decoded["seq"] if decoded else horizon_seq - position_xid = decoded.get("xid", "0") if decoded else horizon_xid - query = _page_query(account=account_id is not None) - if account_id is not None: - params: tuple[Any, ...] = (account_id, position_seq, selected, selected, limit) else: - params = (position_xid, position_seq, selected, selected, limit) - rows = await (await connection.execute(query, params)).fetchall() + position_xid, position_seq = horizon.xid, horizon.seq + query = _page_query(account=account_id is not None) + scope: tuple[Any, ...] = (account_id,) if account_id is not None else () + rows = await ( + await connection.execute( + query, (*scope, position_xid, position_seq, selected, selected, limit) + ) + ).fetchall() # Pruning can commit between the horizon read and the page read, and a prune deletes rows # and advances the horizon in one transaction. Read the horizon again, as a new statement # after the page, and compare it with the position this page started from: a page read @@ -246,31 +354,21 @@ async def changes_after( # skip them silently. This covers a read with no cursor too. Retrying starts from the # new, later horizon. A prune that commits after this read cannot have affected the page. _assert_retained( - account_id, (position_xid, position_seq), await _read_horizon(connection, account_id) + checked or (position_xid, position_seq), await _read_horizon(connection, account_id) ) - if account_id is not None: - # The account feed orders by ``seq``, which equals commit order only while every writer - # takes the account lock before ``nextval``. Defense in depth for a writer that does not: - # stop the page at the first change whose transaction is not below every in-flight - # transaction (``xid >= xmin``). A plain ``xid < xmin`` filter would not do, because seq - # order and xid order can differ, and it would hand out a later change while skipping an - # earlier one it excluded. Truncating keeps the page a prefix of seq order. This narrows - # the window for a second writer but does not close it for every interleaving: one - # writer SDK per database remains the contract. ``xmin`` is cluster-wide, so any older - # open transaction holds account pages back until it ends; an empty page can mean - # "caught up" or "held back". The row scan is still served by the (account_id, seq) index. - blocked = next((index for index, row in enumerate(rows) if row[7]), None) - if blocked is not None: - rows = rows[:blocked] records = tuple(ReportingChangeRecordV1(row[3], row[4], row[5], row[2], row[6]) for row in rows) if rows: last = rows[-1] next_cursor = encode_reporting_change_cursor_v1( - account=account_id, seq=last[0], xid=last[1] if account_id is None else None + account=account_id, seq=last[0], xid=last[1] ) + elif cursor is not None and not legacy: + next_cursor = cursor else: - next_cursor = cursor or encode_reporting_change_cursor_v1( - account=account_id, seq=position_seq, xid=position_xid if account_id is None else None + # A seq-only cursor is replaced by its mapped position even on an empty page, so the + # consumer saves a cursor in the new format. + next_cursor = encode_reporting_change_cursor_v1( + account=account_id, seq=position_seq, xid=position_xid ) return ReportingChangesPageV1(records, next_cursor) @@ -290,8 +388,8 @@ async def save_feed_consumer_cursor( Checkpoints are forward-only within a feed scope: a cursor behind the saved one is ignored, so a delayed or repeated acknowledgment cannot pull the retention hold back or - make the consumer reprocess changes. Account consumers compare ``seq``; deployment - consumers compare ``(xid, seq)``. Returns ``True`` when the checkpoint was written and + make the consumer reprocess changes. Every consumer compares ``(xid, seq)``. Returns + ``True`` when the checkpoint was written and ``False`` when it was ignored (behind the saved position, or equal with ``refresh_unchanged=False``). @@ -306,6 +404,12 @@ async def save_feed_consumer_cursor( ``refresh_unchanged=False`` recovers an acknowledgment without renewing the retention hold when its position has not advanced. + + A seq-only account cursor (from before account feeds carried an ``xid``) is mapped to the + position :func:`changes_after` would read it from, so the hold sits below every unread + change. A consumer row saved before that upgrade has a NULL ``cursor_xid`` and holds by + ``seq`` until its next save; a cursor in the new format replaces it unless its ``seq`` is + behind. """ _require_feed(store) if not isinstance(name, str) or not _CONSUMER_NAME.match(name): @@ -320,8 +424,16 @@ async def save_feed_consumer_cursor( decoded = decode_reporting_change_cursor_v1(cursor, account) # One atomic upsert: the WHERE of DO UPDATE is re-evaluated against the latest committed # row version under the row lock, so concurrent writers cannot interleave a stale position - # over a newer one. No row is returned when the update is skipped. + # over a newer one. No row is returned when the update is skipped. Forward-only in + # (xid, seq) order for every scope. A row saved before account feeds carried an xid (NULL + # cursor_xid) has only a seq to compare: a cursor in the new format replaces it unless its + # seq is behind. async with store._pool.connection() as connection: + xid = ( + await _legacy_account_cursor_xid(connection, account, decoded["seq"]) + if account is not None and is_legacy_account_change_cursor(decoded) + else decoded["xid"] + ) result = await connection.execute( "INSERT INTO adcp_reporting_feed_consumers" " (consumer_name, account_id, cursor_xid, cursor_seq)" @@ -331,22 +443,20 @@ async def save_feed_consumer_cursor( " cursor_seq = EXCLUDED.cursor_seq, updated_at = clock_timestamp()" " WHERE %(rewind)s::boolean" " OR adcp_reporting_feed_consumers.account_id IS DISTINCT FROM EXCLUDED.account_id" - " OR CASE WHEN EXCLUDED.account_id IS NULL" - " THEN (adcp_reporting_feed_consumers.cursor_xid," - " adcp_reporting_feed_consumers.cursor_seq)" - " < (EXCLUDED.cursor_xid, EXCLUDED.cursor_seq)" - " OR (%(refresh)s::boolean AND" - " (adcp_reporting_feed_consumers.cursor_xid," - " adcp_reporting_feed_consumers.cursor_seq)" - " = (EXCLUDED.cursor_xid, EXCLUDED.cursor_seq))" - " ELSE adcp_reporting_feed_consumers.cursor_seq < EXCLUDED.cursor_seq" - " OR (%(refresh)s::boolean" - " AND adcp_reporting_feed_consumers.cursor_seq = EXCLUDED.cursor_seq) END" + " OR (adcp_reporting_feed_consumers.cursor_xid IS NULL" + " AND adcp_reporting_feed_consumers.cursor_seq <= EXCLUDED.cursor_seq)" + " OR (adcp_reporting_feed_consumers.cursor_xid," + " adcp_reporting_feed_consumers.cursor_seq)" + " < (EXCLUDED.cursor_xid, EXCLUDED.cursor_seq)" + " OR (%(refresh)s::boolean AND" + " (adcp_reporting_feed_consumers.cursor_xid," + " adcp_reporting_feed_consumers.cursor_seq)" + " = (EXCLUDED.cursor_xid, EXCLUDED.cursor_seq))" " RETURNING consumer_name", { "name": name, "account": account, - "xid": decoded.get("xid") if account is None else None, + "xid": xid, "seq": decoded["seq"], "refresh": refresh_unchanged, "rewind": rewind, @@ -393,18 +503,16 @@ async def prune_changes( async with store._pool.connection() as connection, connection.transaction(): victims = await ( await connection.execute( - "WITH victims AS (" + "WITH victims AS (" # noqa: S608 # nosec B608 " SELECT change.seq FROM adcp_reporting_changes change" " WHERE change.recorded_at < clock_timestamp() - (%s::integer * INTERVAL '1 day')" " AND NOT EXISTS (SELECT 1 FROM adcp_reporting_feed_consumers consumer" " WHERE consumer.updated_at >" " clock_timestamp() - (%s::integer * INTERVAL '1 day')" - " AND ((consumer.account_id IS NULL" - " AND (change.xid, change.seq) >" - " (consumer.cursor_xid, consumer.cursor_seq))" - " OR (consumer.account_id = change.account_id" - " AND change.seq > consumer.cursor_seq)))" - " ORDER BY change.seq LIMIT %s)" + f" AND {feed_consumer_behind_change('consumer', 'change')})" + # Victims in (xid, seq) order, so a batch cut by the limit leaves nothing + # retained below the per-account (xid, seq) horizon recorded below. + " ORDER BY change.xid, change.seq LIMIT %s)" " DELETE FROM adcp_reporting_changes change USING victims" " WHERE change.seq = victims.seq" " RETURNING change.seq::text, change.xid::text, change.account_id", @@ -421,16 +529,38 @@ async def prune_changes( " (pruned_through_xid, pruned_through_seq)", (highest[1], highest[0], highest[1], highest[0]), ) - per_account: dict[str, int] = {} - for seq, _, account in victims: - per_account[account] = max(per_account.get(account, 0), int(seq)) - for account, seq in per_account.items(): + # Per account: the highest (xid, seq) pruned, which a cursor is compared with, and the + # highest seq pruned whatever its xid, which is the exact check for a seq-only cursor. + per_account: dict[str, tuple[int, int, int]] = {} + for seq_text, xid_text, account in victims: + seq, xid = int(seq_text), int(xid_text) + top_xid, top_seq, max_seq = per_account.get(account, (xid, seq, seq)) + if (xid, seq) > (top_xid, top_seq): + top_xid, top_seq = xid, seq + per_account[account] = (top_xid, top_seq, max(max_seq, seq)) + for account, (top_xid, top_seq, max_seq) in per_account.items(): await connection.execute( - "INSERT INTO adcp_reporting_change_account_horizon (account_id, pruned_through_seq)" - " VALUES (%s, %s::bigint) ON CONFLICT (account_id) DO UPDATE" - " SET pruned_through_seq = GREATEST(" - " adcp_reporting_change_account_horizon.pruned_through_seq," - " EXCLUDED.pruned_through_seq)", - (account, seq), + "INSERT INTO adcp_reporting_change_account_horizon AS horizon" + " (account_id, pruned_through_seq, pruned_through_xid, pruned_through_xid_seq)" + " VALUES (%(account)s, %(max_seq)s::bigint, %(xid)s::text::xid8," + " %(xid_seq)s::bigint)" + " ON CONFLICT (account_id) DO UPDATE" + " SET pruned_through_seq = GREATEST(horizon.pruned_through_seq," + " EXCLUDED.pruned_through_seq)," + " pruned_through_xid = CASE" + " WHEN (EXCLUDED.pruned_through_xid, EXCLUDED.pruned_through_xid_seq)" + " > (horizon.pruned_through_xid, horizon.pruned_through_xid_seq)" + " THEN EXCLUDED.pruned_through_xid ELSE horizon.pruned_through_xid END," + " pruned_through_xid_seq = CASE" + " WHEN (EXCLUDED.pruned_through_xid, EXCLUDED.pruned_through_xid_seq)" + " > (horizon.pruned_through_xid, horizon.pruned_through_xid_seq)" + " THEN EXCLUDED.pruned_through_xid_seq" + " ELSE horizon.pruned_through_xid_seq END", + { + "account": account, + "max_seq": str(max_seq), + "xid": str(top_xid), + "xid_seq": str(top_seq), + }, ) return len(victims) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/orphan_collector.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/orphan_collector.py new file mode 100644 index 0000000000..060fc645a4 --- /dev/null +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/orphan_collector.py @@ -0,0 +1,986 @@ +"""Opt-in collector for orphaned row objects, fenced against a cloned or restored database. + +Shared SDK persistence spec, adcontextprotocol/adcp#7996 section 4.2; mirrors the JavaScript SDK's +``collectOrphanedRowObjects`` (adcp-client#3291) and its ownership fence (adcp-client#3593). + +The write-intent sweep (:mod:`adcp_reporting_storage.ledger.row_retention`) deletes only what an +intent lists. It cannot see an object whose ``PUT`` succeeded before the intent recorded it, an +intent for a binding no running host configures, or an intent lost to a database restore. This +collector lists each configured object binding under its owned prefix, +``{prefix}/{namespace_key}/``, and deletes an object only when all of these hold: + +* its key is a well-formed row object (``.jsonl`` or ``.jsonl.gz``) under that prefix; +* it was written more than ``min_age_seconds`` ago (default 7 days, at least 1 day), so no writer + that still holds a lease can be mid-upload; +* no row set that is not already ``pruned`` has a chunk at its key, and no write intent lists it, + at any version and under any binding (a pre-check that errs towards leaking, because two bindings + can share a bucket and prefix); +* under the per-object advisory lock of **every** binding (taken only here and by a commit that + adopts an object its own intent did not record), no ``live`` or ``unavailable`` row set + under any binding references that exact version and no open intent created it (the check every + other delete makes, widened to every binding), and the provider then deletes that exact version. + +A clone or point-in-time restore of the ledger database shares the live database's installation +identity, namespace key and configuration, so it would also share the opt-in. A real pass therefore +first claims the namespace in an ownership fence +(:mod:`adcp_reporting_storage.ledger.orphan_fence`) and refuses to delete while another database +holds a newer claim. See ``docs/reporting-row-storage.md``. +""" + +from __future__ import annotations + +import asyncio +import math +import time +import uuid +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Any, Literal + +from adcp_reporting_storage.ledger.orphan_fence import ( + OrphanFenceDecision, + OrphanFenceEntry, + OrphanFenceRecord, + OrphanFenceRow, + ReportingOrphanFenceRefusalV1, + decide_orphan_fence, +) +from adcp_reporting_storage.ledger.row_storage import ReportingRowStoreError +from adcp_reporting_storage.ledger.row_storage_object import ( + format_row_instant, + parse_row_instant, +) + +if TYPE_CHECKING: + from adcp_reporting_storage.ledger.pg import PgReportingLedgerStore + +__all__ = [ + "ReportingOrphanBindingResultV1", + "ReportingOrphanCandidateV1", + "ReportingOrphanCollectionOptionsV1", + "ReportingOrphanCollectionResultV1", + "ReportingOrphanCollectionStateV1", + "ReportingOrphanFenceTakeoverV1", + "collect_orphaned_row_objects", + "run_orphan_collection_pass", + "take_over_orphan_collector_fence", +] + +DEFAULT_ORPHAN_MIN_AGE_SECONDS = 7 * 24 * 3600 +MIN_ORPHAN_MIN_AGE_SECONDS = 24 * 3600 +DEFAULT_ORPHAN_INTERVAL_SECONDS = 86_400 +#: A fenced namespace looks again within the hour, so applying the migration or taking the +#: namespace over does not leave collection idle for the rest of a day. +FENCED_RECHECK_SECONDS = 3_600 +MAX_ORPHAN_CANDIDATES = 1_000 +#: Deletes between reads of the collector fence within one listing page. +ORPHAN_FENCE_RECHECK_DELETES = 25 +_FENCE_LOCK_TIMEOUT = "120s" +#: The suffixes ``render_reporting_row_object_key_v1`` gives every key: nothing else under the +#: prefix is a row object. +_ROW_OBJECT_SUFFIXES = (".jsonl", ".jsonl.gz") + +_BindingStatus = Literal["complete", "partial", "failed", "unsupported", "read_only", "fenced"] + + +@dataclass(frozen=True, slots=True) +class ReportingOrphanCandidateV1: + """An object a dry run would delete.""" + + binding_id: str + object_ref: str + native_version_ref: str + size_bytes: int + created_at: str + + +@dataclass(slots=True) +class ReportingOrphanBindingResultV1: + """What one binding's listing did. + + ``status``: ``complete`` (the listing reached its end), ``partial`` (stopped at ``limit``; + resume from ``cursors``), ``failed`` (the listing or its provider failed; ``error`` is the + stable code), ``unsupported`` (the provider has no ``list``), ``read_only`` (skipped because + the binding is ``read_only``) or ``fenced`` (another database holds the namespace's collector + fence, or this database cannot prove it holds it, so no further object was deleted; ``deleted`` + counts any that went before a mid-pass refusal and ``fence`` says why). + """ + + binding_id: str + status: _BindingStatus = "complete" + #: Objects the provider listed, including ones dropped as foreign. + listed: int = 0 + #: Old enough, but named by a ledger record or open upload; left alone. + referenced: int = 0 + #: Younger than ``min_age_seconds``, or with no readable write time. + too_young: int = 0 + #: Outside the owned prefix or not shaped like a row object; never touched. + foreign: int = 0 + deleted: int = 0 + #: Dry run only: orphans that a real pass would delete. + would_delete: int = 0 + failed: int = 0 + error: str | None = None + fence: ReportingOrphanFenceRefusalV1 | None = None + + +@dataclass(slots=True) +class ReportingOrphanCollectionResultV1: + """A collection pass. ``deleted`` counts delete requests that found nothing referenced; an + object another host removed first still counts.""" + + dry_run: bool + listed: int = 0 + referenced: int = 0 + too_young: int = 0 + foreign: int = 0 + deleted: int = 0 + would_delete: int = 0 + #: Delete failures plus bindings whose listing failed. + failed: int = 0 + #: Bindings whose provider cannot list: not covered by this collector. + unsupported: tuple[str, ...] = () + #: ``read_only`` bindings skipped. + skipped_read_only: tuple[str, ...] = () + #: Bindings whose collector fence another database holds: no further object was deleted there. + #: See each binding's ``fence``. + fenced: tuple[str, ...] = () + #: Dry run only: the orphans a real pass would delete, at most 1000. + candidates: tuple[ReportingOrphanCandidateV1, ...] = () + #: Resume tokens for bindings whose listing is not finished. Pass back as ``cursors``. + cursors: Mapping[str, str] = field(default_factory=dict) + #: Bindings whose listing reached its end this pass. Pass them back as ``skip_bindings`` until + #: every binding has. + completed: tuple[str, ...] = () + bindings: tuple[ReportingOrphanBindingResultV1, ...] = () + + +@dataclass(frozen=True, slots=True) +class ReportingOrphanFenceTakeoverV1: + """One fence a takeover claimed.""" + + #: The bindings that share the fence. + binding_ids: tuple[str, ...] + #: The epoch now held. + epoch: int + #: Highest epoch found in the object store before the takeover; ``None`` when the namespace + #: was never claimed. + previous_epoch: int | None + + +@dataclass(frozen=True, slots=True) +class ReportingOrphanCollectionOptionsV1: + """Opt-in scheduling of :func:`collect_orphaned_row_objects` (``PgReportingLedgerStore( + orphan_collection=...)``). + + Enable it only on the ledger database that owns the namespace. A clone or restore of that + database pointed at the same bucket shares its namespace, so a real pass is fenced: it claims + the namespace and refuses, reporting the binding ``fenced``, when the object store holds a newer + claim than this database recorded. After a restore, or when the original is gone, call + ``take_over_orphan_collector_fence``. Start with ``dry_run=True`` and read the candidates. + """ + + #: Report what would be deleted and delete nothing. + dry_run: bool = False + #: Required unless ``dry_run``: your statement that this ledger database is the only one that + #: owns the namespace, and not a clone or restore pointed at the same bucket. A copy of the + #: configuration carries this flag too; the ownership fence is what stops such a copy. + acknowledge_exclusive_namespace: bool = False + #: Minimum time between full listings, counted from when the previous one finished. Passes in + #: between do nothing; a listing that has not finished (it stops at ``limit``) continues on + #: every pass. + interval_seconds: int = DEFAULT_ORPHAN_INTERVAL_SECONDS + #: Objects younger than this are never deleted. At least 1 day; keep it well above the longest + #: write deadline plus intent expiry. + min_age_seconds: int = DEFAULT_ORPHAN_MIN_AGE_SECONDS + #: Objects examined per binding per pass. + limit: int = 1_000 + #: Also collect from ``read_only`` bindings, which are skipped by default. + include_read_only: bool = False + + def __post_init__(self) -> None: + for flag in ("dry_run", "acknowledge_exclusive_namespace", "include_read_only"): + if type(getattr(self, flag)) is not bool: + raise ValueError(f"orphan_collection.{flag} must be a boolean") + if not self.dry_run and not self.acknowledge_exclusive_namespace: + raise ValueError( + "orphan_collection deletes objects: set acknowledge_exclusive_namespace=True to " + "confirm this ledger database is the only one that owns the namespace (not a " + "clone or restore pointed at the same bucket), or set dry_run=True" + ) + for name in ("interval_seconds", "limit"): + value = getattr(self, name) + if type(value) is not int or value < 1: + raise ValueError(f"orphan_collection.{name} must be a positive integer") + if ( + type(self.min_age_seconds) is not int + or self.min_age_seconds < MIN_ORPHAN_MIN_AGE_SECONDS + ): + raise ValueError( + "orphan_collection.min_age_seconds must be an integer of at least 86400 (1 day): " + "a younger object may belong to an upload still in flight" + ) + + +@dataclass(slots=True) +class ReportingOrphanCollectionStateV1: + """Progress of the collector between passes, kept in the process. + + It is not persisted, so a restart begins a new listing. + """ + + cursors: dict[str, str] = field(default_factory=dict) + #: Bindings whose listing finished in the current cycle; not listed again until every binding + #: has. + completed: list[str] = field(default_factory=list) + #: ``time.monotonic()`` seconds before which no new full listing starts. + next_cycle_at: float | None = None + + +@dataclass(frozen=True, slots=True) +class _Gate: + refusal: ReportingOrphanFenceRefusalV1 | None = None + + @property + def ok(self) -> bool: + return self.refusal is None + + +# -- the collector --------------------------------------------------------------------------- + + +def _positive(value: int, name: str, maximum: int = 1_000_000) -> None: + if type(value) is not int or not 1 <= value <= maximum: + raise ValueError(f"{name} must be between 1 and {maximum}") + + +def _names(value: Sequence[str] | None, name: str) -> tuple[str, ...] | None: + """Binding IDs as a tuple. A bare string would otherwise act as a substring test.""" + if value is None: + return None + if isinstance(value, (str, bytes)) or not all(isinstance(item, str) for item in value): + raise ValueError(f"{name} must be a sequence of binding IDs") + return tuple(value) + + +def _code(error: Exception) -> str: + return error.code if isinstance(error, ReportingRowStoreError) else "INTERNAL_ERROR" + + +async def _present_tables(store: PgReportingLedgerStore) -> set[str]: + async with store._pool.connection() as connection: + rows = await ( + await connection.execute( + "SELECT table_name FROM information_schema.tables" + " WHERE table_schema = current_schema() AND table_name = ANY(%s::text[])", + ( + [ + "adcp_reporting_row_sets", + "adcp_reporting_row_write_intents", + "adcp_reporting_orphan_collector_fences", + ], + ), + ) + ).fetchall() + return {row[0] for row in rows} + + +async def _keys_named_by_records(store: PgReportingLedgerStore, keys: Sequence[str]) -> set[str]: + """Of these object keys, the ones a ledger record names at any version, under any binding. + + A chunk of a row set that is not yet ``pruned``, or an object any write intent lists. Version- + and binding-blind on purpose: two bindings can share a bucket and prefix (a retired binding kept + ``read_only`` next to its successor), and the orphan collector would rather leak than delete + near a record. + """ + if not keys: + return set() + async with store._pool.connection() as connection: + rows = await ( + await connection.execute( + "SELECT chunk.object_ref FROM adcp_reporting_row_chunks chunk" + " JOIN adcp_reporting_row_sets row_set ON row_set.row_set_id = chunk.row_set_id" + " WHERE chunk.object_ref = ANY(%(keys)s::text[]) AND row_set.rows_state <> 'pruned'" + " UNION" + " SELECT COALESCE(item ->> 'object_ref', item ->> 'objectKey')" + " FROM adcp_reporting_row_write_intents intent," + " jsonb_array_elements(intent.created_objects) item" + " WHERE COALESCE(item ->> 'object_ref', item ->> 'objectKey')" + " = ANY(%(keys)s::text[])", + {"keys": list(keys)}, + ) + ).fetchall() + return {row[0] for row in rows} + + +async def collect_orphaned_row_objects( + store: PgReportingLedgerStore, + *, + min_age_seconds: int | None = None, + limit: int = 1_000, + page_size: int | None = None, + dry_run: bool = False, + include_read_only: bool = False, + bindings: Sequence[str] | None = None, + cursors: Mapping[str, str] | None = None, + skip_bindings: Sequence[str] | None = None, + acknowledge_exclusive_namespace: bool = False, + now: datetime | None = None, +) -> ReportingOrphanCollectionResultV1: + """Delete row objects that no ledger record or open upload names. + + Opt-in, listing-based and read-only unless it deletes. See the module docstring for the full + list of conditions an object must meet. ``read_only`` bindings are skipped unless + ``include_read_only``. A binding whose provider has no ``list`` is reported in + ``unsupported``. ``dry_run`` deletes nothing and reports what would go in ``candidates``. + ``limit`` bounds the objects examined per binding per pass; pass ``result.cursors`` back as + ``cursors`` to resume where the last pass stopped (a binding absent from ``cursors`` starts from + the beginning, which is also where a binding that finished its listing starts next time), and + ``result.completed`` as ``skip_bindings`` so a cycle that spans several passes does not rescan + bindings that already finished. + + Deleting needs ``acknowledge_exclusive_namespace=True``: enable it only on the ledger database + that owns the namespace. A clone or restore of that database that still points at the same + bucket shares its ``namespace_key``, and would delete the objects of every row set the original + wrote after the clone was taken. That flag is copied with the configuration, so it is backed by + a fence: a real pass first claims the namespace (a create-only object under + ``{prefix}/{namespace_key}/_collector-fence/`` and a row in + ``adcp_reporting_orphan_collector_fences``) and deletes only while the object store's highest + claim is exactly the one this database recorded. A database copied before the original's last + pass sees a newer claim and refuses: its binding is reported ``fenced`` with a ``cause``, and + nothing more is deleted (a fence that advances mid-pass stops the pass at the next check, within + 25 deletes, and ``deleted`` keeps what went before). A ``dry_run`` checks the fence without + claiming it and reports fenced bindings the same way. After a restore, or when the original is + gone, take the namespace over deliberately with ``take_over_orphan_collector_fence``. Everything + the listing finds under the owned prefix with a row-object key is treated as written by this + SDK, so do not store other ``.jsonl`` files there. + """ + storage = store._row_storage + if storage is None: + raise ReportingRowStoreError( + "INVALID_INPUT", "collect_orphaned_row_objects requires row storage" + ) + bindings = _names(bindings, "bindings") + skip_bindings = _names(skip_bindings, "skip_bindings") + if min_age_seconds is None: + min_age_seconds = DEFAULT_ORPHAN_MIN_AGE_SECONDS + if type(min_age_seconds) is not int or min_age_seconds < MIN_ORPHAN_MIN_AGE_SECONDS: + raise ValueError( + f"min_age_seconds must be an integer of at least {MIN_ORPHAN_MIN_AGE_SECONDS} " + "(1 day): an object younger than the longest write deadline plus intent expiry may " + "belong to an upload still in flight" + ) + _positive(limit, "limit") + page_size = min(limit, 500) if page_size is None else page_size + _positive(page_size, "page_size") + if dry_run is not True: + dry_run = False + if not dry_run and acknowledge_exclusive_namespace is not True: + raise ValueError( + "collect_orphaned_row_objects deletes objects: pass " + "acknowledge_exclusive_namespace=True to confirm this ledger database is the only one " + "that owns the namespace (not a clone or restore pointed at the same bucket), or pass " + "dry_run=True" + ) + clock = now if now is not None else datetime.now(timezone.utc) + if not isinstance(clock, datetime) or clock.tzinfo is None: + raise ValueError("now must be a timezone-aware datetime") + await store.ready_row_storage() + summaries = storage.object_binding_summaries() + for binding_id in bindings or (): + if not any(summary.id == binding_id for summary in summaries): + raise ValueError( + f"bindings names {binding_id}, which is not a configured object binding" + ) + + result = ReportingOrphanCollectionResultV1(dry_run=dry_run) + outcomes: list[ReportingOrphanBindingResultV1] = [] + candidates: list[ReportingOrphanCandidateV1] = [] + completed: list[str] = [] + unsupported: list[str] = [] + skipped_read_only: list[str] = [] + fenced: list[str] = [] + next_cursors: dict[str, str] = {} + + def finish() -> ReportingOrphanCollectionResultV1: + for outcome in outcomes: + result.listed += outcome.listed + result.referenced += outcome.referenced + result.too_young += outcome.too_young + result.foreign += outcome.foreign + result.deleted += outcome.deleted + result.would_delete += outcome.would_delete + result.failed += outcome.failed + result.unsupported = tuple(unsupported) + result.skipped_read_only = tuple(skipped_read_only) + result.fenced = tuple(fenced) + result.candidates = tuple(candidates) + result.cursors = next_cursors + result.completed = tuple(completed) + result.bindings = tuple(outcomes) + return result + + tables = await _present_tables(store) + if "adcp_reporting_row_sets" not in tables or "adcp_reporting_row_write_intents" not in tables: + return finish() + has_fence_table = "adcp_reporting_orphan_collector_fences" in tables + # One fence check or claim per physical fence per pass: bindings that share a bucket and prefix + # share it. + gates: dict[str, _Gate] = {} + + async def gate(binding_id: str, mode: Literal["claim", "check"]) -> _Gate: + scope, _, _ = await storage.collector_fence_location(binding_id) + if mode == "claim" and scope in gates: + return gates[scope] + outcome = await _orphan_fence_gate( + store, + binding_id, + scope, + "check" if mode == "claim" and dry_run else mode, + has_fence_table, + ) + if mode == "claim": + gates[scope] = outcome + return outcome + + cutoff = clock - timedelta(seconds=min_age_seconds) + for summary in summaries: + if bindings is not None and summary.id not in bindings: + continue + if skip_bindings is not None and summary.id in skip_bindings: + continue + outcome = ReportingOrphanBindingResultV1(summary.id) + outcomes.append(outcome) + if summary.read_only and not include_read_only: + outcome.status = "read_only" + skipped_read_only.append(summary.id) + continue + if not summary.listable: + outcome.status = "unsupported" + unsupported.append(summary.id) + continue + token = cursors.get(summary.id) if cursors is not None else None + # The ownership fence comes before any listing: a clone or restore must not even enumerate + # for deletion. + try: + claim = await gate(summary.id, "claim") + except Exception as error: + outcome.status = "failed" + outcome.error = _code(error) + outcome.failed += 1 + # The listing never started, so the position it had stays where it was. + if token is not None: + next_cursors[summary.id] = token + continue + if not claim.ok: + outcome.status = "fenced" + outcome.fence = claim.refusal + fenced.append(summary.id) + continue + # A failure on the first page of a resumed listing drops the cursor: a stale or invalid + # token must not wedge the binding, and restarting from the beginning is always safe. + resumed = token is not None + drop_cursor = False + examined = 0 + pages = 0 + # Pages that carry no objects (S3 delete markers, unversioned entries) do not move + # ``examined``, so the pass also stops after a bounded number of requests. + max_pages = math.ceil(limit / page_size) * 4 + try: + while True: + page = await storage.list_objects( + summary.id, page_size=min(page_size, limit - examined), page_token=token + ) + resumed = False + pages += 1 + examined += len(page.objects) + page.foreign + outcome.listed += len(page.objects) + page.foreign + outcome.foreign += page.foreign + old = [] + for entry in page.objects: + if not entry.object_ref.endswith(_ROW_OBJECT_SUFFIXES): + outcome.foreign += 1 + continue + written = parse_row_instant(entry.created_at) + # An unreadable write time is treated as young: never delete on a guess. + if written is None or written > cutoff: + outcome.too_young += 1 + continue + old.append(entry) + named = await _keys_named_by_records(store, [entry.object_ref for entry in old]) + # Another database may claim the namespace while this pass runs: read the fence + # again before the first delete of every page and then every + # ORPHAN_FENCE_RECHECK_DELETES deletes after it. + since_fence_check = ORPHAN_FENCE_RECHECK_DELETES + fenced_mid_pass = False + for entry in old: + if entry.object_ref in named: + outcome.referenced += 1 + continue + if dry_run: + outcome.would_delete += 1 + if len(candidates) < MAX_ORPHAN_CANDIDATES: + candidates.append( + ReportingOrphanCandidateV1( + summary.id, + entry.object_ref, + entry.native_version_ref, + entry.size_bytes, + entry.created_at, + ) + ) + continue + if since_fence_check >= ORPHAN_FENCE_RECHECK_DELETES: + again = await gate(summary.id, "check") + if not again.ok: + outcome.status = "fenced" + outcome.fence = again.refusal + fenced.append(summary.id) + fenced_mid_pass = True + break + since_fence_check = 0 + since_fence_check += 1 + try: + # The only delete that takes the per-object lock: no record names this + # object, so a commit may be adopting it unrecorded. The reference check + # across every binding runs under the lock of every binding. + removed = await storage.delete_orphaned_object( + store._pool, summary.id, entry.object_ref, entry.native_version_ref + ) + if removed: + outcome.deleted += 1 + else: + outcome.referenced += 1 + except Exception: + outcome.failed += 1 + if fenced_mid_pass: + break + if page.next_page_token is not None and page.next_page_token == token: + drop_cursor = True + raise ReportingRowStoreError( + "PROVIDER_UNAVAILABLE", "provider returned the same page token twice" + ) + token = page.next_page_token + if token is None: + completed.append(summary.id) + break + if examined >= limit or pages >= max_pages: + outcome.status = "partial" + next_cursors[summary.id] = token + break + except Exception as error: + outcome.status = "failed" + outcome.error = _code(error) + outcome.failed += 1 + # A listing failure retries the same page next pass. (A delete that fails is counted + # and the pass moves on, so one stuck object cannot hold up the rest; it is retried on + # the next full listing.) + if token is not None and not resumed and not drop_cursor: + next_cursors[summary.id] = token + return finish() + + +# -- the fence: claim, check, take over ----------------------------------------------------- + + +async def _fence_row(connection: Any, scope: str) -> tuple[OrphanFenceRow, str] | None: + row = await ( + await connection.execute( + "SELECT epoch, token, previous_token, claimed_at" + " FROM adcp_reporting_orphan_collector_fences WHERE fence_scope = %s", + (scope,), + ) + ).fetchone() + if row is None: + return None + return OrphanFenceRow(int(row[0]), row[1], row[2]), format_row_instant(row[3]) + + +async def _latest_orphan_fence( + store: PgReportingLedgerStore, binding_id: str +) -> tuple[list[OrphanFenceEntry], tuple[int, str] | None]: + """The object store's current claim: the highest epoch listed, read and verified.""" + storage = store._row_storage + assert storage is not None + entries = await storage.list_collector_fence(binding_id) + if not entries: + return entries, None + record = await storage.read_collector_fence(binding_id, entries[-1]) + return entries, (record.epoch, record.token) + + +def _unreadable() -> _Gate: + return _Gate( + ReportingOrphanFenceRefusalV1( + "fence_unreadable", + "the collector fence objects under this namespace are malformed, oversized, " + "duplicated or name another namespace. Inspect them; once the namespace is known to " + "be owned by this database alone, call " + "take_over_orphan_collector_fence(acknowledge_previous_owner_gone=True).", + ) + ) + + +async def _orphan_fence_gate( + store: PgReportingLedgerStore, + binding_id: str, + scope: str, + mode: Literal["claim", "check"], + fences_table: bool, +) -> _Gate: + """Whether this database may collect orphans under a binding's namespace. + + For a real pass this is also the claim that makes it so. ``check`` only reads: it is what + ``dry_run`` and the per-page recheck use. ``claim`` runs under a per-fence advisory lock: it + verifies the object store's highest epoch is exactly the claim this database recorded, records + the next epoch and a fresh token in the database, and only then writes that epoch's object + create-only. The intent is committed before the object is written, so a crash between the two + is finished by the next pass instead of looking like another database's claim. + + Each real pass claims a new epoch, so a cycle spread over many partial passes writes and prunes + one tiny object per pass. The claim holds a pool connection and the advisory lock across the + object-store listing and read, so a slow bucket pins one connection for up to the read deadline. + """ + if not fences_table: + return _Gate( + ReportingOrphanFenceRefusalV1( + "migration_required", + "the orphan collector fence table does not exist: apply " + "reporting_row_storage.sql (create_schema does), then orphan collection can " + "claim its namespace", + ) + ) + storage = store._row_storage + assert storage is not None + try: + if mode == "check": + # The bucket first, then the row. A claim commits its intent to the row before it + # writes the object, so a row read after the bucket is never behind a claim of this + # database that the bucket already shows (a sibling host claiming between the two + # reads would otherwise look like another database's claim). + _, latest = await _latest_orphan_fence(store, binding_id) + async with store._pool.connection() as connection: + stored = await _fence_row(connection, scope) + decision = decide_orphan_fence(stored[0] if stored else None, latest) + if decision.action == "fenced": + return _Gate(decision.refusal) + return _Gate() + + async def plan(stored: tuple[OrphanFenceRow, str] | None) -> _Plan: + entries, latest = await _latest_orphan_fence(store, binding_id) + decision = decide_orphan_fence(stored[0] if stored else None, latest) + if decision.action == "fenced": + return _Plan(refusal=decision.refusal) + # An unfinished claim is finished under a fresh token, never the recorded one: a copy + # of this database taken while the claim was unfinished carries the same token, and + # both could then "adopt" the same object. With distinct tokens, create-only leaves + # exactly one of them the owner. + completing = decision.action == "complete" + assert stored is not None or not completing + return _Plan( + record=OrphanFenceRecord( + stored[0].epoch if completing and stored else _epoch(decision), + str(uuid.uuid4()), + stored[0].previous_token if completing and stored else decision.previous_token, + format_row_instant(datetime.now(timezone.utc)), + ), + entries=entries, + previous_epoch=latest[0] if latest else None, + ) + + claimed = await _claim_orphan_fence(store, binding_id, scope, True, plan) + if claimed.kind == "fenced": + return _Gate(claimed.refusal) + if claimed.kind == "lost": + assert claimed.record is not None and claimed.found is not None + return _Gate( + ReportingOrphanFenceRefusalV1( + "claim_lost", + f"another database claimed epoch {claimed.record.epoch} first. If this " + "database is the only one that owns this namespace, call " + "take_over_orphan_collector_fence(acknowledge_previous_owner_gone=True); " + "otherwise leave orphan collection off on this copy.", + claimed.found.epoch, + claimed.record.epoch, + ) + ) + return _Gate() + except ReportingRowStoreError as error: + # Fence objects that are not exactly what this collector writes fail closed. + if error.code == "ROWS_INTEGRITY_FAILED": + return _unreadable() + raise + + +def _epoch(decision: OrphanFenceDecision) -> int: + assert decision.epoch is not None + return decision.epoch + + +@dataclass(slots=True) +class _Plan: + refusal: ReportingOrphanFenceRefusalV1 | None = None + record: OrphanFenceRecord | None = None + entries: list[OrphanFenceEntry] = field(default_factory=list) + previous_epoch: int | None = None + + +@dataclass(slots=True) +class _Claim: + kind: Literal["fenced", "lost", "claimed"] + refusal: ReportingOrphanFenceRefusalV1 | None = None + record: OrphanFenceRecord | None = None + found: OrphanFenceRecord | None = None + previous_epoch: int | None = None + + +async def _claim_orphan_fence( + store: PgReportingLedgerStore, + binding_id: str, + scope: str, + keep_predecessor: bool, + plan: Callable[[tuple[OrphanFenceRow, str] | None], Awaitable[_Plan]], +) -> _Claim: + """Claim one fence epoch for this database, under a session-level advisory lock held from the + decision to the object write. + + ``plan`` runs inside the transaction with the stored row, reads the bucket and returns either a + refusal or the claim to make (and the entries it supersedes). The claim's intent (epoch, token) + is committed before the object is written, so a crash between the two is finished by the next + pass instead of looking like another database's claim; holding the lock through the write keeps + a sibling host of this database from finishing that claim under another token while the first + write is still in flight. A clone has its own database and its own lock, which is exactly why + the object store's create-only write decides between them. + """ + storage = store._row_storage + assert storage is not None + lock = f"adcp.orphan_fence:{scope}" + async with store._pool.connection() as connection: + original_autocommit = connection.autocommit + await connection.set_autocommit(True) + locked = False + try: + # A sibling holds the lock across its object-store write, which has its own deadline. + await connection.execute(f"SET lock_timeout = '{_FENCE_LOCK_TIMEOUT}'") + # Marked before the call: a cancellation after the server grants the lock but before + # the client reads the reply must still unlock (unlocking a lock never taken only + # returns false), or the connection would go back to the pool holding it. + locked = True + await connection.execute("SELECT pg_advisory_lock(hashtextextended(%s, 0))", (lock,)) + await connection.execute("RESET lock_timeout") + async with connection.transaction(): + planned = await plan(await _fence_row(connection, scope)) + if planned.record is not None: + await connection.execute( + "INSERT INTO adcp_reporting_orphan_collector_fences" + " (fence_scope, epoch, token, previous_token, claimed_at)" + " VALUES (%(scope)s, %(epoch)s, %(token)s, %(previous)s, %(claimed)s)" + " ON CONFLICT (fence_scope) DO UPDATE" + " SET epoch = EXCLUDED.epoch, token = EXCLUDED.token," + " previous_token = EXCLUDED.previous_token," + " claimed_at = EXCLUDED.claimed_at", + { + "scope": scope, + "epoch": planned.record.epoch, + "token": planned.record.token, + "previous": planned.record.previous_token, + "claimed": parse_row_instant(planned.record.claimed_at), + }, + ) + if planned.refusal is not None: + return _Claim("fenced", refusal=planned.refusal) + assert planned.record is not None + _, written = await storage.put_collector_fence(binding_id, planned.record) + if written.token != planned.record.token: + return _Claim("lost", record=planned.record, found=written) + # A claim keeps its predecessor (an unfinished claim is recognised by it) and prunes + # everything older. A takeover has just written and confirmed its object, so it + # supersedes every object and version it saw. + await storage.delete_collector_fence( + binding_id, + [ + entry + for entry in planned.entries + if entry.epoch < planned.record.epoch - (1 if keep_predecessor else 0) + ], + ) + return _Claim("claimed", record=planned.record, previous_epoch=planned.previous_epoch) + finally: + try: + await connection.execute("RESET lock_timeout") + if locked: + await connection.execute( + "SELECT pg_advisory_unlock(hashtextextended(%s, 0))", (lock,) + ) + await connection.set_autocommit(original_autocommit) + except BaseException: + # A connection that may still hold the session lock must not go back to the pool. + await asyncio.shield(connection.close()) + raise + + +async def take_over_orphan_collector_fence( + store: PgReportingLedgerStore, + *, + acknowledge_previous_owner_gone: bool, + bindings: Sequence[str] | None = None, + include_read_only: bool = False, +) -> list[ReportingOrphanFenceTakeoverV1]: + """Take over the orphan collector's namespace fence. + + The deliberate step after a restore, or when the database that held the fence is gone for + good. It claims the next epoch for this database, so the previous holder (and every other copy + of its state) is fenced out, and this database's collector can delete again. + + Do this only when this database is the one that should own the namespace. If the original is + still running, taking over from a clone fences the original and lets the clone delete its + objects: the acknowledgement exists because nothing else can tell them apart. + ``acknowledge_previous_owner_gone=True`` is required. + + A takeover does not stop a collector that is already mid-pass: that pass notices the new epoch + at its next fence check, which is at most 25 deletes later. Stop the previous owner's + collector before you take over. + + Takes over every writable (or, with ``include_read_only``, every) object binding whose provider + can list, or only the ``bindings`` named; bindings that share a bucket, prefix and installation + share one fence and are reported together. Requires the fence table + (``reporting_row_storage.sql``). + """ + if acknowledge_previous_owner_gone is not True: + raise ValueError( + "take_over_orphan_collector_fence fences out the namespace's current owner: pass " + "acknowledge_previous_owner_gone=True to confirm that database is gone or must never " + "collect again" + ) + storage = store._row_storage + if storage is None: + raise ReportingRowStoreError( + "INVALID_INPUT", "take_over_orphan_collector_fence requires row storage" + ) + bindings = _names(bindings, "bindings") + await store.ready_row_storage() + summaries = storage.object_binding_summaries() + for binding_id in bindings or (): + summary = next((item for item in summaries if item.id == binding_id), None) + if summary is None: + raise ValueError( + f"bindings names {binding_id}, which is not a configured object binding" + ) + if not summary.listable: + raise ValueError(f"bindings names {binding_id}, whose provider cannot list objects") + if "adcp_reporting_orphan_collector_fences" not in await _present_tables(store): + raise ReportingRowStoreError( + "STATE_UNAVAILABLE", + "the orphan collector fence table does not exist: apply reporting_row_storage.sql", + ) + selected = [ + summary + for summary in summaries + if summary.listable + and ( + summary.id in bindings + if bindings is not None + else include_read_only or not summary.read_only + ) + ] + by_scope: dict[str, list[str]] = {} + for summary in selected: + scope, _, _ = await storage.collector_fence_location(summary.id) + by_scope.setdefault(scope, []).append(summary.id) + taken: list[ReportingOrphanFenceTakeoverV1] = [] + for scope, binding_ids in by_scope.items(): + binding_id = binding_ids[0] + + async def plan( + stored: tuple[OrphanFenceRow, str] | None, binding_id: str = binding_id + ) -> _Plan: + entries = await storage.list_collector_fence(binding_id, tolerant=True) + newest = entries[-1] if entries else None + # A malformed newest object is exactly what a takeover is for: its key still gives the + # epoch to supersede. Several versions of the newest epoch are ambiguous, so none of + # them names the previous token. + previous: OrphanFenceRecord | None = None + if newest is not None and sum(1 for item in entries if item.epoch == newest.epoch) == 1: + try: + previous = await storage.read_collector_fence(binding_id, newest) + except ReportingRowStoreError as error: + if error.code != "ROWS_INTEGRITY_FAILED": + raise + return _Plan( + record=OrphanFenceRecord( + max(newest.epoch if newest else 0, stored[0].epoch if stored else 0) + 1, + str(uuid.uuid4()), + previous.token if previous else None, + format_row_instant(datetime.now(timezone.utc)), + ), + entries=entries, + previous_epoch=newest.epoch if newest else None, + ) + + claimed = await _claim_orphan_fence(store, binding_id, scope, False, plan) + if claimed.kind != "claimed" or claimed.record is None: + lost = claimed.kind == "lost" and claimed.record is not None + raise ReportingRowStoreError( + "STATE_UNAVAILABLE", + ( + f"another database claimed epoch {claimed.record.epoch} while this takeover " + "ran; run it again" + if lost and claimed.record is not None + else "the takeover was refused" + ), + ) + taken.append( + ReportingOrphanFenceTakeoverV1( + tuple(binding_ids), claimed.record.epoch, claimed.previous_epoch + ) + ) + return taken + + +# -- scheduling ------------------------------------------------------------------------------ + + +def _idle_result(options: ReportingOrphanCollectionOptionsV1) -> ReportingOrphanCollectionResultV1: + return ReportingOrphanCollectionResultV1(dry_run=options.dry_run) + + +async def run_orphan_collection_pass( + store: PgReportingLedgerStore, + options: ReportingOrphanCollectionOptionsV1, + state: ReportingOrphanCollectionStateV1, +) -> ReportingOrphanCollectionResultV1: + """One scheduled pass: resumes the listing where the last one stopped and honours the interval. + + ``state`` is owned by the caller and updated in place, so a large listing resumes where the + last pass stopped and ``interval_seconds`` is honoured. + """ + if state.next_cycle_at is not None and time.monotonic() < state.next_cycle_at: + # Between full listings: nothing to do this pass. + return _idle_result(options) + outcome = await collect_orphaned_row_objects( + store, + dry_run=options.dry_run, + acknowledge_exclusive_namespace=options.acknowledge_exclusive_namespace, + min_age_seconds=options.min_age_seconds, + limit=options.limit, + include_read_only=options.include_read_only, + cursors=dict(state.cursors), + skip_bindings=list(state.completed), + ) + state.cursors = dict(outcome.cursors) + # A finished, clean cycle rests for the interval; anything else carries on next pass, without + # listing again the bindings that already finished. (A delete that keeps failing does not stop + # the rest: it is tried again on the next listing.) + if not outcome.cursors and not any(item.status == "failed" for item in outcome.bindings): + state.completed = [] + # A fenced binding looks again within the hour, so applying the migration or taking the + # namespace over does not leave collection idle for the rest of a day. + rest = options.interval_seconds + if any(item.status == "fenced" for item in outcome.bindings): + rest = min(rest, FENCED_RECHECK_SECONDS) + state.next_cycle_at = time.monotonic() + rest + else: + state.completed = list(dict.fromkeys([*state.completed, *outcome.completed])) + state.next_cycle_at = None + return outcome diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/orphan_fence.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/orphan_fence.py new file mode 100644 index 0000000000..db2c984a94 --- /dev/null +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/orphan_fence.py @@ -0,0 +1,323 @@ +"""The orphan collector's ownership fence (adcp-client#3593, adcp-client-python#1829). + +The collector deletes objects under ``{prefix}/{namespace_key}/`` that no ledger record names. +A clone or point-in-time restore of the ledger database carries the same configuration, +installation identity and namespace key, but an older set of references, so its collector would +delete the live database's newer objects. Nothing inside the database can tell the two apart; the +object store they share can. + +The fence is a chain of create-only objects, one per epoch, under the binding's owned prefix:: + + {prefix}/{namespace_key}/_collector-fence/0000000000000007.json + +Each holds ``{v, namespace_key, epoch, token, previous_token, claimed_at}``. The highest epoch is +the current claim. The database records the ``(epoch, token)`` it last claimed in +``adcp_reporting_orphan_collector_fences``. A real pass deletes only when the highest epoch in the +store is exactly the claim this database recorded, then claims the next epoch (create-only, so +exactly one claimant wins) before it deletes anything. Every other state refuses with a cause. A +database that was copied before the original's last claim sees a higher epoch and refuses; so does +a restore, until an operator calls ``take_over_orphan_collector_fence``. + +This module is the pure part: the decision table, the key and record formats and the scope. The +I/O and the transaction live in :mod:`adcp_reporting_storage.ledger.orphan_collector`. The bytes +and the scope hash are the JavaScript SDK's, so either SDK reads the other's fence. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Literal + +import rfc8785 + +from adcp_reporting_storage.ledger.row_storage_object import is_row_instant + +__all__ = [ + "ORPHAN_FENCE_DIRECTORY", + "ORPHAN_FENCE_MAX_BYTES", + "OrphanFenceDecision", + "OrphanFenceEntry", + "OrphanFenceRecord", + "OrphanFenceRow", + "ReportingOrphanFenceCauseV1", + "ReportingOrphanFenceRefusalV1", + "decide_orphan_fence", + "encode_orphan_fence_record", + "is_orphan_fence_key", + "orphan_fence_directory", + "orphan_fence_epoch_of_key", + "orphan_fence_key", + "orphan_fence_scope", + "parse_orphan_fence_record", +] + +#: Object-store directory (under the owned prefix) that holds the fence chain. +ORPHAN_FENCE_DIRECTORY = "_collector-fence" +#: Fence objects are tiny; anything larger is not one. +ORPHAN_FENCE_MAX_BYTES = 4096 + +_EPOCH_DIGITS = 16 +_UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") +_KEY_NAME = re.compile(rf"([0-9]{{{_EPOCH_DIGITS}}})\.json") +_FIELDS = frozenset({"claimed_at", "epoch", "namespace_key", "previous_token", "token", "v"}) + +#: Why a binding's collector refused to delete. +#: +#: * ``claimed_elsewhere``: this database never claimed the namespace, but the object store holds +#: a claim. A clone that has not collected before, a database whose fence table was lost, or +#: another installation. +#: * ``fence_advanced``: the object store holds a newer claim than this database recorded. Another +#: database (usually the live one this was cloned from) has collected since this database's +#: state was taken. +#: * ``token_mismatch``: the object store holds this database's epoch under a different token. Two +#: databases claimed the same epoch; the other one won. +#: * ``fence_missing``: this database recorded a claim and the object store holds none: the bucket +#: or prefix was reset or is a different one. +#: * ``fence_regressed``: the object store holds an older claim than this database recorded: a +#: database restored from a newer state, or a bucket restored from an older one. +#: * ``fence_unreadable``: the fence objects are malformed, oversized, duplicated or name another +#: namespace. +#: * ``claim_lost``: this pass tried to claim the next epoch and another database got there first. +#: * ``migration_required``: the fence table does not exist; apply ``reporting_row_storage.sql``. +ReportingOrphanFenceCauseV1 = Literal[ + "claimed_elsewhere", + "fence_advanced", + "token_mismatch", + "fence_missing", + "fence_regressed", + "fence_unreadable", + "claim_lost", + "migration_required", +] + + +@dataclass(frozen=True, slots=True) +class ReportingOrphanFenceRefusalV1: + """Why a binding was fenced, reported on its collection outcome.""" + + cause: ReportingOrphanFenceCauseV1 + #: What to check, and how to take the namespace over if this database really is its only owner. + detail: str + #: Highest epoch found in the object store, when readable. + fence_epoch: int | None = None + #: Epoch this database last recorded, when it has one. + recorded_epoch: int | None = None + + +@dataclass(frozen=True, slots=True) +class OrphanFenceEntry: + """One fence object found by a listing.""" + + epoch: int + object_ref: str + native_version_ref: str + + +@dataclass(frozen=True, slots=True) +class OrphanFenceRecord: + """The contents of one fence object.""" + + epoch: int + token: str + previous_token: str | None + claimed_at: str + + +@dataclass(frozen=True, slots=True) +class OrphanFenceRow: + """What this database recorded for a fence scope.""" + + epoch: int + token: str + previous_token: str | None + + +@dataclass(frozen=True, slots=True) +class OrphanFenceDecision: + """``claim``: this database may collect. A real pass claims ``epoch`` first (the next epoch + after its own claim, or 1 when the namespace was never claimed); a read-only check just + proceeds. ``complete``: this database recorded a claim and crashed or failed before its object + was written: write it now. ``fenced``: refuse, with ``refusal``.""" + + action: Literal["claim", "complete", "fenced"] + epoch: int | None = None + previous_token: str | None = None + refusal: ReportingOrphanFenceRefusalV1 | None = None + + +_TAKEOVER_HINT = ( + "If this database is the only one that owns this namespace (for example after restoring it " + "from a backup, and the original is gone), call " + "take_over_orphan_collector_fence(acknowledge_previous_owner_gone=True). Otherwise leave " + "orphan collection off on this copy." +) + + +def decide_orphan_fence( + row: OrphanFenceRow | None, latest: tuple[int, str] | None +) -> OrphanFenceDecision: + """Decide, from what this database recorded and what the object store holds, whether it may + collect. Pure: the caller does the I/O and the transaction. + + ``latest`` is the highest ``(epoch, token)`` in the object store, or ``None`` when it holds no + fence. + """ + + def refuse(cause: ReportingOrphanFenceCauseV1, detail: str) -> OrphanFenceDecision: + return OrphanFenceDecision( + "fenced", + refusal=ReportingOrphanFenceRefusalV1( + cause, + f"{detail}. {_TAKEOVER_HINT}", + latest[0] if latest else None, + row.epoch if row else None, + ), + ) + + if row is None: + if latest is None: + return OrphanFenceDecision("claim", epoch=1) + return refuse( + "claimed_elsewhere", + f"the namespace is already claimed (epoch {latest[0]}) by a database this one has no " + "record of: it is a clone or restore of that database, or another installation that " + "shares the namespace", + ) + if latest is None: + # An unconfirmed first claim: this database recorded epoch 1 and did not write the object. + if row.epoch == 1 and row.previous_token is None: + return OrphanFenceDecision("complete") + return refuse( + "fence_missing", + f"this database recorded epoch {row.epoch} and the object store holds no fence: the " + "bucket or prefix was reset or is a different one", + ) + latest_epoch, latest_token = latest + if latest_epoch == row.epoch: + if latest_token == row.token: + return OrphanFenceDecision("claim", epoch=row.epoch + 1, previous_token=row.token) + return refuse( + "token_mismatch", + f"epoch {row.epoch} in the object store belongs to another claim: two databases " + "claimed the same epoch and the other one won", + ) + if latest_epoch > row.epoch: + return refuse( + "fence_advanced", + f"the object store holds epoch {latest_epoch} and this database recorded epoch " + f"{row.epoch}: another database that shares this namespace has collected since this " + "database was copied or restored", + ) + # latest_epoch < row.epoch. One behind with the recorded previous token is an unconfirmed + # claim of this database (it wrote the intent and not the object); anything else is a + # regression. + if ( + latest_epoch == row.epoch - 1 + and row.previous_token is not None + and latest_token == row.previous_token + ): + return OrphanFenceDecision("complete") + return refuse( + "fence_regressed", + f"the object store holds epoch {latest_epoch}, older than the epoch {row.epoch} this " + "database recorded: the database was restored from a newer state than the bucket, or " + "the bucket from an older one", + ) + + +def orphan_fence_directory(owned_prefix: str) -> str: + """The listing prefix of the fence chain under a binding's owned prefix.""" + return f"{owned_prefix}{ORPHAN_FENCE_DIRECTORY}/" + + +def orphan_fence_key(owned_prefix: str, epoch: int) -> str: + """Object key of one epoch's fence object under ``{prefix}/{namespace_key}/``.""" + return f"{orphan_fence_directory(owned_prefix)}{epoch:0{_EPOCH_DIGITS}d}.json" + + +def is_orphan_fence_key(owned_prefix: str, key: str) -> bool: + """Whether a listed key is part of the fence chain.""" + return key.startswith(orphan_fence_directory(owned_prefix)) + + +def orphan_fence_epoch_of_key(owned_prefix: str, key: str) -> int | None: + """The epoch a fence key names, or ``None`` for any other key under the fence directory.""" + directory = orphan_fence_directory(owned_prefix) + if not key.startswith(directory): + return None + match = _KEY_NAME.fullmatch(key[len(directory) :]) + if match is None: + return None + epoch = int(match.group(1)) + return epoch if epoch >= 1 else None + + +def encode_orphan_fence_record(namespace_key: str, record: OrphanFenceRecord) -> bytes: + """The fence object's bytes: the JSON the JavaScript SDK writes, one line.""" + return ( + json.dumps( + { + "v": 1, + "namespace_key": namespace_key, + "epoch": record.epoch, + "token": record.token, + "previous_token": record.previous_token, + "claimed_at": record.claimed_at, + }, + separators=(",", ":"), + ensure_ascii=False, + ) + + "\n" + ).encode("utf-8") + + +def parse_orphan_fence_record( + data: bytes, *, namespace_key: str, epoch: int +) -> OrphanFenceRecord | None: + """Parse a fence object strictly. + + ``None`` for anything that is not exactly a fence of this namespace and epoch. + """ + if len(data) > ORPHAN_FENCE_MAX_BYTES: + return None + try: + value = json.loads(data.decode("utf-8")) + except (ValueError, UnicodeError): + return None + if not isinstance(value, dict) or set(value) != _FIELDS: + return None + token = value["token"] + previous = value["previous_token"] + if ( + value["v"] != 1 + or type(value["v"]) is not int + or value["namespace_key"] != namespace_key + or type(value["epoch"]) is not int + or value["epoch"] != epoch + or not isinstance(token, str) + or _UUID.fullmatch(token) is None + or not ( + previous is None + or (isinstance(previous, str) and _UUID.fullmatch(previous) is not None) + ) + or not is_row_instant(value["claimed_at"]) + ): + return None + return OrphanFenceRecord(epoch, token, previous, value["claimed_at"]) + + +def orphan_fence_scope( + *, provider: str, location: Mapping[str, str], prefix: str, namespace_key: str +) -> str: + """Identity of one physical fence: the provider, its location, the prefix and the namespace. + + Bindings that share all four share a fence, because a collector listing through either one + reaches the same objects. + """ + return hashlib.sha256( + rfc8785.dumps(["adcp.orphan_fence.v1", provider, dict(location), prefix, namespace_key]) + ).hexdigest() diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/period_retirement.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/period_retirement.py index 20354f3ace..21707be5ce 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/period_retirement.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/period_retirement.py @@ -44,6 +44,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any +from adcp_reporting_storage.ledger.change_feed import feed_consumer_behind_change from adcp_reporting_storage.ledger.row_retention import NoLongerEligibleError from adcp_reporting_storage.ledger.row_storage import ReportingRowStoreError @@ -122,15 +123,12 @@ async def retire_expired_periods( connection, "adcp_reporting_feed_consumers" ): holds.append( - " AND NOT EXISTS (SELECT 1 FROM adcp_reporting_changes change" + " AND NOT EXISTS (SELECT 1 FROM adcp_reporting_changes change" # noqa: S608 # nosec B608 " JOIN adcp_reporting_feed_consumers consumer" " ON consumer.updated_at > clock_timestamp()" " - (%(hold_days)s::integer * INTERVAL '1 day')" " WHERE change.obligation_id = o.reporting_obligation_id" - " AND ((consumer.account_id IS NULL AND (change.xid, change.seq) >" - " (consumer.cursor_xid, consumer.cursor_seq))" - " OR (consumer.account_id = change.account_id" - " AND change.seq > consumer.cursor_seq)))" + f" AND {feed_consumer_behind_change('consumer', 'change')})" ) parameters = { "account": account_id, diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index faf575296b..3047418979 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -76,7 +76,7 @@ import base64 import hashlib import json -from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence from contextlib import asynccontextmanager from contextvars import ContextVar from copy import deepcopy @@ -172,6 +172,11 @@ ReportingChangeKindV1, ReportingChangesPageV1, ) + from adcp_reporting_storage.ledger.orphan_collector import ( + ReportingOrphanCollectionOptionsV1, + ReportingOrphanCollectionResultV1, + ReportingOrphanFenceTakeoverV1, + ) from adcp_reporting_storage.ledger.period_retirement import ReportingPeriodRetirementV1 from adcp_reporting_storage.ledger.row_migration import ReportingRowMigrationBatchV1 from adcp_reporting_storage.ledger.row_retention import ( @@ -256,6 +261,7 @@ def __init__( notifications: bool = False, row_storage: bool | ReportingRowStorageOptionsV1 | None = None, change_feed: bool = False, + orphan_collection: ReportingOrphanCollectionOptionsV1 | None = None, ) -> None: if not PG_AVAILABLE: raise ImportError(_INSTALL_HINT) @@ -271,6 +277,20 @@ def __init__( row_storage if isinstance(row_storage, ReportingRowStorageOptionsV1) else None ) ) + # Opt-in listing collector for row objects the write-intent sweep cannot see, scheduled by + # the production worker. It needs row storage; its position lives in this process. + if orphan_collection is not None: + from adcp_reporting_storage.ledger.orphan_collector import ( + ReportingOrphanCollectionOptionsV1, + ReportingOrphanCollectionStateV1, + ) + + if not isinstance(orphan_collection, ReportingOrphanCollectionOptionsV1): + raise TypeError("orphan_collection must be ReportingOrphanCollectionOptionsV1") + if self._row_storage is None: + raise ValueError("orphan_collection requires row_storage") + self._orphan_state = ReportingOrphanCollectionStateV1() + self._orphan_collection = orphan_collection self._row_sets_known = False self._tombstones_known = False # Opt-in shared host change feed (adcp_reporting_changes), PostgreSQL 13+. @@ -432,8 +452,16 @@ def maintenance_operations(self) -> tuple[tuple[str, Callable[[], Awaitable[Any] operations.append(("period_retirement", lambda: self.retire_expired_periods(limit=20))) if self._change_feed: operations.append(("change_pruning", lambda: self.prune_changes())) + if self._orphan_collection is not None: + operations.append(("orphan_collection", self._run_orphan_collection)) return tuple(operations) + async def _run_orphan_collection(self) -> ReportingOrphanCollectionResultV1: + from adcp_reporting_storage.ledger.orphan_collector import run_orphan_collection_pass + + assert self._orphan_collection is not None + return await run_orphan_collection_pass(self, self._orphan_collection, self._orphan_state) + @asynccontextmanager async def source_publication( self, account_id: str, *, seals: ReportingSealStore | None = None @@ -558,6 +586,72 @@ async def sweep_row_write_intents( self, limit=limit, account_id=account_id, overdue_after_seconds=overdue_after_seconds ) + async def collect_orphaned_row_objects( + self, + *, + min_age_seconds: int | None = None, + limit: int = 1_000, + page_size: int | None = None, + dry_run: bool = False, + include_read_only: bool = False, + bindings: Sequence[str] | None = None, + cursors: Mapping[str, str] | None = None, + skip_bindings: Sequence[str] | None = None, + acknowledge_exclusive_namespace: bool = False, + now: datetime | None = None, + ) -> ReportingOrphanCollectionResultV1: + """Delete row objects that no ledger record or open upload names. + + Opt-in and listing-based: it covers what the write-intent sweep cannot see (an upload + whose ``PUT`` succeeded before its intent recorded it, intents for a binding no running + host configures, intents lost to a database restore). It lists each configured object + binding under ``{prefix}/{namespace_key}/`` and deletes an object only when it is a + well-formed row object, older than ``min_age_seconds`` (default 7 days, at least 1 day), + and no ledger record or open upload names it, under the per-object lock of every binding. + Deleting needs ``acknowledge_exclusive_namespace=True`` and is fenced against a clone or + restore of this database; ``dry_run`` deletes nothing. See + :func:`adcp_reporting_storage.ledger.orphan_collector.collect_orphaned_row_objects`. + """ + from adcp_reporting_storage.ledger.orphan_collector import collect_orphaned_row_objects + + return await collect_orphaned_row_objects( + self, + min_age_seconds=min_age_seconds, + limit=limit, + page_size=page_size, + dry_run=dry_run, + include_read_only=include_read_only, + bindings=bindings, + cursors=cursors, + skip_bindings=skip_bindings, + acknowledge_exclusive_namespace=acknowledge_exclusive_namespace, + now=now, + ) + + async def take_over_orphan_collector_fence( + self, + *, + acknowledge_previous_owner_gone: bool, + bindings: Sequence[str] | None = None, + include_read_only: bool = False, + ) -> list[ReportingOrphanFenceTakeoverV1]: + """Claim the orphan collector's namespace for this database, fencing every other copy out. + + The deliberate step after a restore, or when the database that held the fence is gone for + good. Stop the previous owner's collector first. See + :func:`adcp_reporting_storage.ledger.orphan_collector.take_over_orphan_collector_fence`. + """ + from adcp_reporting_storage.ledger.orphan_collector import ( + take_over_orphan_collector_fence, + ) + + return await take_over_orphan_collector_fence( + self, + acknowledge_previous_owner_gone=acknowledge_previous_owner_gone, + bindings=bindings, + include_read_only=include_read_only, + ) + async def prune_expired_row_sets( self, *, @@ -591,8 +685,9 @@ async def changes_after( ) -> ReportingChangesPageV1: """Read the shared host change feed; see :mod:`adcp_reporting_storage.ledger.change_feed`. - Account feeds order by ``seq``; the deployment-wide feed orders by ``(xid, seq)`` - below the reading snapshot's ``xmin``. A cursor older than the retained history + Every feed, account-scoped or deployment-wide, orders by ``(xid, seq)`` below the + reading snapshot's ``xmin``; an account cursor from before that carries only ``seq`` + and is mapped to a position that skips nothing. A cursor older than the retained history raises ``ReportingChangeCursorExpiredError``, and so does a read with no cursor when a prune commits while the page is being read (the horizon is read again after the page). Requires ``change_feed=True``. diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_changes.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_changes.sql index e114507b98..eb54c01445 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_changes.sql +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_changes.sql @@ -8,9 +8,16 @@ -- PgReportingLedgerStore(change_feed=True) applies it from create_schema. -- -- A row is written in the same transaction as each obligation, revision, adjustment --- and retirement, taking nextval only after the account lock, so per-account sequence --- order is commit order. Deployment-wide readers order by (xid, seq) and stop below --- the reading snapshot's xmin, so a slow commit is never skipped. +-- and retirement. Every reader, account-scoped or deployment-wide, orders by (xid, seq) +-- and stops below the reading snapshot's xmin, so a slow commit is never skipped and no +-- feed depends on the account lock or on the order a writer drew seq in. Account cursors +-- carry xid; a seq-only cursor from before that is mapped to a position that skips +-- nothing (adcp-client#3289, adcp-client-python#1825). +-- +-- Re-applying this file upgrades an install from before the account feed ordered by +-- (xid, seq): it adds the account horizon's xid columns, replaces the consumer table's +-- check constraint and creates the (account_id, xid, seq) index. On a large changes table +-- create that index first with CREATE INDEX CONCURRENTLY; this file then skips it. -- -- This table is separate from reporting_ledger_changes, the buyer-facing feed behind -- the protocol's changes_checkpoint and changes_after. They converge with the shared @@ -34,8 +41,13 @@ CREATE TABLE IF NOT EXISTS adcp_reporting_changes ( obligation_id TEXT NOT NULL, recorded_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() ); +-- Superseded by adcp_reporting_changes_account_xid for the account feed; kept so +-- re-applying this file never drops an index, and because it serves the mapping of a +-- seq-only account cursor. Drop it once no older host runs and no such cursor remains. CREATE INDEX IF NOT EXISTS adcp_reporting_changes_account ON adcp_reporting_changes (account_id, seq); +CREATE INDEX IF NOT EXISTS adcp_reporting_changes_account_xid + ON adcp_reporting_changes (account_id, xid, seq); CREATE INDEX IF NOT EXISTS adcp_reporting_changes_deployment ON adcp_reporting_changes (xid, seq); CREATE INDEX IF NOT EXISTS adcp_reporting_changes_recorded @@ -50,19 +62,137 @@ CREATE TABLE IF NOT EXISTS adcp_reporting_change_horizon ( ); INSERT INTO adcp_reporting_change_horizon (singleton) VALUES (TRUE) ON CONFLICT DO NOTHING; --- Highest seq ever pruned per account: an account cursor below it fails closed. +-- An account horizon whose xid columns a release that does not maintain them has pruned past. +-- adcp-reporting-storage 0.1.0b5 orders the account feed by (xid, seq), hands out account +-- cursors that carry xid and records the account horizon in (xid, seq). 0.2.0b0 to 0.2.0b2 do +-- not: their prune advances pruned_through_seq only, so on a ledger 0.1.0b5 wrote, a cursor +-- 0.1.0b5 handed out could pass the stale (xid, seq) check and silently skip what 0.2.x +-- pruned. Pruned xids are gone, so each account horizon is raised to a position at or above +-- every change that can have been pruned: the deployment horizon, which every release keeps in +-- (xid, seq), or, when lower, the position just below the account's oldest retained change, so +-- a consumer starting over still reads every retained change. The affected cursors then fail +-- closed with CURSOR_EXPIRED. A cursor behind that position whose changes were all retained +-- expires too, once. +-- +-- It runs on the first create_schema of this release on a ledger that already had the account +-- xid columns (0.1.0b5 ran on it) and that a migrations-aware 0.2 release upgraded in an +-- earlier transaction (0001 is recorded before this one began). A ledger upgraded straight from +-- 0.1.0b5, an install from before the (xid, seq) account feed (whose account cursors carry no +-- xid) and a new install change nothing. The table comment set below records that this +-- release has applied the file, so the step never runs again. Stop every older host's change +-- maintenance before this release starts, as for the (xid, seq) upgrade itself. +DO $adcp_account_horizon$ +DECLARE + account_horizon regclass := + to_regclass(quote_ident(current_schema()) || '.adcp_reporting_change_account_horizon'); + migrations regclass := + to_regclass(quote_ident(current_schema()) || '.adcp_schema_migrations'); + upgraded_earlier BOOLEAN := FALSE; + deployment_xid xid8; + deployment_seq BIGINT; + account RECORD; + bound_xid xid8; + bound_seq BIGINT; + oldest_xid xid8; + oldest_seq BIGINT; +BEGIN + IF account_horizon IS NULL + OR obj_description(account_horizon, 'pg_class') IS NOT DISTINCT FROM + 'adcp: account horizon maintained in (xid, seq)' + OR NOT EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid = account_horizon AND attname = 'pruned_through_xid' AND NOT attisdropped + ) + OR migrations IS NULL THEN + RETURN; + END IF; + EXECUTE format( + 'SELECT EXISTS (SELECT 1 FROM %s WHERE version = %L AND applied_at < now())', + migrations, '0001' + ) INTO upgraded_earlier; + IF NOT upgraded_earlier THEN + RETURN; + END IF; + SELECT pruned_through_xid, pruned_through_seq INTO deployment_xid, deployment_seq + FROM adcp_reporting_change_horizon WHERE singleton; + IF NOT FOUND THEN + RETURN; + END IF; + FOR account IN + SELECT account_id, pruned_through_xid, pruned_through_xid_seq + FROM adcp_reporting_change_account_horizon + LOOP + bound_xid := deployment_xid; + bound_seq := deployment_seq; + SELECT change.xid, change.seq - 1 INTO oldest_xid, oldest_seq + FROM adcp_reporting_changes change + WHERE change.account_id = account.account_id + ORDER BY change.xid, change.seq LIMIT 1; + IF FOUND AND (oldest_xid, oldest_seq) < (bound_xid, bound_seq) THEN + bound_xid := oldest_xid; + bound_seq := oldest_seq; + END IF; + IF (bound_xid, bound_seq) > (account.pruned_through_xid, account.pruned_through_xid_seq) THEN + UPDATE adcp_reporting_change_account_horizon + SET pruned_through_xid = bound_xid, pruned_through_xid_seq = bound_seq + WHERE account_id = account.account_id; + END IF; + END LOOP; +END +$adcp_account_horizon$; + +-- Highest (xid, seq) ever pruned per account: an account cursor below it fails closed. +-- pruned_through_seq is the highest seq pruned whatever its xid; account feeds no longer +-- compare it, it exists so a seq-only cursor from before the (xid, seq) order can still be +-- checked exactly (a tuple compare alone would miss a pruned change with a higher seq and +-- a lower xid). CREATE TABLE IF NOT EXISTS adcp_reporting_change_account_horizon ( account_id TEXT PRIMARY KEY, - pruned_through_seq BIGINT NOT NULL + pruned_through_seq BIGINT NOT NULL, + pruned_through_xid xid8 NOT NULL DEFAULT '0', + pruned_through_xid_seq BIGINT NOT NULL DEFAULT 0 ); +ALTER TABLE adcp_reporting_change_account_horizon + ADD COLUMN IF NOT EXISTS pruned_through_xid xid8 NOT NULL DEFAULT '0'; +ALTER TABLE adcp_reporting_change_account_horizon + ADD COLUMN IF NOT EXISTS pruned_through_xid_seq BIGINT NOT NULL DEFAULT 0; +COMMENT ON TABLE adcp_reporting_change_account_horizon IS + 'adcp: account horizon maintained in (xid, seq)'; --- Registered consumers hold back pruning in their own feed order: account --- consumers by seq within their account, deployment consumers by (xid, seq). +-- Registered consumers hold back pruning in (xid, seq) order, per account for an account +-- consumer. A row saved before the account feed carried an xid has a NULL cursor_xid and holds +-- by seq until the consumer saves its next cursor. CREATE TABLE IF NOT EXISTS adcp_reporting_feed_consumers ( consumer_name TEXT PRIMARY KEY CHECK (consumer_name ~ '^[A-Za-z0-9_.:-]{1,128}$'), account_id TEXT, cursor_xid xid8, cursor_seq BIGINT NOT NULL, updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), - CHECK ((account_id IS NULL) = (cursor_xid IS NOT NULL)) + CONSTRAINT adcp_reporting_feed_consumers_cursor_scope + CHECK (account_id IS NOT NULL OR cursor_xid IS NOT NULL) ); +-- An install from before the account feed carried an xid has the anonymous check +-- (account_id IS NULL) = (cursor_xid IS NOT NULL), which rejects an account consumer that has +-- an xid. Drop it and add the named one; both steps are guarded, so re-applying is a no-op. +DO $adcp_feed_consumers$ +DECLARE stale TEXT; +BEGIN + FOR stale IN + SELECT conname FROM pg_constraint + WHERE conrelid = 'adcp_reporting_feed_consumers'::regclass AND contype = 'c' + AND conname <> 'adcp_reporting_feed_consumers_cursor_scope' + AND strpos(pg_get_constraintdef(oid), 'account_id IS NULL') > 0 + LOOP + EXECUTE format('ALTER TABLE adcp_reporting_feed_consumers DROP CONSTRAINT %I', stale); + END LOOP; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'adcp_reporting_feed_consumers'::regclass + AND conname = 'adcp_reporting_feed_consumers_cursor_scope' + ) THEN + ALTER TABLE adcp_reporting_feed_consumers + ADD CONSTRAINT adcp_reporting_feed_consumers_cursor_scope + CHECK (account_id IS NOT NULL OR cursor_xid IS NOT NULL); + END IF; +END +$adcp_feed_consumers$; diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_row_storage.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_row_storage.sql index 0862039adb..1fe2a715d4 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_row_storage.sql +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_row_storage.sql @@ -107,7 +107,9 @@ CREATE TABLE IF NOT EXISTS adcp_persistence_installation ( INSERT INTO adcp_persistence_installation (singleton) VALUES (TRUE) ON CONFLICT DO NOTHING; CREATE TABLE IF NOT EXISTS adcp_reporting_row_bindings ( - row_binding_id TEXT COLLATE "C" PRIMARY KEY CHECK (row_binding_id ~ '^[A-Za-z0-9_.:-]{1,128}$'), + row_binding_id TEXT COLLATE "C" PRIMARY KEY + CONSTRAINT adcp_reporting_row_bindings_row_binding_id_bounds + CHECK (row_binding_id ~ '^[A-Za-z0-9_.:-]{1,255}$'), kind TEXT NOT NULL CHECK (kind IN ('postgres', 'object')), provider TEXT NOT NULL CHECK (provider ~ '^[a-z][a-z0-9_]{0,63}$'), identity_config JSONB NOT NULL CHECK (jsonb_typeof(identity_config) = 'object'), @@ -316,6 +318,41 @@ BEGIN END $bounds$; +-- row_binding_id is bounded like AdCP's comparable identifiers (reporting_revision_id, +-- resource_ref and destination_ref use ^[A-Za-z0-9_.:-]{1,255}$): 255 characters, where the +-- original CHECK allowed 128 (adcp-client-python#1498, adcp-client#3595). A database installed +-- with the original unnamed CHECK drops it and gets the named constraint a fresh install carries +-- inline. Guarded on the catalog, so a current schema changes nothing. Widening a CHECK needs no +-- data migration, and the tables that reference row_binding_id hold it by foreign key only. The +-- constraint name and the bound are the JavaScript SDK's, so either SDK re-applying is a no-op. +DO $binding_id_bound$ +DECLARE + bindings regclass := to_regclass(quote_ident(current_schema()) || '.adcp_reporting_row_bindings'); + legacy TEXT; +BEGIN + IF bindings IS NULL THEN + RETURN; + END IF; + FOR legacy IN + SELECT conname FROM pg_constraint + WHERE conrelid = bindings AND contype = 'c' + AND conname <> 'adcp_reporting_row_bindings_row_binding_id_bounds' + AND strpos(pg_get_constraintdef(oid), 'row_binding_id') > 0 + AND strpos(pg_get_constraintdef(oid), '{1,128}') > 0 + LOOP + EXECUTE format('ALTER TABLE adcp_reporting_row_bindings DROP CONSTRAINT %I', legacy); + END LOOP; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = bindings AND conname = 'adcp_reporting_row_bindings_row_binding_id_bounds' + ) THEN + ALTER TABLE adcp_reporting_row_bindings + ADD CONSTRAINT adcp_reporting_row_bindings_row_binding_id_bounds + CHECK (row_binding_id ~ '^[A-Za-z0-9_.:-]{1,255}$'); + END IF; +END +$binding_id_bound$; + -- Open write intents written before the rename name objects objectKey / nativeVersion. UPDATE adcp_reporting_row_write_intents SET created_objects = ( @@ -332,3 +369,24 @@ UPDATE adcp_reporting_row_write_intents SELECT 1 FROM jsonb_array_elements(created_objects) AS element(created) WHERE jsonb_typeof(created) = 'object' AND created ? 'objectKey' ); + +-- The orphan collector's ownership fence (adcp-client#3593, adcp-client-python#1829): one row per +-- fence, recording the (epoch, token) this database last claimed in the object store under +-- {prefix}/{namespace_key}/_collector-fence/. collect_orphaned_row_objects deletes only while the +-- object store's highest fence epoch is exactly the claim recorded here, so a clone or restore of +-- this database that the original has since out-claimed refuses to delete. The same table and +-- columns the JavaScript SDK ships, so either SDK reads the other's claim. +-- +-- Deliberately outside the required-schema manifest and REPORTING_ROW_STORAGE_TABLES: the collector +-- is opt-in, so a database that never enables it needs nothing from this table. Until it exists a +-- collector pass refuses every binding as fenced (migration_required) and deletes nothing. +CREATE TABLE IF NOT EXISTS adcp_reporting_orphan_collector_fences ( + fence_scope TEXT COLLATE "C" PRIMARY KEY CHECK (fence_scope ~ '^[0-9a-f]{64}$'), + epoch BIGINT NOT NULL CHECK (epoch >= 1), + token TEXT COLLATE "C" NOT NULL + CHECK (token ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'), + previous_token TEXT COLLATE "C" + CHECK (previous_token IS NULL + OR previous_token ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'), + claimed_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/required_row_storage_schema.json b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/required_row_storage_schema.json index 421c0dd889..bbd7cce0f7 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/required_row_storage_schema.json +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/required_row_storage_schema.json @@ -291,9 +291,9 @@ "enabled": true, "fingerprint": "b0825d60a99952fe5be6f33e0123ae5166e1d17eda49af821d22a5ca8856e13b" }, - "constraint:adcp_reporting_row_bindings.adcp_reporting_row_bindings_row_binding_id_check": { + "constraint:adcp_reporting_row_bindings.adcp_reporting_row_bindings_row_binding_id_bounds": { "enabled": true, - "fingerprint": "a804a095e5b6b0b8434f809b087674c480399eef72e852c297ad00ecb3c5694c" + "fingerprint": "700166f3efc148bd5cab036b1fcdf3ae35acba6f2a127c24fa9f8888187d8c1d" }, "constraint:adcp_reporting_row_bindings.adcp_reporting_row_bindings_state_check": { "enabled": true, diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_retention.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_retention.py index 1834aa4b29..bb1690547f 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_retention.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_retention.py @@ -378,8 +378,9 @@ async def delete_period_objects( An object that a live chunk of another obligation still references is kept: the default key template cannot collide across obligations, but a custom template can, and a - delete must never take bytes another revision still needs. See - ``ReportingRowStorageV1.delete_unreferenced`` for the locking contract. + delete must never take bytes another revision still needs. No object lock is taken: see the + contract in ``ReportingRowStorageV1`` (``# -- object lifetime``) and + ``ReportingRowStorageV1.delete_unreferenced``. """ storage = store._row_storage assert storage is not None @@ -397,8 +398,10 @@ async def delete_period_objects( for binding_id, object_ref, native_version_ref in objects: by_binding.setdefault(binding_id, []).append((object_ref, native_version_ref)) for binding_id, pairs in by_binding.items(): - # Objects a chunk of any other row set still references are kept, checked - # atomically with the delete so a concurrent commit cannot adopt a doomed object. + # Objects a chunk of any other row set still references are kept. The check needs no lock + # against a committing writer: the period's row sets are already ``pruning`` (committed + # under the account lock in phase 1), which orders this delete against every other + # deleter. await storage.delete_unreferenced( store._pool, binding_id, pairs, obligation_id=obligation_id ) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage.py index 39d7ef542f..35c0806411 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage.py @@ -25,9 +25,8 @@ import hashlib import inspect import json -import re import zlib -from collections.abc import Awaitable, Callable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Collection, Mapping, Sequence from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, TypeVar @@ -54,18 +53,38 @@ ReportingRowStoreError, ReportingRowStoreErrorCode, ) +from adcp_reporting_storage.ledger.orphan_fence import ( + ORPHAN_FENCE_MAX_BYTES, + OrphanFenceEntry, + OrphanFenceRecord, + encode_orphan_fence_record, + is_orphan_fence_key, + orphan_fence_directory, + orphan_fence_epoch_of_key, + orphan_fence_key, + orphan_fence_scope, + parse_orphan_fence_record, +) from adcp_reporting_storage.ledger.row_storage_object import ( + REPORTING_ROW_BINDING_ID_MAX_LENGTH, + REPORTING_ROW_CREDENTIAL_REF_MAX_LENGTH, REPORTING_ROW_DEFAULT_KEY_TEMPLATE, + REPORTING_ROW_OBJECT_REF_MAX_LENGTH, ReportingRowObjectContextV1, ReportingRowObjectDeleteInputV1, ReportingRowObjectGetInputV1, + ReportingRowObjectListEntryV1, + ReportingRowObjectListInputV1, ReportingRowObjectProviderV1, ReportingRowObjectPutInputV1, ReportingRowObjectStatInputV1, assert_reporting_row_key_template_v1, assert_reporting_row_prefix_v1, + is_row_instant, + is_row_object_key_shape, render_reporting_row_object_key_v1, ) +from adcp_reporting_storage.ledger.row_storage_provider_io import is_row_identifier if TYPE_CHECKING: from adcp.reporting.ledger.row_encoding import ( @@ -105,8 +124,6 @@ Compression = Literal["none", "gzip"] _DEFAULT_BINDING_ID = "postgres" -_BINDING_ID = re.compile(r"^[A-Za-z0-9_.:-]{1,128}$") -_CREDENTIAL_REF = re.compile(r"^[A-Za-z0-9_.:-]{1,128}$") _NATIVE_VERSION_REF_MAX_LENGTH = 1024 #: Upper bound for adopting an existing object whose encoding may differ byte for byte. _MAX_ADOPTED_OBJECT_BYTES = 2 * REPORTING_ROW_CHUNK_MAX_BYTES + 1024 * 1024 @@ -141,7 +158,8 @@ class ReportingObjectRowBindingV1: key_template: str | None = None #: Stored encoding. Digests always cover uncompressed bytes. compression: Compression = "gzip" - #: Host-mapped client reference passed to the provider; never a secret. + #: Host-mapped client reference passed to the provider; never a secret. 1 to + #: ``REPORTING_ROW_CREDENTIAL_REF_MAX_LENGTH`` (255) characters of ``A-Za-z0-9_.:-``. credential_ref: str | None = None #: Keep this binding configured only to read (and retire) row sets already written through #: it, for example under an older template without ``{revision_key}``. A read-only binding is @@ -171,8 +189,9 @@ class ReportingRowStorageOptionsV1: ``bindings`` maps a logical binding ID to its immutable definition. Changing a definition requires a new ID; keep retired definitions configured while - revisions still reference them. ``deployment_namespace`` is combined with the - schema's installation identity into each binding's ``namespace_key``, so two + revisions still reference them. An ID is 1 to ``REPORTING_ROW_BINDING_ID_MAX_LENGTH`` (255) + characters of ``A-Za-z0-9_.:-``, the AdCP identifier bound. ``deployment_namespace`` is + combined with the schema's installation identity into each binding's ``namespace_key``, so two SDKs on one database must use the same value. ``select_row_binding`` picks the binding for each new row set; it defaults to the only binding, or ``postgres``. """ @@ -352,6 +371,25 @@ def _assert_native_version_ref(value: object) -> None: ) +def recorded_intent_objects(created_objects: object) -> set[tuple[str, str]]: + """The ``(object_ref, native_version_ref)`` pairs a write intent recorded. + + Reads both the current keys and an older writer's pre-alignment ``objectKey`` and + ``nativeVersion``, which a writer still running during a rolling upgrade records. + """ + pairs: set[tuple[str, str]] = set() + if not isinstance(created_objects, list): + return pairs + for item in created_objects: + if not isinstance(item, dict): + continue + object_ref = item.get("object_ref", item.get("objectKey")) + version = item.get("native_version_ref", item.get("nativeVersion")) + if isinstance(object_ref, str) and isinstance(version, str): + pairs.add((object_ref, version)) + return pairs + + def _inflate(physical: bytes, compression: Compression, byte_count: int) -> bytes | None: """Decompress to exactly ``byte_count`` bytes, refusing trailing data and extra members.""" if compression == "none": @@ -371,6 +409,28 @@ def _inflate(physical: bytes, compression: Compression, byte_count: int) -> byte return None +@dataclass(frozen=True, slots=True) +class ReportingObjectBindingSummaryV1: + """An object binding this host configures, as the orphan collector sees it.""" + + id: str + read_only: bool + #: Whether the binding's provider implements ``list``. + listable: bool + + +@dataclass(frozen=True, slots=True) +class ReportingRowObjectListingV1: + """One page of the objects under a binding's owned prefix, and nothing else.""" + + owned_prefix: str + objects: tuple[ReportingRowObjectListEntryV1, ...] + #: Entries dropped: outside the owned prefix, malformed, or left out by the provider as + #: unaddressable. + foreign: int + next_page_token: str | None = None + + class ReportingRowStorageV1: """Reads and writes revision row sets over a caller-owned connection. @@ -386,9 +446,14 @@ def __init__(self, options: ReportingRowStorageOptionsV1 | None = None) -> None: raise ReportingRowStoreError("INVALID_INPUT", "at least one row binding is required") providers = dict(options.providers or {}) for binding_id, binding in bindings.items(): - if not _BINDING_ID.match(binding_id): + if not is_row_identifier(binding_id): + # Quoted and truncated, so a hostile ID is not echoed whole into logs. + text = str(binding_id) + shown = text if len(text) <= 64 else f"{text[:64]}..." raise ReportingRowStoreError( - "INVALID_INPUT", f"invalid row binding ID {binding_id}" + "INVALID_INPUT", + f"invalid row binding ID {json.dumps(shown)}: use 1 to " + f"{REPORTING_ROW_BINDING_ID_MAX_LENGTH} characters of A-Za-z0-9_.:-", ) if isinstance(binding, ReportingPostgresRowBindingV1): continue @@ -408,11 +473,11 @@ def __init__(self, options: ReportingRowStorageOptionsV1 | None = None) -> None: raise ReportingRowStoreError( "INVALID_INPUT", f"row binding {binding_id} has an unsupported compression" ) - if binding.credential_ref is not None and not _CREDENTIAL_REF.match( - binding.credential_ref - ): + if binding.credential_ref is not None and not is_row_identifier(binding.credential_ref): raise ReportingRowStoreError( - "INVALID_INPUT", f"row binding {binding_id} has an invalid credential_ref" + "INVALID_INPUT", + f"row binding {binding_id} has an invalid credential_ref: use 1 to " + f"{REPORTING_ROW_CREDENTIAL_REF_MAX_LENGTH} characters of A-Za-z0-9_.:-", ) namespace = options.deployment_namespace if not namespace or len(namespace) > 256: @@ -1045,15 +1110,342 @@ async def delete_objects(self, binding_id: str, objects: Sequence[tuple[str, str self._options.write_deadline_seconds, ) + # -- listing and the collector fence ---------------------------------- + + def object_binding_ids(self) -> list[str]: + """The object bindings this host configures, read-only ones included.""" + return [ + binding_id + for binding_id, binding in self._bindings.items() + if isinstance(binding, ReportingObjectRowBindingV1) + ] + + def object_binding_summaries(self) -> list[ReportingObjectBindingSummaryV1]: + """Each object binding: whether it is ``read_only`` and whether its provider can list.""" + return [ + ReportingObjectBindingSummaryV1( + binding_id, + binding.read_only, + callable(getattr(self._providers.get(binding.provider), "list", None)), + ) + for binding_id, binding in self._bindings.items() + if isinstance(binding, ReportingObjectRowBindingV1) + ] + + def _lister_binding( + self, binding_id: str + ) -> tuple[ReportingObjectRowBindingV1, ReportingRowObjectProviderV1, Any]: + """The binding, its provider and the provider's ``list``; refuses one that cannot list.""" + if self._installation is None: + raise ReportingRowStoreError("STATE_UNAVAILABLE", "row storage is not ready") + binding = self._bindings.get(binding_id) + if not isinstance(binding, ReportingObjectRowBindingV1): + raise ReportingRowStoreError( + "STATE_UNAVAILABLE", f"row binding {binding_id} is not configured for objects" + ) + provider = self._providers[binding.provider] + lister = getattr(provider, "list", None) + if not callable(lister): + raise ReportingRowStoreError( + "INVALID_INPUT", f"row binding {binding_id}'s provider cannot list objects" + ) + return binding, provider, lister + + async def list_objects( + self, binding_id: str, *, page_size: int, page_token: str | None = None + ) -> ReportingRowObjectListingV1: + """One page of the objects under a binding's owned prefix, ``{prefix}/{namespace_key}/``. + + An entry the provider returns outside it, or one that is not a well-formed object + reference, is dropped and counted in ``foreign`` instead of being returned. The collector's + own fence chain lives under the owned prefix; it is not a row object and not foreign. + Raises ``INVALID_INPUT`` when the binding's provider has no ``list``. + """ + binding, _, lister = self._lister_binding(binding_id) + assert self._installation is not None + owned_prefix = f"{binding.prefix}/{self._installation[1]}/" + page = await self._provider_call( + lister( + ReportingRowObjectListInputV1( + binding.location, owned_prefix, page_size, page_token, binding.credential_ref + ), + ReportingRowObjectContextV1(self._options.read_deadline_seconds), + ), + self._options.read_deadline_seconds, + ) + if page is None or not isinstance(getattr(page, "objects", None), (tuple, list)): + raise ReportingRowStoreError( + "PROVIDER_UNAVAILABLE", "provider returned an invalid listing" + ) + token = getattr(page, "next_page_token", None) + if token is not None and (not isinstance(token, str) or not token): + raise ReportingRowStoreError( + "PROVIDER_UNAVAILABLE", "provider returned an invalid page token" + ) + if len(page.objects) > page_size: + raise ReportingRowStoreError( + "PROVIDER_UNAVAILABLE", "provider returned more objects than the page size" + ) + skipped = getattr(page, "skipped", 0) + foreign = skipped if type(skipped) is int and skipped > 0 else 0 + objects: list[ReportingRowObjectListEntryV1] = [] + for entry in page.objects: + object_ref = getattr(entry, "object_ref", None) + if isinstance(object_ref, str) and is_orphan_fence_key(owned_prefix, object_ref): + continue + version = getattr(entry, "native_version_ref", None) + size = getattr(entry, "size_bytes", None) + if ( + not is_row_object_key_shape(object_ref, REPORTING_ROW_OBJECT_REF_MAX_LENGTH) + or not isinstance(object_ref, str) + or not object_ref.startswith(owned_prefix) + or not isinstance(version, str) + or not 1 <= len(version) <= _NATIVE_VERSION_REF_MAX_LENGTH + or type(size) is not int + or size < 0 + or not is_row_instant(getattr(entry, "created_at", None)) + ): + foreign += 1 + continue + objects.append(entry) + return ReportingRowObjectListingV1(owned_prefix, tuple(objects), foreign, token) + + async def collector_fence_location(self, binding_id: str) -> tuple[str, str, str]: + """``(scope, directory, namespace_key)`` of an object binding's collector fence. + + ``scope`` keys the database row; ``directory`` holds the chain in the object store. + Bindings that share provider, location, prefix and installation share a fence. Raises + ``STATE_UNAVAILABLE`` for a binding that is not an object binding and ``INVALID_INPUT`` + when its provider cannot list (the fence is read by listing). + """ + binding, provider, _ = self._lister_binding(binding_id) + assert self._installation is not None + namespace_key = self._installation[1] + return ( + orphan_fence_scope( + # The provider's own name, not this host's registry key: two registry entries of + # one kind that reach the same bucket and prefix are one physical fence. + provider=provider.name, + location=binding.location, + prefix=binding.prefix, + namespace_key=namespace_key, + ), + orphan_fence_directory(f"{binding.prefix}/{namespace_key}/"), + namespace_key, + ) + + async def list_collector_fence( + self, binding_id: str, *, tolerant: bool = False + ) -> list[OrphanFenceEntry]: + """Every fence object of a binding's namespace, ordered by epoch. + + Keys under the fence directory that are not an epoch name are ignored. Two live versions + of one epoch's key (a versioned store after a manual delete and re-create) are ambiguous + and fail closed with ``ROWS_INTEGRITY_FAILED``, as does a chain too long to be one this + collector wrote (it prunes to two objects). ``tolerant`` is for a takeover, which must be + able to supersede a damaged chain: every version of every epoch is returned, so it can all + be removed, and the page bound is wider. + """ + _, directory, _ = await self.collector_fence_location(binding_id) + binding, _, lister = self._lister_binding(binding_id) + assert self._installation is not None + owned_prefix = f"{binding.prefix}/{self._installation[1]}/" + entries: dict[int, OrphanFenceEntry] = {} + versions: list[OrphanFenceEntry] = [] + page_size = 50 + token: str | None = None + page_number = 0 + while True: + if page_number >= (200 if tolerant else 20): + raise ReportingRowStoreError( + "ROWS_INTEGRITY_FAILED", "the collector fence directory holds too many objects" + ) + page_number += 1 + listed = await self._provider_call( + lister( + ReportingRowObjectListInputV1( + binding.location, directory, page_size, token, binding.credential_ref + ), + ReportingRowObjectContextV1(self._options.read_deadline_seconds), + ), + self._options.read_deadline_seconds, + ) + if ( + listed is None + or not isinstance(getattr(listed, "objects", None), (tuple, list)) + or len(listed.objects) > page_size + ): + raise ReportingRowStoreError( + "PROVIDER_UNAVAILABLE", "provider returned an invalid listing" + ) + skipped = getattr(listed, "skipped", 0) + if not tolerant and type(skipped) is int and skipped > 0: + # The provider left out an entry it could not address or age. In a directory that + # holds only this collector's own objects that could be the newest claim, and a + # check cannot tell "no claim" from "a claim I cannot see": fail closed. + raise ReportingRowStoreError( + "ROWS_INTEGRITY_FAILED", "the collector fence listing skipped an object" + ) + for item in listed.objects: + object_ref = getattr(item, "object_ref", None) + version = getattr(item, "native_version_ref", None) + if not isinstance(object_ref, str) or not isinstance(version, str): + continue + if not 1 <= len(version) <= _NATIVE_VERSION_REF_MAX_LENGTH: + continue + epoch = orphan_fence_epoch_of_key(owned_prefix, object_ref) + if epoch is None or object_ref != orphan_fence_key(owned_prefix, epoch): + continue + entry = OrphanFenceEntry(epoch, object_ref, version) + if tolerant: + versions.append(entry) + if epoch in entries: + if tolerant: + continue + raise ReportingRowStoreError( + "ROWS_INTEGRITY_FAILED", + "the collector fence holds two versions of one epoch", + ) + entries[epoch] = entry + following = getattr(listed, "next_page_token", None) + if following is None: + break + if not isinstance(following, str) or not following or following == token: + raise ReportingRowStoreError( + "PROVIDER_UNAVAILABLE", "provider returned an invalid page token" + ) + token = following + return sorted(versions if tolerant else entries.values(), key=lambda item: item.epoch) + + async def read_collector_fence( + self, binding_id: str, entry: OrphanFenceEntry + ) -> OrphanFenceRecord: + """Read and verify one fence object. + + A missing version is ``PROVIDER_UNAVAILABLE`` (it was listed a moment ago, so this is a + race to retry); content that is not exactly a fence of this namespace and epoch is + ``ROWS_INTEGRITY_FAILED``. + """ + binding, provider = self._object_binding(binding_id) + assert self._installation is not None + try: + data = await self._provider_call( + provider.get( + ReportingRowObjectGetInputV1( + binding.location, + entry.object_ref, + entry.native_version_ref, + ORPHAN_FENCE_MAX_BYTES, + None, + binding.credential_ref, + ), + ReportingRowObjectContextV1(self._options.read_deadline_seconds), + ), + self._options.read_deadline_seconds, + ) + except ReportingRowStoreError as error: + # An object over the read cap is the same finding as bad content. + if error.code == "ROWS_INTEGRITY_FAILED": + raise ReportingRowStoreError( + "ROWS_INTEGRITY_FAILED", "the collector fence object is malformed" + ) from None + raise + if data is None: + raise ReportingRowStoreError( + "PROVIDER_UNAVAILABLE", "a collector fence object disappeared while it was read" + ) + record = parse_orphan_fence_record( + data, namespace_key=self._installation[1], epoch=entry.epoch + ) + if record is None: + raise ReportingRowStoreError( + "ROWS_INTEGRITY_FAILED", "the collector fence object is malformed" + ) + return record + + async def put_collector_fence( + self, binding_id: str, record: OrphanFenceRecord + ) -> tuple[bool, OrphanFenceRecord]: + """Create-only write of one epoch's fence object. + + When the key is taken, returns the record already there (verified), so the caller can tell + its own earlier write from another database's claim: ``(created, record)``. + """ + binding, provider = self._object_binding(binding_id) + assert self._installation is not None + installation_id, namespace_key = self._installation + object_ref = orphan_fence_key(f"{binding.prefix}/{namespace_key}/", record.epoch) + put = await self._provider_call( + provider.put_if_absent( + ReportingRowObjectPutInputV1( + binding.location, + object_ref, + encode_orphan_fence_record(namespace_key, record), + "application/x-ndjson", + { + "adcp-installation": installation_id, + "adcp-collector-fence": str(record.epoch), + }, + binding.credential_ref, + ), + ReportingRowObjectContextV1(self._options.write_deadline_seconds), + ), + self._options.write_deadline_seconds, + ) + _assert_native_version_ref(put.native_version_ref) + if put.created: + return True, record + return False, await self.read_collector_fence( + binding_id, OrphanFenceEntry(record.epoch, object_ref, put.native_version_ref) + ) + + async def delete_collector_fence( + self, binding_id: str, entries: Sequence[OrphanFenceEntry] + ) -> None: + """Delete superseded fence objects. Best effort: a failure leaves a stale object that the + next claim prunes.""" + if not isinstance(self._bindings.get(binding_id), ReportingObjectRowBindingV1): + return + binding, provider = self._object_binding(binding_id) + context = ReportingRowObjectContextV1(self._options.write_deadline_seconds) + for entry in entries: + try: + await self._provider_call( + provider.delete( + ReportingRowObjectDeleteInputV1( + binding.location, + entry.object_ref, + entry.native_version_ref, + binding.credential_ref, + ), + context, + ), + self._options.write_deadline_seconds, + ) + except ReportingRowStoreError: + continue + # -- object lifetime -------------------------------------------------- # - # Chunks are adopted create-only, so a custom key template that omits an identity - # placeholder lets two row sets name the same (binding, key, version). Every path that - # deletes an object and every commit that adopts one serialize on a transaction-scoped - # advisory lock per object: the deleter checks references and deletes under it, and the - # committer re-verifies the adopted object under it, so whichever runs second sees the - # other's effect. The lock text is shared with the JavaScript SDK and hashed in SQL so - # both SDKs derive the same lock id in one database. + # The cross-SDK contract (adcp-client#3606, adcp-client-python#1492). The per-object advisory + # lock ``adcp-reporting-object:{binding}:{object_ref}`` is kept in exactly two places that pair + # with each other, because nothing else orders them: the orphan collector's delete + # (``delete_orphaned_object``) and a commit that adopts an object its own intent did not record + # (``lock_and_verify_adopted``). Retirement, the intent sweep and a commit that adopts an object + # its own intent recorded take no object lock and hold no connection across a provider delete. + # Lock order everywhere: the account lock, then object locks. The lock text is hashed in SQL so + # both SDKs derive one lock id from it in one database. + # + # Fences that exist without the lock: (A) the account lock held by a commit, retirement phase 1, + # an intent takeover and the sweep claim; (B) ``rows_state`` moves to ``pruning`` under the + # account lock before any delete, and ``pruning`` row sets neither count as references nor can + # be shared; (C) the intent moves ``open -> sweeping`` against the commit's ``DELETE ... WHERE + # state = 'open'``, in the same transaction as the chunk insert, so a reader sees the open + # intent or the chunk rows, never neither; (D) deletes name an exact native version, so native + # versions must be write-unique; (E) every writable key includes ``{revision_key}`` and + # ``{content_sha256}``, so only one row set can name a key through a writable binding. + # Precondition, documented and not enforced: two writable bindings must not share one bucket and + # prefix, because the reference check is per binding outside the collector. @staticmethod def _object_lock_text(binding_id: str, object_ref: str) -> str: @@ -1064,19 +1456,33 @@ async def lock_and_verify_adopted( connection: Any, binding_id: str, locators: Sequence[ReportingRowChunkLocatorV1], + recorded: Collection[tuple[str, str]] = (), ) -> None: - """Hold the object lock for every adopted object and prove each still exists. - - Call inside the ledger transaction, after the account lock and before the chunk rows - are inserted. Objects this upload created are protected by its open write intent - instead. (A writer's own just-created object is not locked: the window between its - create and the intent record is the same one the JavaScript SDK leaves open.) + """Prove every adopted object is still there, locking the ones no record names. + + Call inside the ledger transaction, after the account lock and the intent close, before + the chunk rows are inserted. ``recorded`` is the ``(object_ref, native_version_ref)`` pairs + the closed intent listed in ``created_objects`` (``recorded_intent_objects``). + + An object the writer's own intent recorded as created (an earlier attempt at this row set + that was taken over) needs no lock: until this transaction commits the open intent names + it, so no delete skips the reference check, the sweep cannot claim an intent this commit + just closed or one a takeover renewed, and the key is unique to this row set. An adopted + object the intent does not record (an earlier attempt's PUT that landed before it was + recorded, or one created after a takeover) is named by no record; only the orphan collector + deletes such an object, and it deletes under the per-object lock, so take it here, in key + order, until this transaction ends: a delete that held it finished first and the + version-pinned ``stat`` fails this commit, and any later delete sees the chunk rows + inserted next and skips. (A writer's own just-created object is not locked: the window + between its create and the intent record is the same one the JavaScript SDK leaves open.) """ adopted = sorted((x for x in locators if not x.created), key=lambda x: x.object_ref) if not adopted: return binding, provider = self._object_binding(binding_id) for locator in adopted: + if (locator.object_ref, locator.native_version_ref) in recorded: + continue await connection.execute( "SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (self._object_lock_text(binding_id, locator.object_ref),), @@ -1138,67 +1544,121 @@ async def delete_unreferenced( ) -> int: """Delete each object nothing outside the operation's scope still needs. - An object is kept while a chunk of a live or unavailable row set of the same binding - references it (outside ``obligation_id`` / ``row_set_id``, the thing being deleted), - or another open write intent created it. Row sets already pruning or pruned do not - count: their own operation deletes the object. One transaction per object: the lock, - the reference check and the provider delete happen together, so a concurrent commit - adopting the object either already references it (it is kept) or finds it gone and - fails closed. Returns the number of objects deleted. + Takes no advisory lock and holds no transaction or pooled connection across the provider + delete: per object, one statement checks the references, then the exact version is + deleted. An object is kept while a chunk of a live or unavailable row set of the same + binding references it (outside ``obligation_id`` / ``row_set_id``, the thing being + deleted), or another open write intent created it. Row sets already ``pruning`` or + ``pruned`` do not count: their own operation deletes the object, and counting them would + let two concurrent retirements each skip the object and leak it. Each deleter commits + ``pruning`` before its check, so for any two at least one deletes. See the contract above + (``# -- object lifetime``) for why no lock is needed against a committing writer. A skipped + object stays for the row set that owns it: its own retirement or sweep deletes it. + Returns the number of objects deleted. """ deleted = 0 for object_ref, native_version_ref in objects: - async with pool.connection() as connection, connection.transaction(): + async with pool.connection() as connection: + referenced = await self._is_referenced( + connection, + binding_id, + object_ref, + native_version_ref, + obligation_id=obligation_id, + row_set_id=row_set_id, + ) + if referenced: + continue + await self.delete_objects(binding_id, [(object_ref, native_version_ref)]) + deleted += 1 + return deleted + + async def delete_orphaned_object( + self, pool: Any, binding_id: str, object_ref: str, native_version_ref: str + ) -> bool: + """Delete one object no ledger record named when the collector listed it. + + Under the advisory lock ``adcp-reporting-object:{binding}:{object_ref}`` of every binding + that can name it: the only delete that takes the lock. An object no record names can be + adopted, unrecorded, by a writer's commit (a retry that finds an object an earlier attempt + created but never recorded), and that commit takes the same lock before it proves the + object and inserts its chunk rows. So either this delete wins and the commit's + version-pinned ``stat`` fails it cleanly, or the commit wins and this check sees its chunk + rows and skips. A writer takes the lock of its own binding only, so every binding (those + this host configures and those the database has registered, which can share a bucket and + prefix) is locked, sorted by binding ID. The reference check then runs across all bindings. + Lock order: the account lock, then object locks; this never waits on the account lock while + holding an object lock. Returns whether the object was deleted. + """ + async with pool.connection() as connection, connection.transaction(): + known = await ( + await connection.execute("SELECT row_binding_id FROM adcp_reporting_row_bindings") + ).fetchall() + for lock_binding in sorted( + {binding_id, *self.object_binding_ids(), *(r[0] for r in known)} + ): await connection.execute( "SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", - (self._object_lock_text(binding_id, object_ref),), + (self._object_lock_text(lock_binding, object_ref),), ) - referenced = await ( - await connection.execute( - "(SELECT 1 FROM adcp_reporting_row_chunks chunk" - " JOIN adcp_reporting_row_sets row_set" - " ON row_set.row_set_id = chunk.row_set_id" - " WHERE row_set.row_binding_id = %(binding)s" - " AND chunk.object_ref = %(key)s" - " AND chunk.native_version_ref = %(version)s" - " AND row_set.rows_state IN ('live', 'unavailable')" - " AND (%(obligation)s::text IS NULL" - " OR chunk.obligation_id <> %(obligation)s)" - " AND (%(row_set)s::text IS NULL OR chunk.row_set_id <> %(row_set)s)" - " LIMIT 1)" - " UNION ALL" - " (SELECT 1 FROM adcp_reporting_row_write_intents intent" - " WHERE intent.row_binding_id = %(binding)s AND intent.state = 'open'" - " AND (intent.created_objects @> %(created)s::jsonb" - " OR intent.created_objects @> %(created_legacy)s::jsonb)" - " LIMIT 1)", - { - "binding": binding_id, - "key": object_ref, - "version": native_version_ref, - "obligation": obligation_id, - "row_set": row_set_id, - "created": json.dumps( - [ - { - "object_ref": object_ref, - "native_version_ref": native_version_ref, - } - ] - ), - # A writer from before the AdCP field alignment, still running during a - # rolling upgrade, records the earlier keys. - "created_legacy": json.dumps( - [{"objectKey": object_ref, "nativeVersion": native_version_ref}] - ), - }, - ) - ).fetchone() - if referenced is not None: - continue - await self.delete_objects(binding_id, [(object_ref, native_version_ref)]) - deleted += 1 - return deleted + if await self._is_referenced( + connection, binding_id, object_ref, native_version_ref, any_binding=True + ): + return False + await self.delete_objects(binding_id, [(object_ref, native_version_ref)]) + return True + + async def _is_referenced( + self, + connection: Any, + binding_id: str, + object_ref: str, + native_version_ref: str, + *, + obligation_id: str | None = None, + row_set_id: str | None = None, + any_binding: bool = False, + ) -> bool: + """Whether a live or unavailable row set outside the scope, or an open write intent, still + names this exact object version. ``any_binding`` widens the check to every binding.""" + found = await ( + await connection.execute( + "(SELECT 1 FROM adcp_reporting_row_chunks chunk" + " JOIN adcp_reporting_row_sets row_set" + " ON row_set.row_set_id = chunk.row_set_id" + " WHERE (%(any_binding)s::boolean OR row_set.row_binding_id = %(binding)s)" + " AND chunk.object_ref = %(key)s" + " AND chunk.native_version_ref = %(version)s" + " AND row_set.rows_state IN ('live', 'unavailable')" + " AND (%(obligation)s::text IS NULL OR chunk.obligation_id <> %(obligation)s)" + " AND (%(row_set)s::text IS NULL OR chunk.row_set_id <> %(row_set)s)" + " LIMIT 1)" + " UNION ALL" + " (SELECT 1 FROM adcp_reporting_row_write_intents intent" + " WHERE (%(any_binding)s::boolean OR intent.row_binding_id = %(binding)s)" + " AND intent.state = 'open'" + " AND (intent.created_objects @> %(created)s::jsonb" + " OR intent.created_objects @> %(created_legacy)s::jsonb)" + " LIMIT 1)", + { + "binding": binding_id, + "any_binding": any_binding, + "key": object_ref, + "version": native_version_ref, + "obligation": obligation_id, + "row_set": row_set_id, + "created": json.dumps( + [{"object_ref": object_ref, "native_version_ref": native_version_ref}] + ), + # A writer from before the AdCP field alignment, still running during a + # rolling upgrade, records the earlier keys. + "created_legacy": json.dumps( + [{"objectKey": object_ref, "nativeVersion": native_version_ref}] + ), + }, + ) + ).fetchone() + return found is not None # -- read ------------------------------------------------------------- diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_azure.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_azure.py index a2d12e811c..f585a7f2c2 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_azure.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_azure.py @@ -24,6 +24,7 @@ import re import uuid from collections.abc import Mapping +from datetime import datetime from typing import Any, Literal from adcp.reporting.ledger.row_storage_errors import ReportingRowStoreError @@ -31,11 +32,15 @@ ReportingRowObjectContextV1, ReportingRowObjectDeleteInputV1, ReportingRowObjectGetInputV1, + ReportingRowObjectListEntryV1, + ReportingRowObjectListInputV1, + ReportingRowObjectListPageV1, ReportingRowObjectProviderV1, ReportingRowObjectPutInputV1, ReportingRowObjectPutResultV1, ReportingRowObjectStatInputV1, ReportingRowObjectStatResultV1, + format_row_instant, ) from adcp_reporting_storage.ledger.row_storage_provider_io import ( assert_location_keys, @@ -366,6 +371,69 @@ def _stat(blob: Any, version: str) -> ReportingRowObjectStatResultV1 | None: raise ReportingRowStoreError("PROVIDER_UNAVAILABLE", "blob metadata is malformed") return ReportingRowObjectStatResultV1(version, size) + async def list( + self, request: ReportingRowObjectListInputV1, context: ReportingRowObjectContextV1 + ) -> ReportingRowObjectListPageV1: + container = _Budgeted( + self._container(request.location, request.credential_ref), context.timeout_seconds + ) + if not isinstance(request.prefix, str) or len(request.prefix) > 1024: + raise ReportingRowStoreError("INVALID_INPUT", "list prefix is invalid") + if type(request.page_size) is not int or request.page_size < 1: + raise ReportingRowStoreError("INVALID_INPUT", "page_size must be a positive integer") + return await asyncio.to_thread(self._list, container, request) + + @staticmethod + def _list( + container: Any, request: ReportingRowObjectListInputV1 + ) -> ReportingRowObjectListPageV1: + # Versioned listing: every (name, version_id) a delete could name, the current version + # included. + pages = container.list_blobs( + name_starts_with=request.prefix, + include=["versions"], + results_per_page=min(request.page_size, 5000), + ).by_page(continuation_token=request.page_token) + page = next(pages, None) + objects: list[ReportingRowObjectListEntryV1] = [] + skipped = 0 + for item in list(page) if page is not None else []: + name = getattr(item, "name", None) + version = getattr(item, "version_id", None) + size = getattr(item, "size", None) + # The later of creation and last modification: an overwrite can move the second + # without the first. + times = [ + moment + for moment in ( + getattr(item, "creation_time", None), + getattr(item, "last_modified", None), + ) + if isinstance(moment, datetime) and moment.tzinfo is not None + ] + # An entry that cannot be addressed or aged is left out: it is never collected, and + # one bad entry must not stop the listing of the rest. A blob written before + # versioning was enabled has no version ID and is never addressable by a recorded + # one. + if ( + not isinstance(name, str) + or not name.startswith(request.prefix) + or not isinstance(version, str) + or not _VERSION_ID.fullmatch(version) + or type(size) is not int + or size < 0 + or not times + ): + skipped += 1 + continue + objects.append( + ReportingRowObjectListEntryV1(name, version, size, format_row_instant(max(times))) + ) + token = getattr(pages, "continuation_token", None) + return ReportingRowObjectListPageV1( + tuple(objects), token if isinstance(token, str) and token else None, skipped + ) + async def delete( self, delete: ReportingRowObjectDeleteInputV1, context: ReportingRowObjectContextV1 ) -> Literal["deleted", "absent"]: diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_gcs.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_gcs.py index 782465687d..2c1bff8e87 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_gcs.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_gcs.py @@ -17,6 +17,7 @@ import asyncio import re from collections.abc import Mapping +from datetime import datetime from typing import Any, Literal from adcp.reporting.ledger.row_storage_errors import ReportingRowStoreError @@ -24,11 +25,15 @@ ReportingRowObjectContextV1, ReportingRowObjectDeleteInputV1, ReportingRowObjectGetInputV1, + ReportingRowObjectListEntryV1, + ReportingRowObjectListInputV1, + ReportingRowObjectListPageV1, ReportingRowObjectProviderV1, ReportingRowObjectPutInputV1, ReportingRowObjectPutResultV1, ReportingRowObjectStatInputV1, ReportingRowObjectStatResultV1, + format_row_instant, ) from adcp_reporting_storage.ledger.row_storage_provider_io import ( assert_key, @@ -294,6 +299,68 @@ def _stat( raise ReportingRowStoreError("PROVIDER_UNAVAILABLE", "object metadata is malformed") return ReportingRowObjectStatResultV1(str(generation), size) + async def list( + self, request: ReportingRowObjectListInputV1, context: ReportingRowObjectContextV1 + ) -> ReportingRowObjectListPageV1: + bucket = self._bucket(request.location, request.credential_ref) + if not isinstance(request.prefix, str) or len(request.prefix.encode()) > 1024: + raise ReportingRowStoreError("INVALID_INPUT", "list prefix is invalid") + if type(request.page_size) is not int or request.page_size < 1: + raise ReportingRowStoreError("INVALID_INPUT", "page_size must be a positive integer") + return await asyncio.to_thread(self._list, bucket, request, context.timeout_seconds) + + @staticmethod + def _list( + bucket: Any, request: ReportingRowObjectListInputV1, timeout: float + ) -> ReportingRowObjectListPageV1: + # ``versions=True`` lists every generation, noncurrent ones included, so each + # (key, generation) a delete could name is visible. On a bucket without versioning it + # lists the live objects. + iterator = bucket.list_blobs( + prefix=request.prefix, + versions=True, + max_results=min(request.page_size, 1000), + page_token=request.page_token, + timeout=timeout, + ) + page = next(iter(iterator.pages), None) + objects: list[ReportingRowObjectListEntryV1] = [] + skipped = 0 + for blob in list(page) if page is not None else []: + name = getattr(blob, "name", None) + size = getattr(blob, "size", None) + generation = str(getattr(blob, "generation", "") or "") + created = getattr(blob, "time_created", None) + updated = getattr(blob, "updated", None) + # The later of the generation's creation and last metadata update: an object is + # never reported older than it is. + times = [ + moment + for moment in (created, updated) + if isinstance(moment, datetime) and moment.tzinfo is not None + ] + # An entry that cannot be addressed or aged is left out: it is never collected, and + # one bad entry must not stop the listing of the rest. + if ( + not isinstance(name, str) + or not name.startswith(request.prefix) + or type(size) is not int + or size < 0 + or not times + or not _GENERATION.fullmatch(generation) + ): + skipped += 1 + continue + objects.append( + ReportingRowObjectListEntryV1( + name, generation, size, format_row_instant(max(times)) + ) + ) + token = getattr(iterator, "next_page_token", None) + return ReportingRowObjectListPageV1( + tuple(objects), token if isinstance(token, str) and token else None, skipped + ) + async def delete( self, delete: ReportingRowObjectDeleteInputV1, context: ReportingRowObjectContextV1 ) -> Literal["deleted", "absent"]: diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_object.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_object.py index ecb4f18cdf..704d726967 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_object.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_object.py @@ -22,14 +22,20 @@ import uuid from collections.abc import Mapping from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path -from typing import Literal, Protocol, runtime_checkable +from typing import Any, Literal, Protocol, runtime_checkable import rfc8785 from adcp.reporting.ledger.row_storage_errors import ReportingRowStoreError __all__ = [ + "ReportingRowObjectListEntryV1", + "ReportingRowObjectListInputV1", + "ReportingRowObjectListPageV1", + "REPORTING_ROW_BINDING_ID_MAX_LENGTH", + "REPORTING_ROW_CREDENTIAL_REF_MAX_LENGTH", "REPORTING_ROW_DEFAULT_KEY_TEMPLATE", "REPORTING_ROW_KEY_PLACEHOLDERS", "REPORTING_ROW_NATIVE_VERSION_REF_MAX_LENGTH", @@ -65,6 +71,13 @@ class ReportingRowObjectContextV1: #: characters. The row-storage ``CHECK`` constraints enforce the same. REPORTING_ROW_OBJECT_REF_MAX_LENGTH = 1024 REPORTING_ROW_NATIVE_VERSION_REF_MAX_LENGTH = 1024 +#: Bound on a row binding ID, 1 to 255 characters of ``A-Za-z0-9_.:-``: the AdCP identifier +#: bound that ``reporting_revision_id``, ``resource_ref`` and ``destination_ref`` use +#: (``^[A-Za-z0-9_.:-]{1,255}$``). The ``adcp_reporting_row_bindings`` ``CHECK`` enforces the same. +REPORTING_ROW_BINDING_ID_MAX_LENGTH = 255 +#: Bound on a binding's ``credential_ref`` (and on the keys of a provider's ``clients`` map): the +#: same 1 to 255 characters of ``A-Za-z0-9_.:-`` as a row binding ID. +REPORTING_ROW_CREDENTIAL_REF_MAX_LENGTH = 255 @dataclass(frozen=True, slots=True) @@ -124,6 +137,43 @@ class ReportingRowObjectStatResultV1: size_bytes: int +@dataclass(frozen=True, slots=True) +class ReportingRowObjectListInputV1: + location: Mapping[str, str] + #: Key prefix to list under. The SDK passes the binding's owned prefix, + #: ``{prefix}/{namespace_key}/``. + prefix: str + #: Upper bound on objects in the page. A provider may return fewer and still return a token. + page_size: int + #: Opaque continuation from the previous page; ``None`` for the first page. + page_token: str | None = None + credential_ref: str | None = None + + +@dataclass(frozen=True, slots=True) +class ReportingRowObjectListEntryV1: + """One stored object version found by a listing.""" + + #: AdCP ``object_ref``: the key, as it would be passed to ``get`` and ``delete``. + object_ref: str + #: AdCP ``native_version_ref``: the exact version, as ``put_if_absent`` would have recorded it. + native_version_ref: str + size_bytes: int + #: When this version was written, as an RFC 3339 UTC instant (``...Z``). Never earlier than + #: the write. + created_at: str + + +@dataclass(frozen=True, slots=True) +class ReportingRowObjectListPageV1: + objects: tuple[ReportingRowObjectListEntryV1, ...] + #: Absent on the last page. + next_page_token: str | None = None + #: Entries the provider left out because they cannot be addressed or aged (no usable version, + #: size or write time). They are never collected; one bad entry must not stop the listing. + skipped: int = 0 + + @runtime_checkable class ReportingRowObjectProviderV1(Protocol): """The port a row object store implements. @@ -183,6 +233,21 @@ async def delete( # versions are write-unique, so the same version at the recorded size is the same bytes) # without downloading it under the account lock. A provider without ``stat`` falls back to # a full read there. Never return a size for a different version. + # + # async def list(self, request: ReportingRowObjectListInputV1, context) + # -> ReportingRowObjectListPageV1 + # + # Every stored version whose key starts with ``prefix``, in a stable order, one page at a + # time. The orphan collector (``collect_orphaned_row_objects``) and its ownership fence use + # it; a binding whose provider has no ``list`` is reported ``unsupported`` there. Return each + # version once as ``(object_ref, native_version_ref)`` exactly as ``put_if_absent`` recorded + # it (on a versioned store, noncurrent versions too, never delete markers), the stored size + # and the time the version was written. Return only keys under ``prefix``. A page may be + # short; ``next_page_token`` is absent on the last page. A token is only valid for the same + # prefix and location. Leave out an entry that cannot be addressed or aged and count it in + # ``skipped`` rather than failing the page. The listing must be strongly consistent for keys + # ``put_if_absent`` has already returned (read-after-write): the collector's fence reads the + # newest claim by listing. #: Placeholders allowed in object key templates. @@ -204,6 +269,54 @@ async def delete( _KEY = re.compile(r"^[A-Za-z0-9_.=-]+(?:/[A-Za-z0-9_.=-]+)*$") _PLACEHOLDER = re.compile(r"\{([a-z0-9_]+)\}") +_INSTANT = re.compile( + r"([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2})(?:\.([0-9]{1,9}))?Z" +) + + +def is_row_object_key_shape( + key: object, max_length: int = REPORTING_ROW_OBJECT_REF_MAX_LENGTH +) -> bool: + """Whether a key has the shape ``render_reporting_row_object_key_v1`` gives every row object. + + The same character set, no empty, ``.`` or ``..`` segment, and within AdCP's ``object_ref`` + bound. ``fullmatch``: ``$`` would also accept a trailing newline. + """ + return ( + isinstance(key, str) + and len(key) <= max_length + and _KEY.fullmatch(key) is not None + and not any(segment in (".", "..") for segment in key.split("/")) + ) + + +def parse_row_instant(value: object) -> datetime | None: + """A strict RFC 3339 UTC instant (``2026-10-01T00:00:00.123Z``) as an aware datetime. + + Anything else, including an offset other than ``Z``, a date-only value or an impossible + date, is ``None``: an object whose write time cannot be read is treated as young, never + deleted on a guess. + """ + if not isinstance(value, str): + return None + match = _INSTANT.fullmatch(value) + if match is None: + return None + fraction = (match.group(2) or "").ljust(6, "0")[:6] + try: + return datetime.fromisoformat(f"{match.group(1)}.{fraction}+00:00") + except ValueError: + return None + + +def is_row_instant(value: object) -> bool: + return parse_row_instant(value) is not None + + +def format_row_instant(moment: datetime) -> str: + """An aware datetime as the millisecond RFC 3339 UTC instant providers report.""" + utc = moment.astimezone(timezone.utc) + return utc.strftime("%Y-%m-%dT%H:%M:%S.") + f"{utc.microsecond // 1000:03d}Z" _REVISION_KEY_REQUIRED = ( @@ -620,6 +733,101 @@ def _stat(target: str, native_version_ref: str) -> ReportingRowObjectStatResultV return None return ReportingRowObjectStatResultV1(version, int(version.rsplit(":", 1)[1])) + async def list( + self, request: ReportingRowObjectListInputV1, context: ReportingRowObjectContextV1 + ) -> ReportingRowObjectListPageV1: + """List objects under ``prefix`` in key order, one page at a time. + + Reads and sorts each directory it walks in full, so keep directories small: it is for + tests and single hosts, not millions of objects per prefix. + """ + if type(request.page_size) is not int or request.page_size < 1: + raise ReportingRowStoreError("INVALID_INPUT", "page_size must be a positive integer") + root = self.roots.get(request.location.get("root", "")) + if root is None: + raise ReportingRowStoreError("UNSAFE_BINDING", "filesystem root is not configured") + if not isinstance(request.prefix, str) or ".." in request.prefix.split("/"): + raise ReportingRowStoreError("INVALID_INPUT", "list prefix is invalid") + return await asyncio.to_thread(self._list, root, request) + + @staticmethod + def _list(root: str, request: ReportingRowObjectListInputV1) -> ReportingRowObjectListPageV1: + after = None if request.page_token is None else tuple(request.page_token.split("/")) + objects: list[ReportingRowObjectListEntryV1] = [] + more = False + + def walk(directory: tuple[str, ...]) -> bool: + nonlocal more + try: + with os.scandir(os.path.join(root, *directory)) as scanner: + # A symlink is neither listed nor followed: a link inside the root must not + # make a key resolve outside it. + entries = sorted( + (entry for entry in scanner if not entry.is_symlink()), + key=lambda entry: entry.name, + ) + except (FileNotFoundError, NotADirectoryError): + return True + except OSError: + raise ReportingRowStoreError( + "PROVIDER_UNAVAILABLE", "filesystem list failed" + ) from None + for entry in entries: + name = entry.name + segments = (*directory, name) + path = entry.path + is_directory = entry.is_dir(follow_symlinks=False) + if after is not None: + head = after[: len(segments)] + if segments < head: + continue + if segments == head and not is_directory and len(segments) == len(after): + continue + if is_directory: + # Keep only subtrees that can hold a key under the prefix. + rendered = "/".join(segments) + "/" + if not rendered.startswith(request.prefix) and not request.prefix.startswith( + rendered + ): + continue + if not walk(segments): + return False + continue + if name.endswith(".tmp") or not entry.is_file(follow_symlinks=False): + continue + key = "/".join(segments) + if not key.startswith(request.prefix): + continue + if len(objects) >= request.page_size: + more = True + return False + try: + stat = os.stat(path) + version = _filesystem_version(path) + birth = _birth_time_ns(path, stat) + except FileNotFoundError: + continue + except OSError: + raise ReportingRowStoreError( + "PROVIDER_UNAVAILABLE", "filesystem list failed" + ) from None + written_ns = max(birth, stat.st_mtime_ns) + objects.append( + ReportingRowObjectListEntryV1( + key, + version, + stat.st_size, + format_row_instant( + datetime.fromtimestamp(written_ns / 1_000_000_000, tz=timezone.utc) + ), + ) + ) + return True + + walk(()) + token = objects[-1].object_ref if more and objects else None + return ReportingRowObjectListPageV1(tuple(objects), token) + async def delete( self, delete: ReportingRowObjectDeleteInputV1, context: ReportingRowObjectContextV1 ) -> Literal["deleted", "absent"]: @@ -776,4 +984,111 @@ def stat_input(version: str) -> ReportingRowObjectStatInputV1: outcome = await provider.delete(delete(recreated.native_version_ref), context) assert outcome == "deleted", "delete did not remove the re-created version" passed.append("write-unique-version") + lister = getattr(provider, "list", None) + if lister is not None: + await _check_listing(lister, location, prefix, credential_ref, provider, context) + passed.append("list") return passed + + +async def _check_listing( + lister: Any, + location: Mapping[str, str], + prefix: str, + credential_ref: str | None, + provider: ReportingRowObjectProviderV1, + context: ReportingRowObjectContextV1, +) -> None: + """The listing contract the orphan collector depends on. + + Every object under the prefix exactly once across pages, each with the version, size and + write time ``put_if_absent`` reports, nothing outside the prefix, and deleted objects gone. + """ + folder = f"{prefix}/conformance/{uuid.uuid4()}/" + bodies = { + f"{folder}a.jsonl": b'{"a":1}\n', + f"{folder}b.jsonl.gz": b'{"b":2}\n{"b":3}\n', + f"{folder}nested/c.jsonl": b'{"c":4}\n', + } + metadata = {"adcp-installation": "conformance", "adcp-intent": "conformance"} + created: dict[str, str] = {} + started = datetime.now(timezone.utc) + + async def list_all(under: str, page_size: int) -> list[ReportingRowObjectListEntryV1]: + found: list[ReportingRowObjectListEntryV1] = [] + token: str | None = None + for _ in range(100): + page = await lister( + ReportingRowObjectListInputV1(location, under, page_size, token, credential_ref), + context, + ) + assert ( + isinstance(page.objects, tuple) and len(page.objects) <= page_size + ), "list returned more objects than page_size" + found.extend(page.objects) + previous, token = token, page.next_page_token + if token is None: + return found + assert ( + isinstance(token, str) and token and token != previous + ), "list returned an invalid or repeated token" + raise AssertionError("list did not finish within 100 pages") + + try: + for key, data in bodies.items(): + put = await provider.put_if_absent( + ReportingRowObjectPutInputV1( + location, + key, + data, + "application/gzip" if key.endswith(".gz") else "application/x-ndjson", + metadata, + credential_ref, + ), + context, + ) + assert put.created, "list: put_if_absent did not create a fresh object" + created[key] = put.native_version_ref + for page_size in (1, 2, 100): + found = await list_all(folder, page_size) + keys = [entry.object_ref for entry in found] + assert len(set(keys)) == len( + keys + ), f"list returned an object twice (page_size {page_size})" + assert set(keys) == set( + bodies + ), f"list did not return exactly the objects under the prefix (page_size {page_size})" + for entry in found: + assert ( + entry.native_version_ref == created[entry.object_ref] + ), "list reported a native version that put_if_absent did not" + assert entry.size_bytes == len( + bodies[entry.object_ref] + ), "list reported a wrong size" + written = parse_row_instant(entry.created_at) + # Object-store clocks differ from this host's; a wide margin still catches a + # missing or zero time. + assert ( + written is not None and abs((written - started).total_seconds()) <= 600 + ), "list did not report when the version was written" + assert len(await list_all(f"{folder}nested/", 10)) == 1, "list ignored the prefix" + assert ( + len(await list_all(f"{folder}zz", 10)) == 0 + ), "list returned objects for an unused prefix" + first = next(iter(created)) + outcome = await provider.delete( + ReportingRowObjectDeleteInputV1(location, first, created[first], credential_ref), + context, + ) + assert outcome == "deleted", "list: delete did not remove the object" + del created[first] + remaining = sorted(entry.object_ref for entry in await list_all(folder, 2)) + assert remaining == sorted(created), "list returned a deleted object, or lost a live one" + finally: + for key, version in created.items(): + try: + await provider.delete( + ReportingRowObjectDeleteInputV1(location, key, version, credential_ref), context + ) + except Exception: # noqa: S110 # nosec B110 + pass diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_provider_io.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_provider_io.py index d59b6cc464..bd65b115f5 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_provider_io.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_provider_io.py @@ -13,16 +13,33 @@ from adcp_reporting_storage.ledger.row_storage_object import ReportingRowObjectGetInputV1 __all__ = [ + "ROW_IDENTIFIER_MAX_LENGTH", "assert_key", "assert_location_keys", "assert_read_bounds", "client_map", + "is_row_identifier", "prefix_filter_overlaps", "select_client", ] _T = TypeVar("_T") -_CREDENTIAL_REF = re.compile(r"^[A-Za-z0-9_.:-]{1,128}$") + +#: Bound for the row-storage identifiers a host chooses, ``row_binding_id`` and the binding's +#: credential reference: 1 to 255 characters of ``A-Za-z0-9_.:-``, the AdCP identifier bound +#: (``reporting_revision_id``, ``resource_ref`` and ``destination_ref`` use +#: ``^[A-Za-z0-9_.:-]{1,255}$``). The ``adcp_reporting_row_bindings`` ``CHECK`` enforces the same. +ROW_IDENTIFIER_MAX_LENGTH = 255 +_ROW_IDENTIFIER = re.compile(rf"[A-Za-z0-9_.:-]{{1,{ROW_IDENTIFIER_MAX_LENGTH}}}") + + +def is_row_identifier(value: object) -> bool: + """Whether ``value`` is a well-formed host-chosen row-storage identifier. + + ``fullmatch``, not ``match`` with ``$``: ``$`` also matches before a trailing newline, + which the database ``CHECK`` refuses. + """ + return isinstance(value, str) and _ROW_IDENTIFIER.fullmatch(value) is not None def assert_location_keys( @@ -66,7 +83,7 @@ def client_map(clients: Mapping[str, _T] | None) -> dict[str, _T]: return {} result: dict[str, _T] = {} for reference, client in clients.items(): - if not _CREDENTIAL_REF.match(reference) or client is None: + if not is_row_identifier(reference) or client is None: raise ReportingRowStoreError( "INVALID_INPUT", "clients must map credential references to clients" ) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_s3.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_s3.py index d3d96e4232..9484172ab2 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_s3.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_storage_s3.py @@ -20,9 +20,13 @@ from __future__ import annotations import asyncio +import base64 +import binascii +import json import re import uuid from collections.abc import Mapping +from datetime import datetime from typing import Any, Literal from adcp.reporting.ledger.row_storage_errors import ReportingRowStoreError @@ -30,11 +34,15 @@ ReportingRowObjectContextV1, ReportingRowObjectDeleteInputV1, ReportingRowObjectGetInputV1, + ReportingRowObjectListEntryV1, + ReportingRowObjectListInputV1, + ReportingRowObjectListPageV1, ReportingRowObjectProviderV1, ReportingRowObjectPutInputV1, ReportingRowObjectPutResultV1, ReportingRowObjectStatInputV1, ReportingRowObjectStatResultV1, + format_row_instant, ) from adcp_reporting_storage.ledger.row_storage_provider_io import ( assert_key, @@ -458,6 +466,92 @@ def _stat( raise ReportingRowStoreError("PROVIDER_UNAVAILABLE", "object metadata is malformed") return ReportingRowObjectStatResultV1(version, size) + async def list( + self, request: ReportingRowObjectListInputV1, context: ReportingRowObjectContextV1 + ) -> ReportingRowObjectListPageV1: + client, bucket = self._target(request.location, request.credential_ref) + if not isinstance(request.prefix, str) or len(request.prefix.encode()) > 1024: + raise ReportingRowStoreError("INVALID_INPUT", "list prefix is invalid") + if type(request.page_size) is not int or request.page_size < 1: + raise ReportingRowStoreError("INVALID_INPUT", "page_size must be a positive integer") + markers: tuple[str, str] | None = None + if request.page_token is not None: + try: + decoded = json.loads( + base64.urlsafe_b64decode( + request.page_token + "=" * (-len(request.page_token) % 4) + ) + ) + if not isinstance(decoded.get("k"), str) or not isinstance(decoded.get("v"), str): + raise ValueError("invalid") + markers = (decoded["k"], decoded["v"]) + except (ValueError, binascii.Error, UnicodeError, AttributeError): + raise ReportingRowStoreError( + "INVALID_INPUT", "list page token is invalid" + ) from None + return await asyncio.to_thread(self._list, client, bucket, request, markers) + + @staticmethod + def _list( + client: Any, + bucket: str, + request: ReportingRowObjectListInputV1, + markers: tuple[str, str] | None, + ) -> ReportingRowObjectListPageV1: + # Row buckets are versioned and the native version is the VersionId, so list versions: + # each (Key, VersionId) is what a delete would name. Delete markers are not objects. + parameters: dict[str, Any] = { + "Bucket": bucket, + "Prefix": request.prefix, + "MaxKeys": min(request.page_size, 1000), + } + if markers is not None: + parameters["KeyMarker"], parameters["VersionIdMarker"] = markers + output = client.list_object_versions(**parameters) + objects: list[ReportingRowObjectListEntryV1] = [] + skipped = 0 + for version in output.get("Versions") or []: + key = version.get("Key") + version_id = version.get("VersionId") + size = version.get("Size") + modified = version.get("LastModified") + # An entry that cannot be addressed or aged is left out: it is never collected, and + # one bad entry must not stop the listing of the rest. A version written before + # versioning was enabled is ``null`` and is never addressable by a recorded version. + if ( + not isinstance(key, str) + or not key.startswith(request.prefix) + or not isinstance(version_id, str) + or version_id == "null" + or not _VERSION_ID.fullmatch(version_id) + or type(size) is not int + or size < 0 + or not isinstance(modified, datetime) + or modified.tzinfo is None + ): + skipped += 1 + continue + objects.append( + ReportingRowObjectListEntryV1(key, version_id, size, format_row_instant(modified)) + ) + next_key = output.get("NextKeyMarker") + next_version = output.get("NextVersionIdMarker") + token: str | None = None + if output.get("IsTruncated"): + if not isinstance(next_key, str) or not isinstance(next_version, str): + # Ending here would report a partial listing as complete. + raise ReportingRowStoreError( + "PROVIDER_UNAVAILABLE", "object listing is truncated without a marker" + ) + token = ( + base64.urlsafe_b64encode( + json.dumps({"k": next_key, "v": next_version}, separators=(",", ":")).encode() + ) + .rstrip(b"=") + .decode("ascii") + ) + return ReportingRowObjectListPageV1(tuple(objects), token, skipped) + async def delete( self, delete: ReportingRowObjectDeleteInputV1, context: ReportingRowObjectContextV1 ) -> Literal["deleted", "absent"]: diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_write.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_write.py index 7f36249938..01a986da0b 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_write.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/row_write.py @@ -44,6 +44,7 @@ ReportingPreparedRowSetV1, ReportingRowChunkLocatorV1, ReportingRowStoreError, + recorded_intent_objects, ) if TYPE_CHECKING: @@ -338,16 +339,25 @@ async def persist_revision_row_set( "STATE_UNAVAILABLE", "the shared row set is no longer live" ) else: + # Closing the intent and inserting the chunk rows are one transaction, so a reader of the + # ledger sees an object named by this open intent or by committed chunk rows, never neither. closed = await connection.execute( "DELETE FROM adcp_reporting_row_write_intents" " WHERE row_binding_id = %s AND row_set_id = %s AND content_sha256 = %s" - " AND owner_token = %s AND state = 'open'", + " AND owner_token = %s AND state = 'open' RETURNING created_objects", (prepared.binding_id, prepared.row_set_id, prepared.content_sha256, staged.owner_token), ) - if closed.rowcount != 1: + closed_row = await closed.fetchone() + if closed_row is None: raise ReportingRowStoreError( "STATE_UNAVAILABLE", "the row upload intent was taken over or swept" ) - if staged.shared_from is None: - await storage.lock_and_verify_adopted(connection, prepared.binding_id, staged.locators) + # Adopted objects the intent recorded take no lock; any other adopted object takes its + # binding's object lock, then every adopted object is proven by a version-pinned stat. + await storage.lock_and_verify_adopted( + connection, + prepared.binding_id, + staged.locators, + recorded_intent_objects(closed_row[0]), + ) await storage.write_in_transaction(connection, prepared, staged.locators, staged.shared_from) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/migration.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/migration.py index fb56b1ee3c..9de0f5018b 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/migration.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/migration.py @@ -123,6 +123,9 @@ class _Statement: _STARTUP_ROWS = { "adcp_reporting_change_horizon": "pruned_through_seq = 0 AND pruned_through_xid = '0'", "adcp_reporting_row_bindings": "TRUE", + # An opt-in orphan collector's claim on its bucket namespace. A replacement also drops the + # installation identity, so the namespace it fenced is not this ledger's any more. + "adcp_reporting_orphan_collector_fences": "TRUE", } _DEPENDENT_OBJECTS_STILL_EXIST = "2BP01" _SAVEPOINT = "adcp_reporting_replacement" diff --git a/scripts/check_main_policies.py b/scripts/check_main_policies.py index 2b3a11b697..7aea2fe3e6 100644 --- a/scripts/check_main_policies.py +++ b/scripts/check_main_policies.py @@ -106,6 +106,18 @@ def pages(self, suffix: str, key: str | None = None) -> list[Any]: ), } +# #1810's PR title ended in an issue reference, "(#1780)". The PR check +# stripped it as if it were the PR number, and the squash subject then carried +# a second reference, so the integrated description keeps "(#1780)". PR titles +# are now checked without that stripping; only this exact commit and subject +# keep the reference. +HISTORICAL_DESCRIPTION_SUBJECTS = { + "272fced52d57632dedf32f2c175cf294100bff63": ( + "feat(decisioning): let a raised AdcpError reject a request with every error" + " (#1780) (#1810)" + ), +} + # Rebase-merged PRs record only their last commit as merge_commit_sha, so the # earlier commits they put on main match no PR by merge commit. These two were # rebase-merged so a storage-only commit would not release adcp; each maps to @@ -131,15 +143,28 @@ def main_commits(target: str) -> list[str]: return commits -def validate_message(message: str, *, commit_sha: str | None = None) -> None: +def validate_message( + message: str, *, commit_sha: str | None = None, integrated: bool = True +) -> None: subject, _, body = message.partition("\n") match = CONVENTIONAL.fullmatch(subject) require(match is not None, "commit subject is not a conventional commit") assert match is not None - # GitHub integration subjects may include the PR number. Parentheses/quotes in - # the actual description remain forbidden by the repository's parser rule. - description = re.sub(r" \(#[1-9][0-9]*\)$", "", match["description"]) - require(description and not any(c in description for c in '()"'), "unsafe release description") + # A squash integration appends the PR number to the subject. A PR title has + # none yet, so a trailing "(#N)" there is part of the description. + # Parentheses/quotes in the actual description remain forbidden by the + # repository's parser rule. + description = match["description"] + if integrated: + description = re.sub(r" \(#[1-9][0-9]*\)$", "", description) + historical_description_exception = ( + commit_sha is not None and HISTORICAL_DESCRIPTION_SUBJECTS.get(commit_sha) == subject + ) + require( + description + and (historical_description_exception or not any(c in description for c in '()"')), + "unsafe release description", + ) historical_footer_exception = ( commit_sha is not None and HISTORICAL_BREAKING_SUBJECTS.get(commit_sha) == subject ) @@ -204,7 +229,7 @@ def conventional( api = GitHub(os.environ["GH_TOKEN"]) current = api.repo(f"pulls/{pr['number']}") require(current["head"]["sha"] == pr["head"]["sha"], "PR head changed; run current CI") - validate_message(current["title"] + "\n\n" + (current["body"] or "")) + validate_message(current["title"] + "\n\n" + (current["body"] or ""), integrated=False) head, base = pr["head"]["sha"], pr["base"]["sha"] require(SHA.fullmatch(head) and SHA.fullmatch(base), "invalid PR source identities") fork = git("merge-base", base, head) diff --git a/src/adcp/reporting/production/_diagnostics.py b/src/adcp/reporting/production/_diagnostics.py index 5796357ca8..5cfa90d335 100644 --- a/src/adcp/reporting/production/_diagnostics.py +++ b/src/adcp/reporting/production/_diagnostics.py @@ -78,7 +78,7 @@ def _owed_sets_failed(*, failed: int, total: int) -> None: _MAINTENANCE_OPERATIONS = frozenset( - {"intent_sweep", "row_pruning", "period_retirement", "change_pruning"} + {"intent_sweep", "row_pruning", "period_retirement", "change_pruning", "orphan_collection"} ) #: The only counters a maintenance event may carry: integers, never identifiers. _MAINTENANCE_COUNTERS = frozenset( @@ -92,6 +92,8 @@ def _owed_sets_failed(*, failed: int, total: int) -> None: "stuck", "skipped_unconfigured", "overdue", + "unsupported", + "fenced", } ) diff --git a/src/adcp/reporting/production/service.py b/src/adcp/reporting/production/service.py index b5beb13452..524c6ea42e 100644 --- a/src/adcp/reporting/production/service.py +++ b/src/adcp/reporting/production/service.py @@ -810,14 +810,17 @@ async def _maintain(self) -> None: # is visible instead of looking like a healthy idle worker. pending = { key: counts[key] - for key in ("protected", "held", "deferred", "skipped_unconfigured") + for key in ("protected", "held", "deferred", "skipped_unconfigured", "unsupported") if counts.get(key) } if pending: _maintenance_pending(operation=name, counts=pending) - # Work nobody is reaching: intents expired past the overdue threshold, or periods that - # keep failing (five or more attempts). The same signals the JavaScript SDK reports. - stalled = {key: counts[key] for key in ("overdue", "stuck") if counts.get(key)} + # Work nobody is reaching: intents expired past the overdue threshold, periods that + # keep failing (five or more attempts), or orphan collection refused by another + # database's claim on its namespace. The same signals the JavaScript SDK reports. + stalled = { + key: counts[key] for key in ("overdue", "stuck", "fenced") if counts.get(key) + } if stalled: _maintenance_stalled(operation=name, counts=stalled) @@ -933,6 +936,8 @@ def _counts(result: Any) -> dict[str, int]: "stuck", "skipped_unconfigured", "overdue", + "unsupported", + "fenced", ): value = getattr(result, name, None) if isinstance(value, (tuple, list)): diff --git a/tests/conformance/reporting/_orphan_support.py b/tests/conformance/reporting/_orphan_support.py new file mode 100644 index 0000000000..92a9c73120 --- /dev/null +++ b/tests/conformance/reporting/_orphan_support.py @@ -0,0 +1,429 @@ +"""Shared fixture for the orphan collector and its ownership fence. + +A real PostgreSQL schema and the filesystem provider, with hooks on the provider: fail a delete, +fail or replace a listing page, make every object look older, or count fence reads. A "clone" is a +point-in-time copy of every table in the live schema, which is how a database clone or restore looks +to the collector: the same installation identity, namespace key, bindings and fence row. +""" + +from __future__ import annotations + +import os +import secrets +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from dataclasses import dataclass, field, replace +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import pytest +from adcp_reporting_storage import PgReportingLedgerStore, ReportingRowStorageOptionsV1 +from adcp_reporting_storage.ledger.orphan_collector import ( + ReportingOrphanBindingResultV1, + ReportingOrphanCollectionResultV1, +) +from adcp_reporting_storage.ledger.row_storage import ( + ReportingObjectRowBindingV1, + ReportingPostgresRowBindingV1, +) +from adcp_reporting_storage.ledger.row_storage_object import ( + ReportingRowObjectContextV1, + ReportingRowObjectDeleteInputV1, + ReportingRowObjectListEntryV1, + ReportingRowObjectListInputV1, + ReportingRowObjectListPageV1, + ReportingRowObjectPutInputV1, + create_filesystem_reporting_row_object_provider_v1, + format_row_instant, + parse_row_instant, +) + +from ._generation_support import isolated_reporting_pool +from .test_reporting_row_storage_pg import fetch_all, many_rows, seeded, typed_revision + +DAY = timedelta(days=1) +CONTEXT = ReportingRowObjectContextV1(timeout_seconds=10.0) +LOCATION = {"root": "rows"} +# In foreign-key order. +OWNED_TABLES = ( + "adcp_persistence_installation", + "adcp_reporting_row_bindings", + "adcp_reporting_row_sets", + "adcp_reporting_row_chunks", + "adcp_reporting_chunk_bodies", + "adcp_reporting_row_write_intents", + "adcp_reporting_orphan_collector_fences", +) + + +def later() -> datetime: + """A clock past the age threshold, so every object written by the test is old enough.""" + return datetime.now(timezone.utc) + 8 * DAY + + +class HookedProvider: + """The filesystem provider with test hooks on ``delete`` and ``list``.""" + + name = "filesystem" + ranged_reads = True + + def __init__(self, inner: Any) -> None: + self.inner = inner + self.failing_keys: set[str] = set() + self.list_failures: list[BaseException] = [] + self.delete_calls: list[str] = [] + self.list_calls: list[str] = [] + self.list_override: ( + Callable[ + [ReportingRowObjectListInputV1, Callable[..., Awaitable[Any]]], + Awaitable[ReportingRowObjectListPageV1], + ] + | None + ) = None + self.age_shift: timedelta = timedelta(0) + self.put_failures: list[BaseException] = [] + self.on_put: Callable[[ReportingRowObjectPutInputV1], Awaitable[None]] | None = None + self.on_delete: Callable[[str], Awaitable[None]] | None = None + self.fence_list_failures: list[BaseException] = [] + self.fence_list_override: ReportingRowObjectListPageV1 | None = None + + def validate_location(self, location: Any) -> None: + self.inner.validate_location(location) + + async def probe(self, *args: Any, **kwargs: Any) -> None: + await self.inner.probe(*args, **kwargs) + + async def put_if_absent(self, put: ReportingRowObjectPutInputV1, context: Any) -> Any: + if self.put_failures: + raise self.put_failures.pop(0) + if self.on_put is not None: + await self.on_put(put) + return await self.inner.put_if_absent(put, context) + + async def get(self, *args: Any, **kwargs: Any) -> Any: + return await self.inner.get(*args, **kwargs) + + async def stat(self, *args: Any, **kwargs: Any) -> Any: + return await self.inner.stat(*args, **kwargs) + + async def delete(self, delete: ReportingRowObjectDeleteInputV1, context: Any) -> Any: + if self.on_delete is not None: + await self.on_delete(delete.object_ref) + if delete.object_ref in self.failing_keys: + raise RuntimeError("provider unavailable") + self.delete_calls.append(f"{delete.object_ref}@{delete.native_version_ref}") + return await self.inner.delete(delete, context) + + async def list( + self, request: ReportingRowObjectListInputV1, context: Any + ) -> ReportingRowObjectListPageV1: + self.list_calls.append(request.prefix) + # The hooks aim at the collector's object listing, not at the fence chain it also lists. + collecting = "_collector-fence" not in request.prefix + if not collecting and self.fence_list_failures: + raise self.fence_list_failures.pop(0) + if not collecting and self.fence_list_override is not None: + return self.fence_list_override + if collecting and self.list_failures: + raise self.list_failures.pop(0) + + async def real(page: ReportingRowObjectListInputV1) -> ReportingRowObjectListPageV1: + listed = await self.inner.list(page, context) + if not self.age_shift: + return listed # type: ignore[no-any-return] + shifted = [] + for entry in listed.objects: + moment = parse_row_instant(entry.created_at) + assert moment is not None + shifted.append( + replace(entry, created_at=format_row_instant(moment - self.age_shift)) + ) + return replace(listed, objects=tuple(shifted)) + + if collecting and self.list_override is not None: + return await self.list_override(request, real) + return await real(request) + + +class NoListProvider: + """The same provider without ``list``: a binding that cannot be collected.""" + + name = "filesystem" + ranged_reads = True + + def __init__(self, inner: Any) -> None: + for method in ("validate_location", "probe", "put_if_absent", "get", "stat", "delete"): + setattr(self, method, getattr(inner, method)) + + +@dataclass +class Fixture: + pool: Any + store: PgReportingLedgerStore + root: Path + provider: HookedProvider + namespace_key: str + options: ReportingRowStorageOptionsV1 + obligation: Any = None + revisions: int = 0 + extras: dict[str, Any] = field(default_factory=dict) + + def key(self, name: str, prefix: str = "orph") -> str: + return f"{prefix}/{self.namespace_key}/{name}" + + def on_disk(self, object_ref: str) -> bool: + return (self.root / "rows" / object_ref).exists() + + async def put_object(self, object_ref: str, body: bytes = b'{"a":1}\n') -> str: + created = await self.provider.inner.put_if_absent( + ReportingRowObjectPutInputV1( + LOCATION, + object_ref, + body, + "application/x-ndjson", + {"adcp-installation": "x", "adcp-intent": "y"}, + ), + CONTEXT, + ) + return str(created.native_version_ref) + + async def add_intent( + self, + row_set_id: str, + object_ref: str | None, + native_version_ref: str | None, + *, + binding: str = "objects", + state: str = "open", + legacy_names: bool = False, + ) -> None: + import json + + created = [] + if object_ref is not None: + created = [ + ( + {"objectKey": object_ref, "nativeVersion": native_version_ref} + if legacy_names + else {"object_ref": object_ref, "native_version_ref": native_version_ref} + ) + ] + async with self.pool.connection() as connection: + await connection.execute( + "INSERT INTO adcp_reporting_row_write_intents (row_binding_id, account_id," + " row_set_id, content_sha256, state, owner_token, created_objects, expires_at)" + " VALUES (%s, 'acct_orphans', %s, %s, %s, 'owner', %s::jsonb," + " clock_timestamp() + INTERVAL '1 hour')", + (binding, row_set_id, "e" * 64, state, json.dumps(created)), + ) + + async def commit_revision(self, suffix: str) -> tuple[str, str]: + """Commit an object-bound revision; returns the chunk's ``(object_ref, native_version)``.""" + if self.obligation is None: + self.obligation = await seeded(self.store) + async with self.pool.connection() as connection: + await connection.execute( + "UPDATE reporting_configurations SET lease_worker_id = 'worker-1'," + " lease_expires_at = clock_timestamp() + INTERVAL '10 minutes'" + ) + rows = many_rows(4) + revision = typed_revision(self.obligation, rows, suffix=suffix) + assert await self.store.stage_revision_rows(revision, rows) is True + await self.store.commit_revision(revision, rows) + found = await fetch_all( + self.pool, + "SELECT object_ref, native_version_ref FROM adcp_reporting_row_chunks" + " WHERE row_set_id = %s", + (revision.reporting_revision_id,), + ) + return str(found[0][0]), str(found[0][1]) + + def fence_dir(self) -> Path: + return self.root / "rows" / "orph" / self.namespace_key / "_collector-fence" + + def fence_files(self) -> list[str]: + directory = self.fence_dir() + return sorted(item.name for item in directory.iterdir()) if directory.exists() else [] + + def write_fence( + self, + epoch: int, + *, + token: str | None = None, + previous: str | None = None, + namespace: str | None = None, + raw: bytes | None = None, + ) -> str: + """Put a fence object in the bucket by hand, as another database would have.""" + import uuid + + from adcp_reporting_storage.ledger.orphan_fence import ( + OrphanFenceRecord, + encode_orphan_fence_record, + orphan_fence_key, + ) + + key = orphan_fence_key(f"orph/{self.namespace_key}/", epoch) + path = self.root / "rows" / key + path.parent.mkdir(parents=True, exist_ok=True) + chosen = token or str(uuid.uuid4()) + path.write_bytes( + raw + if raw is not None + else encode_orphan_fence_record( + namespace or self.namespace_key, + OrphanFenceRecord(epoch, chosen, previous, "2026-01-01T00:00:00.000Z"), + ) + ) + return chosen + + async def fence_row(self, pool: Any = None) -> tuple[Any, ...] | None: + rows = await fetch_all( + pool or self.pool, + "SELECT epoch, token, previous_token FROM adcp_reporting_orphan_collector_fences", + ) + return rows[0] if rows else None + + async def collect(self, **overrides: Any) -> ReportingOrphanCollectionResultV1: + arguments: dict[str, Any] = {"now": later(), "acknowledge_exclusive_namespace": True} + arguments.update(overrides) + return await self.store.collect_orphaned_row_objects(**arguments) + + +def binding_result( + result: ReportingOrphanCollectionResultV1, binding_id: str +) -> ReportingOrphanBindingResultV1: + return next(item for item in result.bindings if item.binding_id == binding_id) + + +def row_storage_options(tmp_path: Path, provider: HookedProvider) -> ReportingRowStorageOptionsV1: + return ReportingRowStorageOptionsV1( + deployment_namespace="orphans", + bindings={ + "inline": ReportingPostgresRowBindingV1(), + "objects": ReportingObjectRowBindingV1( + provider="filesystem", location=LOCATION, prefix="orph" + ), + "legacy": ReportingObjectRowBindingV1( + provider="filesystem", location=LOCATION, prefix="legacy", read_only=True + ), + "nolist": ReportingObjectRowBindingV1( + provider="nolist", location=LOCATION, prefix="nolist" + ), + # Shares ``objects``' root and prefix, as a renamed or successor binding does. + "twin": ReportingObjectRowBindingV1( + provider="filesystem", + location=LOCATION, + prefix="orph", + key_template="{prefix}/{namespace_key}/twin/{revision_key}/{content_sha256}/{chunk_index}", + read_only=True, + ), + }, + providers={ + "filesystem": provider, # type: ignore[dict-item] + "nolist": NoListProvider(provider.inner), # type: ignore[dict-item] + }, + select_row_binding=lambda selection: "objects", + ) + + +@asynccontextmanager +async def orphan_fixture(tmp_path: Path, **store_options: Any) -> AsyncIterator[Fixture]: + async with isolated_reporting_pool() as pool: + provider = HookedProvider( + create_filesystem_reporting_row_object_provider_v1(roots={"rows": tmp_path / "rows"}) + ) + options = row_storage_options(tmp_path, provider) + store = PgReportingLedgerStore(pool=pool, row_storage=options, **store_options) + await store.create_schema() + assert store._row_storage is not None + namespace_key = store._row_storage.installation_identity[1] + yield Fixture(pool, store, tmp_path, provider, namespace_key, options) + + +async def clone_schema(pool: Any) -> str: + """Copy every row-storage table into a new schema: a point-in-time clone of the ledger. + + Returns the schema name; ``cloned_store`` opens a store on it. The copy keeps the installation + identity, the namespace key, the bindings and the fence row the live database had when it was + taken. The row-storage DDL is applied first so the clone carries the same triggers and + constraints the catalog check requires. + """ + from adcp_reporting_storage.ledger.row_storage import REPORTING_ROW_STORAGE_DDL_PATH + from psycopg import AsyncConnection, sql + + url = os.environ["ADCP_PG_TEST_URL"] + async with pool.connection() as live: + source = (await (await live.execute("SELECT current_schema()")).fetchone())[0] + target = f"adcp_reporting_clone_{secrets.token_hex(6)}" + async with await AsyncConnection.connect(url, autocommit=True) as admin: + await admin.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(target))) + await admin.execute(sql.SQL("SET search_path TO {}").format(sql.Identifier(target))) + await admin.execute(REPORTING_ROW_STORAGE_DDL_PATH.read_text()) + await admin.execute("DELETE FROM adcp_persistence_installation") + for table in OWNED_TABLES: + await admin.execute( + sql.SQL("INSERT INTO {}.{} SELECT * FROM {}.{}").format( + sql.Identifier(target), + sql.Identifier(table), + sql.Identifier(source), + sql.Identifier(table), + ) + ) + return target + + +@asynccontextmanager +async def cloned_store(fixture: Fixture, schema: str) -> AsyncIterator[PgReportingLedgerStore]: + """A second store on the clone: same configuration, its own database state.""" + url = os.environ["ADCP_PG_TEST_URL"] + from psycopg import AsyncConnection, sql + from psycopg_pool import AsyncConnectionPool + + async with AsyncConnectionPool( + url, + kwargs={ + "options": f"-csearch_path={schema} -cstatement_timeout=15000", + "autocommit": False, + }, + min_size=1, + max_size=4, + open=False, + ) as pool: + await pool.wait(timeout=10) + store = PgReportingLedgerStore(pool=pool, row_storage=fixture.options) + await store.ready_row_storage() + try: + yield store + finally: + async with await AsyncConnection.connect(url, autocommit=True) as admin: + await admin.execute( + sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema)) + ) + + +def require_database() -> None: + if not os.environ.get("ADCP_PG_TEST_URL"): + pytest.skip("ADCP_PG_TEST_URL not set — requires real PostgreSQL") + + +async def count_changes(fixture: Fixture) -> int: + rows = await fetch_all(fixture.pool, "SELECT count(*) FROM adcp_reporting_row_write_intents") + return int(rows[0][0]) + + +__all__ = [ + "CONTEXT", + "DAY", + "LOCATION", + "Fixture", + "HookedProvider", + "NoListProvider", + "binding_result", + "clone_schema", + "cloned_store", + "later", + "orphan_fixture", + "ReportingRowObjectListEntryV1", +] diff --git a/tests/conformance/reporting/test_reporting_change_feed_consumers_pg.py b/tests/conformance/reporting/test_reporting_change_feed_consumers_pg.py index b7f7195944..ef1fabbb1a 100644 --- a/tests/conformance/reporting/test_reporting_change_feed_consumers_pg.py +++ b/tests/conformance/reporting/test_reporting_change_feed_consumers_pg.py @@ -1,7 +1,8 @@ """Feed consumer checkpoints are forward-only within a scope (adcp-client-python #1509). Ported from the JavaScript SDK (adcp-client #3197): a delayed or repeated acknowledgment -must not move a consumer, or its retention hold, backward. +must not move a consumer, or its retention hold, backward. Account consumers compare +``(xid, seq)`` like deployment consumers (adcp-client #3289, adcp-client-python #1825). """ from __future__ import annotations @@ -24,7 +25,15 @@ async def feed_store(pool: Any) -> PgReportingLedgerStore: return store -def account_cursor(seq: int, account: str = "acct_a") -> str: +def account_cursor(seq: int, account: str = "acct_a", xid: int | None = None) -> str: + """A new-format account cursor; ``xid`` defaults to ``seq`` so the two orders agree.""" + return encode_reporting_change_cursor_v1( + account=account, seq=str(seq), xid=str(seq if xid is None else xid) + ) + + +def legacy_account_cursor(seq: int, account: str = "acct_a") -> str: + """The cursor an older SDK saved: a ``seq`` and no ``xid``.""" return encode_reporting_change_cursor_v1(account=account, seq=str(seq)) @@ -66,29 +75,29 @@ async def test_an_account_consumer_never_moves_backward_unless_asked_to() -> Non store = await feed_store(pool) assert await store.save_feed_consumer_cursor("sink", account_cursor(10)) is True assert await store.save_feed_consumer_cursor("sink", account_cursor(20)) is True - assert await saved(pool, "sink") == ("acct_a", None, 20) + assert await saved(pool, "sink") == ("acct_a", "20", 20) # A delayed acknowledgment is ignored and says so. applied = await store.save_feed_consumer_cursor("sink", account_cursor(15)) - assert await saved(pool, "sink") == ("acct_a", None, 20) + assert await saved(pool, "sink") == ("acct_a", "20", 20) assert applied is False # A deliberate replay rewinds, with or without the refresh option. assert await store.save_feed_consumer_cursor("sink", account_cursor(5), rewind=True) is True - assert await saved(pool, "sink") == ("acct_a", None, 5) + assert await saved(pool, "sink") == ("acct_a", "5", 5) assert ( await store.save_feed_consumer_cursor( "sink", account_cursor(3), rewind=True, refresh_unchanged=False ) is True ) - assert await saved(pool, "sink") == ("acct_a", None, 3) + assert await saved(pool, "sink") == ("acct_a", "3", 3) # Compared numerically, not as text: 100 is ahead of 95. await store.save_feed_consumer_cursor("sink", account_cursor(95)) assert await store.save_feed_consumer_cursor("sink", account_cursor(100)) is True assert await store.save_feed_consumer_cursor("sink", account_cursor(95)) is False - assert await saved(pool, "sink") == ("acct_a", None, 100) + assert await saved(pool, "sink") == ("acct_a", "100", 100) async def test_a_repeated_acknowledgment_renews_the_hold_unless_told_not_to() -> None: @@ -151,11 +160,11 @@ async def test_a_change_of_feed_scope_replaces_the_position() -> None: await store.save_feed_consumer_cursor("sink", account_cursor(500, "acct_a")) # Positions in different feeds are not comparable, so a scope change always applies. assert await store.save_feed_consumer_cursor("sink", account_cursor(1, "acct_b")) is True - assert await saved(pool, "sink") == ("acct_b", None, 1) + assert await saved(pool, "sink") == ("acct_b", "1", 1) assert await store.save_feed_consumer_cursor("sink", deployment_cursor(5, 1)) is True assert await saved(pool, "sink") == (None, "5", 1) assert await store.save_feed_consumer_cursor("sink", account_cursor(900, "acct_a")) is True - assert await saved(pool, "sink") == ("acct_a", None, 900) + assert await saved(pool, "sink") == ("acct_a", "900", 900) async def test_concurrent_writers_cannot_interleave_a_stale_position_over_a_newer_one( @@ -216,7 +225,7 @@ async def connection(*args: Any, **kwargs: Any): task.cancel() await asyncio.gather(*tasks, return_exceptions=True) - assert await saved(pool, "sink") == ("acct_a", None, 9), "a stale write landed last" + assert await saved(pool, "sink") == ("acct_a", "9", 9), "a stale write landed last" assert results == (True, False) @@ -243,6 +252,77 @@ async def test_remove_feed_consumer_releases_the_hold() -> None: assert await store.remove_feed_consumer("sink") is False # Saving under the same name registers it again, at the position given. assert await store.save_feed_consumer_cursor("sink", account_cursor(2)) is True - assert await saved(pool, "sink") == ("acct_a", None, 2) + assert await saved(pool, "sink") == ("acct_a", "2", 2) with pytest.raises(ValueError): await store.remove_feed_consumer("bad name") + + +async def test_an_account_consumer_compares_xid_then_seq() -> None: + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + assert await store.save_feed_consumer_cursor("sink", account_cursor(9, xid=50)) is True + # A later transaction with a lower seq is ahead in (xid, seq) order; an earlier + # transaction with a higher seq is behind. + assert await store.save_feed_consumer_cursor("sink", account_cursor(2, xid=60)) is True + assert await saved(pool, "sink") == ("acct_a", "60", 2) + assert await store.save_feed_consumer_cursor("sink", account_cursor(99, xid=55)) is False + assert await saved(pool, "sink") == ("acct_a", "60", 2) + # Same transaction: seq decides. + assert await store.save_feed_consumer_cursor("sink", account_cursor(3, xid=60)) is True + assert await store.save_feed_consumer_cursor("sink", account_cursor(1, xid=60)) is False + assert await saved(pool, "sink") == ("acct_a", "60", 3) + + +async def test_a_null_xid_consumer_row_is_replaced_by_a_new_format_cursor_unless_its_seq_is_behind() -> ( + None +): + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + async with pool.connection() as connection: + await connection.execute( + "INSERT INTO adcp_reporting_feed_consumers" + " (consumer_name, account_id, cursor_xid, cursor_seq)" + " VALUES ('old', 'acct_a', NULL, 9000)" + ) + # A consumer row saved before the upgrade holds by seq; a cursor behind it is ignored. + assert await store.save_feed_consumer_cursor("old", account_cursor(5, xid=7)) is False + assert await saved(pool, "old") == ("acct_a", None, 9000) + assert await store.save_feed_consumer_cursor("old", account_cursor(9000, xid=7)) is True + assert await saved(pool, "old") == ("acct_a", "7", 9000) + assert await store.save_feed_consumer_cursor("old", account_cursor(5, xid=7)) is False + assert ( + await store.save_feed_consumer_cursor("old", account_cursor(5, xid=7), rewind=True) + is True + ) + + +async def test_saving_a_seq_only_account_cursor_registers_the_position_changes_after_reads() -> ( + None +): + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + async with pool.connection() as connection: + await connection.execute( + "INSERT INTO adcp_reporting_changes (xid, account_id, record_kind, record_id," + " obligation_id) VALUES ('10'::xid8, 'acct_a', 'revision', 'r1', 'o')," + " ('20'::xid8, 'acct_a', 'revision', 'r2', 'o')," + " ('30'::xid8, 'acct_a', 'revision', 'r3', 'o')" + ) + first_seq = (await fetch_all(pool, "SELECT seq FROM adcp_reporting_changes ORDER BY seq"))[ + 0 + ][0] + # The hold sits below every unread change: the oldest one after the cursor, xid 20. + assert await store.save_feed_consumer_cursor("sink", legacy_account_cursor(first_seq)) + assert await saved(pool, "sink") == ("acct_a", "20", first_seq) + page = await store.changes_after( + account_id="acct_a", cursor=legacy_account_cursor(first_seq) + ) + assert [r.record_id for r in page.records] == ["r2", "r3"] + # A new-format cursor replaces it, forward only. + assert await store.save_feed_consumer_cursor("sink", page.cursor) is True + row = await saved(pool, "sink") + assert row is not None and row[1] == "30" + # A mapped seq-only cursor is forward-only against a new-format registration. + assert ( + await store.save_feed_consumer_cursor("sink", legacy_account_cursor(first_seq)) is False + ) diff --git a/tests/conformance/reporting/test_reporting_change_feed_horizon_pg.py b/tests/conformance/reporting/test_reporting_change_feed_horizon_pg.py index 9948666ec6..b1c76c34b9 100644 --- a/tests/conformance/reporting/test_reporting_change_feed_horizon_pg.py +++ b/tests/conformance/reporting/test_reporting_change_feed_horizon_pg.py @@ -89,6 +89,7 @@ def __getattr__(self, name: str) -> Any: def feed_cases() -> list[Any]: return [ pytest.param(ACCOUNT, "with-cursor", id="account-with-cursor"), + pytest.param(ACCOUNT, "legacy-cursor", id="account-seq-only-cursor"), pytest.param(ACCOUNT, "no-cursor", id="account-without-cursor"), pytest.param(None, "with-cursor", id="deployment-with-cursor"), pytest.param(None, "no-cursor", id="deployment-without-cursor"), @@ -99,7 +100,9 @@ def starting_cursor(account: str | None, mode: str) -> str | None: if mode == "no-cursor": return None if account is not None: - return encode_reporting_change_cursor_v1(account=account, seq="0") + # A seq-only cursor is the form an older SDK saved: the horizon check is exact for it. + xid = None if mode == "legacy-cursor" else "0" + return encode_reporting_change_cursor_v1(account=account, seq="0", xid=xid) return encode_reporting_change_cursor_v1(account=None, seq="0", xid="0") @@ -150,7 +153,7 @@ async def test_a_prune_that_commits_before_the_read_starts_is_not_a_race( await seed_changes(pool) assert await store.prune_changes() == 2 - if mode == "with-cursor": + if mode != "no-cursor": with pytest.raises(ReportingChangeCursorExpiredError): await store.changes_after(account_id=account, cursor=starting_cursor(account, mode)) else: diff --git a/tests/conformance/reporting/test_reporting_change_feed_horizon_upgrade_pg.py b/tests/conformance/reporting/test_reporting_change_feed_horizon_upgrade_pg.py new file mode 100644 index 0000000000..aafc220936 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_change_feed_horizon_upgrade_pg.py @@ -0,0 +1,269 @@ +"""An account horizon a release without the ``(xid, seq)`` account feed pruned past is raised. + +adcp-reporting-storage 0.1.0b5 hands out account cursors that carry ``xid`` and keeps the +account prune horizon in ``(xid, seq)``. 0.2.0b0 to 0.2.0b2 prune by ``seq`` and advance only +``pruned_through_seq``, so on a ledger 0.1.0b5 wrote, a cursor 0.1.0b5 handed out passed the +stale ``(xid, seq)`` check and silently skipped what 0.2.x pruned. The first ``create_schema`` +of this release on such a ledger raises every account horizon to the deployment horizon, which +every release keeps in ``(xid, seq)``, or just below the account's oldest retained change when +that is lower; a ledger upgraded straight from 0.1.0b5 keeps its own. +The 0.2.x prune is staged with the statements that release runs. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest +from adcp_reporting_storage import PgReportingLedgerStore +from adcp_reporting_storage.ledger.change_feed import ( + ReportingChangeCursorExpiredError, + encode_reporting_change_cursor_v1, +) + +from ._generation_support import isolated_reporting_pool + +ACCOUNT = "acct_interlude" +OTHER = "acct_other" +INSERT_CHANGE = ( + "INSERT INTO adcp_reporting_changes (account_id, record_kind, record_id, obligation_id)" + " VALUES (%s, 'revision', %s, 'robl_interlude')" +) +MARKER = "adcp: account horizon maintained in (xid, seq)" + + +async def feed_store(pool: Any) -> PgReportingLedgerStore: + store = PgReportingLedgerStore(pool=pool, change_feed=True) + await store.create_schema() + return store + + +async def insert(pool: Any, account: str, *record_ids: str) -> None: + for record_id in record_ids: # one transaction each, so each change has its own xid + async with pool.connection() as connection: + await connection.execute(INSERT_CHANGE, (account, record_id)) + + +async def age(pool: Any, *record_ids: str) -> None: + async with pool.connection() as connection: + await connection.execute( + "UPDATE adcp_reporting_changes SET recorded_at = recorded_at - INTERVAL '40 days'" + " WHERE record_id = ANY(%s)", + (list(record_ids),), + ) + + +async def cursor_at(pool: Any, account: str, record_id: str) -> str: + """The account cursor 0.1.0b5 handed out after ``record_id``.""" + async with pool.connection() as connection: + row = await ( + await connection.execute( + "SELECT seq::text, xid::text FROM adcp_reporting_changes WHERE record_id = %s", + (record_id,), + ) + ).fetchone() + return encode_reporting_change_cursor_v1(account=account, seq=row[0], xid=row[1]) + + +async def prune_like_0_2(pool: Any, *record_ids: str) -> None: + """What 0.2.0b0-0.2.0b2's prune writes: the deployment horizon and pruned_through_seq.""" + async with pool.connection() as connection, connection.transaction(): + victims = await ( + await connection.execute( + "DELETE FROM adcp_reporting_changes WHERE record_id = ANY(%s)" + " RETURNING seq::text, xid::text, account_id", + (list(record_ids),), + ) + ).fetchall() + highest = max(victims, key=lambda row: (int(row[1]), int(row[0]))) + await connection.execute( + "UPDATE adcp_reporting_change_horizon" + " SET (pruned_through_xid, pruned_through_seq) = (%s::text::xid8, %s::bigint)" + " WHERE singleton AND (%s::text::xid8, %s::bigint) >" + " (pruned_through_xid, pruned_through_seq)", + (highest[1], highest[0], highest[1], highest[0]), + ) + per_account: dict[str, int] = {} + for seq, _, account in victims: + per_account[account] = max(per_account.get(account, 0), int(seq)) + for account, seq in per_account.items(): + await connection.execute( + "INSERT INTO adcp_reporting_change_account_horizon (account_id, pruned_through_seq)" + " VALUES (%s, %s::bigint) ON CONFLICT (account_id) DO UPDATE" + " SET pruned_through_seq = GREATEST(" + " adcp_reporting_change_account_horizon.pruned_through_seq," + " EXCLUDED.pruned_through_seq)", + (account, seq), + ) + + +async def forget_this_release(pool: Any, *, keep_migrations: bool) -> None: + """The ledger as 0.1.0b5 (and, with ``keep_migrations``, then 0.2.x) left it.""" + async with pool.connection() as connection: + await connection.execute("COMMENT ON TABLE adcp_reporting_change_account_horizon IS NULL") + if not keep_migrations: + await connection.execute("DELETE FROM adcp_schema_migrations") + + +async def account_horizon(pool: Any, account: str) -> tuple[str, str, str]: + async with pool.connection() as connection: + row = await ( + await connection.execute( + "SELECT pruned_through_xid::text, pruned_through_xid_seq::text," + " pruned_through_seq::text FROM adcp_reporting_change_account_horizon" + " WHERE account_id = %s", + (account,), + ) + ).fetchone() + return row + + +async def deployment_horizon(pool: Any) -> tuple[str, str]: + async with pool.connection() as connection: + row = await ( + await connection.execute( + "SELECT pruned_through_xid::text, pruned_through_seq::text" + " FROM adcp_reporting_change_horizon WHERE singleton" + ) + ).fetchone() + return row + + +async def marker(pool: Any) -> str | None: + async with pool.connection() as connection: + row = await ( + await connection.execute( + "SELECT obj_description('adcp_reporting_change_account_horizon'::regclass," + " 'pg_class')" + ) + ).fetchone() + return row[0] + + +async def read(store: PgReportingLedgerStore, cursor: str | None) -> list[str]: + """Poll: other transactions on the cluster can briefly hold a page back.""" + ids: list[str] = [] + for _ in range(200): + page = await store.changes_after(account_id=ACCOUNT, cursor=cursor) + ids = [record.record_id for record in page.records] + if ids: + break + await asyncio.sleep(0.05) + return ids + + +async def test_a_new_install_records_that_it_maintains_the_account_horizon() -> None: + async with isolated_reporting_pool() as pool: + await feed_store(pool) + assert await marker(pool) == MARKER + + +async def test_a_0_2_prune_after_0_1_0b5_expires_the_cursors_it_could_have_skipped() -> None: + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + await insert(pool, ACCOUNT, "rrev_1", "rrev_2", "rrev_3", "rrev_4") + # 0.1.0b5 prunes rrev_1 and records the account horizon in (xid, seq). + handed_out = await cursor_at(pool, ACCOUNT, "rrev_1") + await age(pool, "rrev_1") + assert await store.prune_changes() == 1 + assert await read(store, handed_out) == ["rrev_2", "rrev_3", "rrev_4"] + stale = await account_horizon(pool, ACCOUNT) + + # 0.2.x then prunes rrev_2 and rrev_3 without touching the xid columns. + await forget_this_release(pool, keep_migrations=True) + await prune_like_0_2(pool, "rrev_2", "rrev_3") + assert (await account_horizon(pool, ACCOUNT))[:2] == stale[:2] + # The hazard: on the stale horizon the old cursor reads on and skips both changes. + assert await read(store, handed_out) == ["rrev_4"] + + await store.create_schema() # this release starts on the ledger + xid, xid_seq, _ = await account_horizon(pool, ACCOUNT) + assert (xid, xid_seq) == await deployment_horizon(pool) + assert await marker(pool) == MARKER + with pytest.raises(ReportingChangeCursorExpiredError): + await store.changes_after(account_id=ACCOUNT, cursor=handed_out) + # A cursor at the last pruned change, and a consumer starting over, read on. + at_horizon = encode_reporting_change_cursor_v1(account=ACCOUNT, seq=xid_seq, xid=xid) + assert await read(store, at_horizon) == ["rrev_4"] + assert await read(store, None) == ["rrev_4"] + + # Once recorded, the step never runs again: a later horizon is this release's own. + async with pool.connection() as connection: + await connection.execute( + "UPDATE adcp_reporting_change_horizon SET pruned_through_seq =" + " pruned_through_seq + 1000 WHERE singleton" + ) + await store.create_schema() + assert (await account_horizon(pool, ACCOUNT))[:2] == (xid, xid_seq) + + +async def test_the_raised_horizon_never_passes_a_retained_change() -> None: + """When 0.2.x pruned another account past this one's retained changes, they still read.""" + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + await insert(pool, ACCOUNT, "rrev_1", "rrev_2", "rrev_3") + await insert(pool, OTHER, "rrev_o1") + handed_out = await cursor_at(pool, ACCOUNT, "rrev_1") + await forget_this_release(pool, keep_migrations=True) + await prune_like_0_2(pool, "rrev_1", "rrev_2", "rrev_o1") + + await store.create_schema() + xid, xid_seq, _ = await account_horizon(pool, ACCOUNT) + async with pool.connection() as connection: + oldest = await ( + await connection.execute( + "SELECT xid::text, (seq - 1)::text FROM adcp_reporting_changes" + " WHERE record_id = 'rrev_3'" + ) + ).fetchone() + assert (xid, xid_seq) == tuple(oldest) + deployment = await deployment_horizon(pool) + assert (int(xid), int(xid_seq)) < (int(deployment[0]), int(deployment[1])) + with pytest.raises(ReportingChangeCursorExpiredError): + await store.changes_after(account_id=ACCOUNT, cursor=handed_out) + assert await read(store, None) == ["rrev_3"] + + +async def test_a_ledger_upgraded_straight_from_0_1_0b5_keeps_its_account_horizon() -> None: + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + await insert(pool, ACCOUNT, "rrev_a1", "rrev_a2") + await insert(pool, OTHER, "rrev_b1", "rrev_b2") + handed_out = await cursor_at(pool, ACCOUNT, "rrev_a1") + await age(pool, "rrev_a1") + assert await store.prune_changes() == 1 + await age(pool, "rrev_b1") + assert await store.prune_changes() == 1 # the deployment horizon moves past ACCOUNT's + kept = await account_horizon(pool, ACCOUNT) + deployment = await deployment_horizon(pool) + assert (int(kept[0]), int(kept[1])) < (int(deployment[0]), int(deployment[1])) + + # 0.1.0b5 records no schema version; this release's first boot records 0001 itself. + await forget_this_release(pool, keep_migrations=False) + await store.create_schema() + assert await account_horizon(pool, ACCOUNT) == kept + assert await marker(pool) == MARKER + assert await read(store, handed_out) == ["rrev_a2"] + + +async def test_an_install_from_before_the_xid_account_feed_keeps_the_seq_horizon() -> None: + """Its account cursors carry no xid; seq-only cursors are checked by pruned_through_seq.""" + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + await insert(pool, ACCOUNT, "rrev_1", "rrev_2") + await insert(pool, OTHER, "rrev_o1", "rrev_o2") + async with pool.connection() as connection: + await connection.execute( + "ALTER TABLE adcp_reporting_change_account_horizon" + " DROP COLUMN pruned_through_xid, DROP COLUMN pruned_through_xid_seq" + ) + await forget_this_release(pool, keep_migrations=True) + await prune_like_0_2(pool, "rrev_1", "rrev_o1") + await store.create_schema() + xid, xid_seq, seq = await account_horizon(pool, ACCOUNT) + assert (xid, xid_seq) == ("0", "0") and seq != "0" + legacy = encode_reporting_change_cursor_v1(account=ACCOUNT, seq="0", xid=None) + with pytest.raises(ReportingChangeCursorExpiredError): + await store.changes_after(account_id=ACCOUNT, cursor=legacy) + assert await read(store, None) == ["rrev_2"] diff --git a/tests/conformance/reporting/test_reporting_change_feed_pg.py b/tests/conformance/reporting/test_reporting_change_feed_pg.py index fb3bcbe4a0..5250559552 100644 --- a/tests/conformance/reporting/test_reporting_change_feed_pg.py +++ b/tests/conformance/reporting/test_reporting_change_feed_pg.py @@ -2,7 +2,9 @@ from __future__ import annotations +import asyncio import base64 +import json from typing import Any import pytest @@ -11,12 +13,17 @@ ReportingChangeCursorExpiredError, decode_reporting_change_cursor_v1, encode_reporting_change_cursor_v1, + is_legacy_account_change_cursor, ) from ._generation_support import configuration, isolated_reporting_pool, obligation_for from .test_reporting_row_storage_pg import fetch_all, many_rows, typed_revision +def b64(body: dict[str, Any]) -> str: + return base64.urlsafe_b64encode(json.dumps(body).encode()).decode() + + async def feed_store(pool: Any) -> PgReportingLedgerStore: store = PgReportingLedgerStore(pool=pool, change_feed=True) await store.create_schema() @@ -42,10 +49,22 @@ def test_cursors_use_the_format_the_javascript_sdk_reads_and_writes() -> None: "seq": "7", "xid": "42", } + # A seq-only account cursor is what an older SDK saved; it is still accepted. js_account = ( base64.urlsafe_b64encode(b'{"v":1,"account":"acct_a","seq":"9"}').rstrip(b"=").decode() ) assert encode_reporting_change_cursor_v1(account="acct_a", seq="9") == js_account + assert is_legacy_account_change_cursor(decode_reporting_change_cursor_v1(js_account, "acct_a")) + # The current account cursor carries the xid of the last change, as the JavaScript SDK writes it. + js_account_xid = ( + base64.urlsafe_b64encode(b'{"v":1,"account":"acct_a","seq":"9","xid":"42"}') + .rstrip(b"=") + .decode() + ) + assert encode_reporting_change_cursor_v1(account="acct_a", seq="9", xid="42") == js_account_xid + decoded_account = decode_reporting_change_cursor_v1(js_account_xid, "acct_a") + assert decoded_account["xid"] == "42" + assert not is_legacy_account_change_cursor(decoded_account) for bad, account in ( ("not base64!", None), (js_account, "acct_b"), @@ -53,6 +72,15 @@ def test_cursors_use_the_format_the_javascript_sdk_reads_and_writes() -> None: (js_deployment, "acct_a"), (base64.urlsafe_b64encode(b'{"v":2,"account":null,"seq":"1","xid":"1"}').decode(), None), (base64.urlsafe_b64encode(b'{"v":1,"account":null,"seq":"x","xid":"1"}').decode(), None), + # A deployment cursor always needs an xid; values the database cast rejects fail here. + (base64.urlsafe_b64encode(b'{"v":1,"account":null,"seq":"1"}').decode(), None), + (b64({"v": 1, "account": "a", "seq": "9223372036854775808"}), "a"), + (b64({"v": 1, "account": "a", "seq": "1", "xid": "18446744073709551616"}), "a"), + (b64({"v": 1, "account": None, "seq": "1", "xid": "99999999999999999999"}), None), + (b64({"v": 1, "account": "a", "seq": ""}), "a"), + (b64({"v": 1, "account": "a", "seq": "1", "xid": "x"}), "a"), + (b64({"v": 1, "account": "a", "seq": "1", "xid": None}), "a"), + (b64({"v": 1, "account": "a", "seq": "\u0663"}), "a"), # a non-ASCII digit ): with pytest.raises(ValueError): decode_reporting_change_cursor_v1(bad, account) @@ -244,6 +272,8 @@ async def test_pages_cross_99_to_100_in_numeric_order_on_both_feeds() -> None: (xid,) = (await fetch_all(pool, "SELECT xid::text FROM adcp_reporting_changes LIMIT 1"))[0] for account, start in ( + ("acct_a", encode_reporting_change_cursor_v1(account="acct_a", seq="94", xid=xid)), + # A seq-only account cursor maps to a position below every later change. ("acct_a", encode_reporting_change_cursor_v1(account="acct_a", seq="94")), (None, encode_reporting_change_cursor_v1(account=None, seq="94", xid=xid)), ): @@ -256,11 +286,12 @@ async def test_pages_cross_99_to_100_in_numeric_order_on_both_feeds() -> None: assert seen == list(range(95, 131)), (account, seen) -# -- account feed ordering, ported from the JavaScript SDK (adcp-client #3228) --------------- +# -- account feed ordering by (xid, seq), ported from the JavaScript SDK (adcp-client #3289) ---- +ACCOUNT = "acct_hol_feed" INSERT_CHANGE = ( "INSERT INTO adcp_reporting_changes (account_id, record_kind, record_id, obligation_id)" - " VALUES ('acct_hol_feed', 'revision', %s, 'robl_hol_feed')" + " VALUES (%s, 'revision', %s, 'robl_hol_feed')" ) @@ -268,16 +299,50 @@ def ids(page: Any) -> list[str]: return [record.record_id for record in page.records] -async def committed(pool: Any, record_id: str) -> None: +async def committed(pool: Any, record_id: str, account: str = ACCOUNT) -> None: async with pool.connection() as connection: - await connection.execute(INSERT_CHANGE, (record_id,)) + await connection.execute(INSERT_CHANGE, (account, record_id)) + + +async def insert_at( + pool: Any, account: str, record_id: str, xid: int, *, age_days: int = 0 +) -> None: + """A committed change with a chosen (historical, so always below the snapshot's xmin) + transaction id, and optionally an old ``recorded_at``, for ordering and pruning cases that + need exact values.""" + async with pool.connection() as connection: + await connection.execute( + "INSERT INTO adcp_reporting_changes (xid, account_id, record_kind, record_id," + " obligation_id, recorded_at) VALUES (%s::text::xid8, %s, 'revision', %s," + " 'robl_hol_feed', clock_timestamp() - (%s::integer * INTERVAL '1 day'))", + (str(xid), account, record_id, age_days), + ) + + +async def seq_of(pool: Any, record_id: str) -> str: + rows = await fetch_all( + pool, "SELECT seq::text FROM adcp_reporting_changes WHERE record_id = %s", (record_id,) + ) + return str(rows[0][0]) + + +async def remaining(pool: Any, account: str) -> list[str]: + rows = await fetch_all( + pool, + "SELECT record_id FROM adcp_reporting_changes WHERE account_id = %s ORDER BY seq", + (account,), + ) + return [row[0] for row in rows] + + +def legacy_cursor(account: str, seq: str) -> str: + """The cursor an older SDK saved: a ``seq`` and no ``xid``.""" + return encode_reporting_change_cursor_v1(account=account, seq=seq) async def until(read: Any, expected: list[str]) -> Any: """Other tests share the cluster, so an unrelated older transaction can briefly hold a page back: positive reads poll. The assertions that a page is held back do not need it.""" - import asyncio - for _ in range(200): page = await read() if ids(page) == expected: @@ -289,30 +354,30 @@ async def until(read: Any, expected: list[str]) -> Any: async def test_a_lower_seq_that_commits_after_a_higher_one_is_never_passed_over() -> None: async with isolated_reporting_pool() as pool: store = await feed_store(pool) - start = await store.changes_after(account_id="acct_hol_feed") + start = await store.changes_after(account_id=ACCOUNT) slow = await pool.getconn() try: - await slow.execute(INSERT_CHANGE, ("rrev_low",)) # the lower seq, commits last + await slow.execute(INSERT_CHANGE, (ACCOUNT, "rrev_low")) # the lower seq, commits last await committed(pool, "rrev_high") # a higher seq, committed already - during = await store.changes_after(account_id="acct_hol_feed", cursor=start.cursor) + during = await store.changes_after(account_id=ACCOUNT, cursor=start.cursor) assert ids(during) == [], "the page stops before the unfinished transaction" assert during.cursor == start.cursor, "the cursor does not move past it" await slow.commit() finally: await pool.putconn(slow) after = await until( - lambda: store.changes_after(account_id="acct_hol_feed", cursor=start.cursor), + lambda: store.changes_after(account_id=ACCOUNT, cursor=start.cursor), ["rrev_low", "rrev_high"], ) assert ids(after) == ["rrev_low", "rrev_high"] -async def test_a_held_back_change_is_never_skipped_when_a_later_one_passes_a_plain_xid_filter() -> ( +async def test_a_lower_xid_with_a_higher_seq_is_returned_first_and_the_held_back_one_after_it() -> ( None ): async with isolated_reporting_pool() as pool: store = await feed_store(pool) - start = await store.changes_after(account_id="acct_hol_feed") + start = await store.changes_after(account_id=ACCOUNT) early = await pool.getconn() blocker = await pool.getconn() try: @@ -320,23 +385,88 @@ async def test_a_held_back_change_is_never_skipped_when_a_later_one_passes_a_pla await early.execute("SELECT pg_current_xact_id()") await blocker.execute("SELECT pg_current_xact_id()") await committed(pool, "rrev_first") # lower seq, but a transaction after `blocker` - await early.execute(INSERT_CHANGE, ("rrev_second",)) # higher seq, xid before `blocker` + await early.execute( + INSERT_CHANGE, (ACCOUNT, "rrev_second") + ) # higher seq, xid < blocker await early.commit() - # `blocker` is the oldest transaction in flight, so rrev_first is held back while - # rrev_second sits below it. Filtering on xid alone would return rrev_second and a - # cursor past rrev_first, which would then never be seen. - during = await store.changes_after(account_id="acct_hol_feed", cursor=start.cursor) - assert ids(during) == [] - assert during.cursor == start.cursor, "the cursor does not move past the held change" + # `blocker` is the oldest transaction in flight, so rrev_first (a later xid) is held + # back while rrev_second sits below it. The feed orders by (xid, seq), so rrev_second + # comes first and the cursor stops at it, before rrev_first. + during = await until( + lambda: store.changes_after(account_id=ACCOUNT, cursor=start.cursor), + ["rrev_second"], + ) + assert ids(during) == ["rrev_second"] + assert during.cursor != start.cursor await blocker.commit() + rest = await until( + lambda: store.changes_after(account_id=ACCOUNT, cursor=during.cursor), + ["rrev_first"], + ) + assert ids(rest) == ["rrev_first"], "the held-back change is delivered once, after it" finally: await pool.putconn(early) await pool.putconn(blocker) + everything = await until( + lambda: store.changes_after(account_id=ACCOUNT, cursor=start.cursor), + ["rrev_second", "rrev_first"], + ) + assert ids(everything) == ["rrev_second", "rrev_first"], "a fresh reader sees the order" + + +async def test_a_lower_seq_with_a_higher_xid_that_commits_late_is_delivered_after_the_cursor() -> ( + None +): + """No account lock is taken anywhere here: two writers under different lock keys.""" + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + start = await store.changes_after(account_id=ACCOUNT) + quick = await pool.getconn() + slow = await pool.getconn() + try: + await quick.execute("SELECT pg_current_xact_id()") # xid Q + await slow.execute("SELECT pg_current_xact_id()") # xid S > Q + await slow.execute(INSERT_CHANGE, (ACCOUNT, "rrev_late")) # lower seq, higher xid + await quick.execute(INSERT_CHANGE, (ACCOUNT, "rrev_quick")) # higher seq, lower xid + await quick.commit() + during = await until( + lambda: store.changes_after(account_id=ACCOUNT, cursor=start.cursor), + ["rrev_quick"], + ) + assert ids(during) == ["rrev_quick"], "the late transaction is still open" + await slow.commit() + rest = await until( + lambda: store.changes_after(account_id=ACCOUNT, cursor=during.cursor), + ["rrev_late"], + ) + assert ids(rest) == ["rrev_late"] + assert ( + ids(await store.changes_after(account_id=ACCOUNT, cursor=rest.cursor)) == [] + ), "and nothing is delivered twice" + finally: + await pool.putconn(quick) + await pool.putconn(slow) + + +async def test_an_open_transaction_on_another_account_holds_a_later_change_back() -> None: + other = "acct_hol_feed_other" + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + start = await store.changes_after(account_id=ACCOUNT) + opened = await pool.getconn() + try: + await opened.execute(INSERT_CHANGE, (other, "rrev_other_inflight")) # in flight + await committed(pool, "rrev_after_other") + # xmin is cluster-wide, so a later xid is held back whichever account the open one wrote. + assert ids(await store.changes_after(account_id=ACCOUNT, cursor=start.cursor)) == [] + await opened.commit() + finally: + await pool.putconn(opened) after = await until( - lambda: store.changes_after(account_id="acct_hol_feed", cursor=start.cursor), - ["rrev_first", "rrev_second"], + lambda: store.changes_after(account_id=ACCOUNT, cursor=start.cursor), + ["rrev_after_other"], ) - assert ids(after) == ["rrev_first", "rrev_second"] + assert ids(after) == ["rrev_after_other"] async def test_a_committed_prefix_is_returned_and_the_cursor_stops_before_the_first_in_flight() -> ( @@ -344,20 +474,20 @@ async def test_a_committed_prefix_is_returned_and_the_cursor_stops_before_the_fi ): async with isolated_reporting_pool() as pool: store = await feed_store(pool) - start = await store.changes_after(account_id="acct_hol_feed") + start = await store.changes_after(account_id=ACCOUNT) await committed(pool, "rrev_prefix_a") opened = await pool.getconn() try: - await opened.execute(INSERT_CHANGE, ("rrev_prefix_b",)) # in flight + await opened.execute(INSERT_CHANGE, (ACCOUNT, "rrev_prefix_b")) # in flight await committed(pool, "rrev_prefix_c") # committed after it during = await until( - lambda: store.changes_after(account_id="acct_hol_feed", cursor=start.cursor), + lambda: store.changes_after(account_id=ACCOUNT, cursor=start.cursor), ["rrev_prefix_a"], ) - assert ids(during) == ["rrev_prefix_a"], "truncated, not filtered" + assert ids(during) == ["rrev_prefix_a"], "c is not handed out past b" await opened.commit() rest = await until( - lambda: store.changes_after(account_id="acct_hol_feed", cursor=during.cursor), + lambda: store.changes_after(account_id=ACCOUNT, cursor=during.cursor), ["rrev_prefix_b", "rrev_prefix_c"], ) assert ids(rest) == ["rrev_prefix_b", "rrev_prefix_c"] @@ -365,23 +495,416 @@ async def test_a_committed_prefix_is_returned_and_the_cursor_stops_before_the_fi await pool.putconn(opened) -async def test_the_account_feed_query_is_served_by_the_account_seq_index() -> None: - from adcp_reporting_storage.ledger.change_feed import _page_query +async def test_the_account_cursor_carries_the_xid_of_the_last_change_and_is_bound_to_its_account() -> ( + None +): + scoped = "acct_hol_feed_cursor" + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + await insert_at(pool, scoped, "rrev_cursor_a", 20) + page = await store.changes_after(account_id=scoped) + decoded = decode_reporting_change_cursor_v1(page.cursor, scoped) + assert decoded["xid"] == "20" + assert not is_legacy_account_change_cursor(decoded) + with pytest.raises(ValueError): + decode_reporting_change_cursor_v1(page.cursor, "acct_hol_feed_elsewhere") + with pytest.raises(ValueError, match="does not belong to this feed"): + await store.changes_after(cursor=page.cursor) + + +async def test_cursor_values_the_database_would_reject_fail_as_value_error() -> None: + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + for body, account in ( + ({"v": 1, "account": "a", "seq": "9223372036854775808"}, "a"), + ({"v": 1, "account": "a", "seq": "1", "xid": "18446744073709551616"}, "a"), + ({"v": 1, "account": None, "seq": "1", "xid": "99999999999999999999"}, None), + ({"v": 1, "account": None, "seq": "1"}, None), + ({"v": 1, "account": "a", "seq": ""}, "a"), + ({"v": 1, "account": "a", "seq": "1", "xid": "x"}, "a"), + ): + with pytest.raises(ValueError): + await store.changes_after(account_id=account, cursor=b64(body)) + decode_reporting_change_cursor_v1( + b64( + { + "v": 1, + "account": "a", + "seq": "9223372036854775807", + "xid": "18446744073709551615", + } + ), + "a", + ) + +# -- account cursors issued before the (xid, seq) order -------------------------------------- + + +async def test_a_seq_only_cursor_maps_to_a_position_that_skips_nothing_and_upgrades() -> None: + scoped = "acct_hol_legacy_map" + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + # r2 took a higher seq but a lower xid than r1: the late commit the old order could skip. + await insert_at(pool, scoped, "r0", 5) # read long ago, older than any unread change + await insert_at(pool, scoped, "r1", 20) + await insert_at(pool, scoped, "r2", 10) + await insert_at(pool, scoped, "r3", 30) + page = await store.changes_after( + account_id=scoped, cursor=legacy_cursor(scoped, await seq_of(pool, "r1")) + ) + # r2 is delivered; r1 is redelivered because its xid is above r2's; r0 is not, so the + # position is not simply xid 0. + assert ids(page) == ["r2", "r1", "r3"] + upgraded = decode_reporting_change_cursor_v1(page.cursor, scoped) + assert upgraded["xid"] == "30" + assert ids(await store.changes_after(account_id=scoped, cursor=page.cursor)) == [] + + # A seq-only cursor at the end of the feed has nothing unread: it maps to the snapshot's + # xmin, and an empty page still upgrades the cursor. + end = await store.changes_after( + account_id=scoped, cursor=legacy_cursor(scoped, await seq_of(pool, "r3")) + ) + assert ids(end) == [] + mapped = decode_reporting_change_cursor_v1(end.cursor, scoped) + assert mapped["seq"] == await seq_of(pool, "r3") + assert int(mapped["xid"]) > 30 + assert not is_legacy_account_change_cursor(mapped) + + +async def test_a_caught_up_seq_only_cursor_does_not_skip_a_change_from_an_in_flight_transaction() -> ( + None +): + scoped = "acct_hol_legacy_inflight" async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + await insert_at(pool, scoped, "caught_up", 10) + head = await seq_of(pool, "caught_up") + opened = await pool.getconn() + try: + await opened.execute(INSERT_CHANGE, (scoped, "inflight")) # at or above xmin + during = await store.changes_after( + account_id=scoped, cursor=legacy_cursor(scoped, head) + ) + assert ids(during) == [] + await opened.commit() + rest = await until( + lambda: store.changes_after(account_id=scoped, cursor=during.cursor), ["inflight"] + ) + assert ids(rest) == ["inflight"], "the mapped position is below the open transaction" + finally: + await pool.putconn(opened) + + +async def test_a_seq_only_cursor_expires_when_any_change_with_a_higher_seq_was_pruned() -> None: + scoped = "acct_hol_legacy_pruned" + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + # Pruned: (xid 20, seq s1) and (xid 10, seq s2). The highest (xid, seq) is s1's, but s2 > s1. + await insert_at(pool, scoped, "p1", 20, age_days=40) + await insert_at(pool, scoped, "p2", 10, age_days=40) + await insert_at(pool, scoped, "p3", 30) + s1, s2 = await seq_of(pool, "p1"), await seq_of(pool, "p2") + assert await store.prune_changes(change_retention_days=30) >= 2 + + async def expired(cursor: str) -> bool: + try: + await store.changes_after(account_id=scoped, cursor=cursor) + except ReportingChangeCursorExpiredError as error: + assert error.code == "CURSOR_EXPIRED" + return True + return False + + # p2 was pruned after seq s1: a tuple compare would miss it. + assert await expired(legacy_cursor(scoped, s1)) is True + assert await expired(legacy_cursor(scoped, "0")) is True + assert await expired(legacy_cursor(scoped, s2)) is False + # A cursor at the horizon has passed everything pruned; one below it in (xid, seq) order + # may have missed a pruned change. + at_horizon = encode_reporting_change_cursor_v1(account=scoped, seq=s1, xid="20") + assert await expired(at_horizon) is False + below = encode_reporting_change_cursor_v1(account=scoped, seq=s2, xid="10") + assert await expired(below) is True + assert ids(await store.changes_after(account_id=scoped)) == ["p3"], "no cursor: horizon" + + +async def test_a_seq_only_consumer_registration_holds_by_seq_until_it_saves_a_new_cursor() -> None: + scoped = "acct_hol_legacy_consumer" + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + await insert_at(pool, scoped, "c1", 10, age_days=40) + await insert_at(pool, scoped, "c2", 20, age_days=40) + await insert_at(pool, scoped, "c3", 30, age_days=40) + s1 = await seq_of(pool, "c1") + async with pool.connection() as connection: + await connection.execute( + "INSERT INTO adcp_reporting_feed_consumers" + " (consumer_name, account_id, cursor_xid, cursor_seq)" + " VALUES ('legacy-sink', %s, NULL, %s::bigint)", + (scoped, s1), + ) + + async def registered() -> tuple[Any, ...]: + return ( + await fetch_all( + pool, + "SELECT cursor_xid::text, cursor_seq::text FROM adcp_reporting_feed_consumers" + " WHERE consumer_name = 'legacy-sink'", + ) + )[0] + + await store.prune_changes(change_retention_days=30) + assert await remaining(pool, scoped) == ["c2", "c3"], "c1 was passed; c2 and c3 are held" + # Saving a seq-only cursor registers a mapped position, below every unread change. + assert await store.save_feed_consumer_cursor("legacy-sink", legacy_cursor(scoped, s1)) + assert (await registered())[0] == "20", "the oldest unread change is c2, at xid 20" + # A cursor in the new format replaces it, forward only. + page = await store.changes_after(account_id=scoped, cursor=legacy_cursor(scoped, s1)) + assert ids(page) == ["c2", "c3"] + assert await store.save_feed_consumer_cursor("legacy-sink", page.cursor) is True + assert await registered() == ("30", await seq_of(pool, "c3")) + + +# -- pruning in (xid, seq) order ------------------------------------------------------------- + + +async def test_an_account_consumer_holds_every_change_after_its_cursor_in_xid_seq_order() -> None: + scoped = "acct_hol_prune_order" + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + await insert_at(pool, scoped, "o1", 10, age_days=40) + await insert_at(pool, scoped, "o2", 30, age_days=40) + await insert_at(pool, scoped, "o3", 20, age_days=40) # higher seq than o2, lower xid + o1, o3 = await seq_of(pool, "o1"), await seq_of(pool, "o3") + cursor = encode_reporting_change_cursor_v1(account=scoped, seq=o3, xid="20") + await store.save_feed_consumer_cursor("prune-order", cursor) + await store.prune_changes(change_retention_days=30) + assert await remaining(pool, scoped) == ["o2"], "o2 follows the cursor in xid order" + horizon = await fetch_all( + pool, + "SELECT pruned_through_xid::text, pruned_through_xid_seq::text," + " pruned_through_seq::text FROM adcp_reporting_change_account_horizon" + " WHERE account_id = %s", + (scoped,), + ) + assert horizon == [("20", o3, o3)] + # The consumer reads on from where it was and is not told it missed anything. + assert ids(await store.changes_after(account_id=scoped, cursor=cursor)) == ["o2"] + # A consumer that had only passed o1 has missed o3 (pruned): expired. + behind = encode_reporting_change_cursor_v1(account=scoped, seq=o1, xid="10") + with pytest.raises(ReportingChangeCursorExpiredError): + await store.changes_after(account_id=scoped, cursor=behind) + + +async def test_the_horizon_only_moves_forward_and_another_accounts_consumer_holds_nothing() -> None: + scoped = "acct_hol_prune_forward" + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + await insert_at(pool, scoped, "f1", 50, age_days=40) + await store.save_feed_consumer_cursor( + "prune-elsewhere", + encode_reporting_change_cursor_v1(account="acct_hol_prune_elsewhere", seq="1", xid="1"), + ) + await store.prune_changes(change_retention_days=30) + assert await remaining(pool, scoped) == [] + + async def horizon_xid() -> str: + rows = await fetch_all( + pool, + "SELECT pruned_through_xid::text FROM adcp_reporting_change_account_horizon" + " WHERE account_id = %s", + (scoped,), + ) + return str(rows[0][0]) + + assert await horizon_xid() == "50" + await insert_at(pool, scoped, "f2", 40, age_days=40) # lower xid than the horizon + await store.prune_changes(change_retention_days=30) + assert await horizon_xid() == "50", "a lower (xid, seq) never lowers the horizon" + + +async def test_a_pruning_batch_cut_by_its_limit_leaves_nothing_retained_below_the_horizon() -> None: + scoped = "acct_hol_prune_batch" + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + await insert_at(pool, scoped, "a", 20, age_days=40) + await insert_at(pool, scoped, "b", 30, age_days=40) + await insert_at(pool, scoped, "c", 10, age_days=40) # the highest seq, the lowest xid + await store.prune_changes(change_retention_days=30, limit=2) + kept = await remaining(pool, scoped) + assert kept == ["b"], "victims are taken in (xid, seq) order: c then a" + assert ( + ids(await store.changes_after(account_id=scoped)) == kept + ), "a cold start loses nothing" + + +async def test_period_retirement_holds_a_period_in_xid_seq_order_for_account_consumers() -> None: + from adcp_reporting_storage.ledger.change_feed import feed_consumer_behind_change + + async with isolated_reporting_pool() as pool: + store = await feed_store(pool) + async with pool.connection() as connection: + await connection.execute( + "INSERT INTO adcp_reporting_changes (xid, account_id, record_kind, record_id," + " obligation_id) VALUES ('30'::xid8, 'acct_a', 'revision', 'k1', 'o')," + " ('20'::xid8, 'acct_a', 'revision', 'k2', 'o')" + ) + k1, k2 = await seq_of(pool, "k1"), await seq_of(pool, "k2") + sql = ( + "SELECT change.record_id FROM adcp_reporting_changes change" + " JOIN adcp_reporting_feed_consumers consumer ON TRUE WHERE " + + feed_consumer_behind_change("consumer", "change") + + " ORDER BY change.xid, change.seq" + ) + # The consumer passed (xid 20, seq k2): k1 (xid 30, a lower seq) is still ahead of it. + await store.save_feed_consumer_cursor( + "hold", encode_reporting_change_cursor_v1(account="acct_a", seq=k2, xid="20") + ) + assert [row[0] for row in await fetch_all(pool, sql)] == ["k1"] + # A seq-only registration holds by seq alone. + async with pool.connection() as connection: + await connection.execute( + "UPDATE adcp_reporting_feed_consumers SET cursor_xid = NULL, cursor_seq = %s::bigint", + (k1,), + ) + assert [row[0] for row in await fetch_all(pool, sql)] == ["k2"] + + +# -- the migration ----------------------------------------------------------------------------- + +#: The change tables as the previous release created them. +PREVIOUS_CHANGES_DDL = """ +CREATE TABLE adcp_reporting_changes ( + seq BIGSERIAL PRIMARY KEY, xid xid8 NOT NULL DEFAULT pg_current_xact_id(), + account_id TEXT NOT NULL, + record_kind TEXT NOT NULL + CHECK (record_kind IN ('obligation', 'revision', 'adjustment', 'retirement')), + record_id TEXT NOT NULL, obligation_id TEXT NOT NULL, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()); +CREATE INDEX adcp_reporting_changes_account ON adcp_reporting_changes (account_id, seq); +CREATE TABLE adcp_reporting_change_account_horizon ( + account_id TEXT PRIMARY KEY, pruned_through_seq BIGINT NOT NULL); +INSERT INTO adcp_reporting_change_account_horizon VALUES ('acct_upgrade', 77); +CREATE TABLE adcp_reporting_feed_consumers ( + consumer_name TEXT PRIMARY KEY CHECK (consumer_name ~ '^[A-Za-z0-9_.:-]{1,128}$'), + account_id TEXT, cursor_xid xid8, cursor_seq BIGINT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CHECK ((account_id IS NULL) = (cursor_xid IS NOT NULL))); +INSERT INTO adcp_reporting_feed_consumers (consumer_name, account_id, cursor_xid, cursor_seq) + VALUES ('old-account-sink', 'acct_upgrade', NULL, 5); +""" + + +async def test_concurrent_appliers_of_the_changes_migration_all_succeed() -> None: + from adcp_reporting_storage.ledger.change_feed import REPORTING_CHANGES_DDL_PATH + + async with isolated_reporting_pool() as pool: + await PgReportingLedgerStore(pool=pool).create_schema() # the ledger the index needs + ddl = REPORTING_CHANGES_DDL_PATH.read_text() + + async def apply() -> None: + async with pool.connection() as connection: + await connection.execute(ddl) + + results = await asyncio.gather(*(apply() for _ in range(8)), return_exceptions=True) + assert [r for r in results if isinstance(r, BaseException)] == [] + + +async def test_reapplying_the_changes_migration_upgrades_an_older_install_in_place() -> None: + from adcp_reporting_storage.ledger.change_feed import REPORTING_CHANGES_DDL_PATH + + psycopg = pytest.importorskip("psycopg") + + async with isolated_reporting_pool() as pool: + await PgReportingLedgerStore(pool=pool).create_schema() + async with pool.connection() as connection: + await connection.execute(PREVIOUS_CHANGES_DDL) + with pytest.raises(psycopg.errors.CheckViolation): + async with pool.connection() as connection: + await connection.execute( + "INSERT INTO adcp_reporting_feed_consumers" + " (consumer_name, account_id, cursor_xid, cursor_seq)" + " VALUES ('old-deployment-sink', NULL, NULL, 5)" + ) + ddl = REPORTING_CHANGES_DDL_PATH.read_text() + for _ in range(2): # idempotent + async with pool.connection() as connection: + await connection.execute(ddl) + + assert await fetch_all( + pool, + "SELECT account_id, pruned_through_seq::text, pruned_through_xid::text," + " pruned_through_xid_seq::text FROM adcp_reporting_change_account_horizon", + ) == [("acct_upgrade", "77", "0", "0")], "the existing seq horizon is kept" + assert await fetch_all( + pool, + "SELECT consumer_name, cursor_xid::text, cursor_seq::text FROM" + " adcp_reporting_feed_consumers", + ) == [("old-account-sink", None, "5")], "a seq-only account consumer is kept" + # The deployment feed still needs an xid; the old check is gone, the new one stays. + with pytest.raises(psycopg.errors.CheckViolation): + async with pool.connection() as connection: + await connection.execute( + "INSERT INTO adcp_reporting_feed_consumers" + " (consumer_name, account_id, cursor_xid, cursor_seq)" + " VALUES ('bad-deployment', NULL, NULL, 1)" + ) + async with pool.connection() as connection: + await connection.execute( + "INSERT INTO adcp_reporting_feed_consumers" + " (consumer_name, account_id, cursor_xid, cursor_seq)" + " VALUES ('new-account-sink', 'acct_upgrade', '9'::xid8, 1)" + ) + checks = await fetch_all( + pool, + "SELECT conname, pg_get_constraintdef(oid) FROM pg_constraint" + " WHERE conrelid = 'adcp_reporting_feed_consumers'::regclass AND contype = 'c'" + " ORDER BY conname", + ) + assert [name for name, _ in checks if "account_id IS NULL) =" in _] == [] + assert [name for name, _ in checks].count("adcp_reporting_feed_consumers_cursor_scope") == 1 + indexes = { + row[0] + for row in await fetch_all( + pool, + "SELECT indexname FROM pg_indexes WHERE schemaname = current_schema()" + " AND tablename = 'adcp_reporting_changes'", + ) + } + assert "adcp_reporting_changes_account_xid" in indexes + assert "adcp_reporting_changes_account" in indexes, "re-applying never drops an index" + + +async def test_the_account_feed_query_is_served_by_the_account_xid_seq_index_in_order() -> None: + from adcp_reporting_storage.ledger.change_feed import _page_query + + # Autocommit: rows written inside one open transaction carry that transaction's own xid, at or + # above the snapshot's xmin, and the planner would see a histogram nothing satisfies. + async with isolated_reporting_pool(autocommit=True) as pool: await feed_store(pool) async with pool.connection() as connection: await connection.execute( "INSERT INTO adcp_reporting_changes (account_id, record_kind, record_id," - " obligation_id) SELECT 'acct_filler_' || n, 'revision', 'rrev_filler_' || n," - " 'robl_filler' FROM generate_series(1, 2000) AS n" + " obligation_id) SELECT 'acct_filler_' || (n % 2000), 'revision'," + " 'rrev_filler_' || n, 'robl_filler' FROM generate_series(1, 100000) AS n" + ) + # Enough history for the account that a page of 500 is a small slice of it. + await connection.execute( + "INSERT INTO adcp_reporting_changes (account_id, record_kind, record_id," + " obligation_id) SELECT %s, 'revision', 'rrev_history_' || n, 'robl_history'" + " FROM generate_series(1, 3000) AS n", + (ACCOUNT,), ) await connection.execute("ANALYZE adcp_reporting_changes") # The very query the store runs for an account page, planned with its parameters. - plan = await ( - await connection.execute( - "EXPLAIN " + _page_query(account=True), - ("acct_hol_feed", 0, None, None, 500), - ) - ).fetchall() - assert "adcp_reporting_changes_account" in "\n".join(row[0] for row in plan) + plan = "\n".join( + row[0] + for row in await ( + await connection.execute( + "EXPLAIN " + _page_query(account=True), + (ACCOUNT, "0", 0, None, None, 500), + ) + ).fetchall() + ) + assert "adcp_reporting_changes_account_xid" in plan, plan + assert "Sort" not in plan, "the index already yields (xid, seq) order" diff --git a/tests/conformance/reporting/test_reporting_orphan_collector_pg.py b/tests/conformance/reporting/test_reporting_orphan_collector_pg.py new file mode 100644 index 0000000000..497a161f70 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_orphan_collector_pg.py @@ -0,0 +1,469 @@ +"""The opt-in listing collector for orphaned row objects, on PostgreSQL and the filesystem provider. + +Ported from the JavaScript SDK (adcp-client #3291, ``reporting-row-orphan-collector-pg``). +""" + +from __future__ import annotations + +import asyncio +from dataclasses import replace +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import pytest +from adcp_reporting_storage.ledger.row_storage import ReportingRowStoreError +from adcp_reporting_storage.ledger.row_storage_object import ( + ReportingRowObjectListEntryV1, + ReportingRowObjectListInputV1, + ReportingRowObjectListPageV1, +) + +from ._orphan_support import binding_result, orphan_fixture +from .test_reporting_row_storage_pg import fetch_all + + +async def test_deletes_an_orphan_and_keeps_committed_intent_named_and_young_objects( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + committed, _ = await fx.commit_revision("orphans_commit") + # An upload that succeeded and was never recorded anywhere. + orphan = fx.key("acct/orphan.jsonl.gz") + await fx.put_object(orphan) + # Named by an open upload that has not committed yet. + in_flight = fx.key("acct/in-flight.jsonl.gz") + await fx.add_intent("rrev_in_flight", in_flight, await fx.put_object(in_flight)) + young = fx.key("acct/young.jsonl") + await fx.put_object(young) + + everything = [orphan, committed, in_flight, young] + dry = await fx.collect(dry_run=True, bindings=["objects"]) + assert dry.dry_run is True + assert sorted(c.object_ref for c in dry.candidates) == sorted([orphan, young]) + assert {c.binding_id for c in dry.candidates} == {"objects"} + assert dry.would_delete == 2 and dry.deleted == 0 + assert all(fx.on_disk(ref) for ref in everything), "a dry run deletes nothing" + + # At the real clock nothing is old enough. + early = await fx.store.collect_orphaned_row_objects(acknowledge_exclusive_namespace=True) + assert early.deleted == 0 + assert binding_result(early, "objects").too_young == 4 + assert all(fx.on_disk(ref) for ref in everything) + + real = await fx.collect(bindings=["objects"]) + assert real.deleted == 2 + assert not fx.on_disk(orphan) and not fx.on_disk(young) + assert fx.on_disk(committed) and fx.on_disk(in_flight) + assert binding_result(real, "objects").referenced == 2 + assert binding_result(real, "objects").status == "complete" + assert real.completed == ("objects",) + + +async def test_never_touches_keys_outside_the_owned_prefix_or_objects_that_are_not_rows( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + for name in ("a.txt", "b.json", "nested/c.bin"): + await fx.put_object(fx.key(name)) + elsewhere = "orph/not-the-namespace/x.jsonl" + sibling = f"orphan-extra/{fx.namespace_key}/y.jsonl" + for ref in (elsewhere, sibling): + await fx.put_object(ref) + result = await fx.collect(bindings=["objects"]) + assert result.deleted == 0 and result.foreign >= 3 + assert fx.on_disk(elsewhere) and fx.on_disk(sibling) + assert all(fx.on_disk(fx.key(name)) for name in ("a.txt", "b.json", "nested/c.bin")) + + +async def test_skips_read_only_bindings_unless_asked_and_reports_a_provider_that_cannot_list( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + legacy = fx.key("acct/legacy.jsonl", "legacy") + await fx.put_object(legacy) + skipped = await fx.collect() + assert "legacy" in skipped.skipped_read_only + assert binding_result(skipped, "legacy").status == "read_only" + assert fx.on_disk(legacy) + assert skipped.unsupported == ("nolist",) + assert binding_result(skipped, "nolist").status == "unsupported" + assert skipped.fenced == () + + included = await fx.collect(include_read_only=True, bindings=["legacy"]) + assert binding_result(included, "legacy").deleted == 1 + assert not fx.on_disk(legacy) + + +async def test_pages_with_a_per_pass_budget_and_resumes_from_the_returned_cursors( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + refs = [fx.key(f"acct/o{index}.jsonl") for index in range(7)] + for ref in refs: + await fx.put_object(ref) + cursors: dict[str, str] = {} + deleted = 0 + for _ in range(10): + result = await fx.collect(bindings=["objects"], limit=3, page_size=2, cursors=cursors) + deleted += result.deleted + cursors = dict(result.cursors) + if not cursors: + break + assert deleted == 7 and not any(fx.on_disk(ref) for ref in refs) + + +async def test_reports_the_bindings_whose_listing_finished_and_skips_the_ones_it_is_told_to( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + await fx.put_object(fx.key("acct/a.jsonl")) + first = await fx.collect(bindings=["objects", "legacy"], include_read_only=True) + assert set(first.completed) == {"objects", "legacy"} + fx.provider.list_calls.clear() + again = await fx.collect( + bindings=["objects", "legacy"], include_read_only=True, skip_bindings=["objects"] + ) + assert again.completed == ("legacy",) + assert [b.binding_id for b in again.bindings] == ["legacy"] + assert all( + "/legacy/" in call or call.startswith("legacy/") for call in fx.provider.list_calls + ) + + +async def test_a_delete_that_fails_is_counted_and_does_not_stop_the_rest(tmp_path: Path) -> None: + async with orphan_fixture(tmp_path) as fx: + stuck, fine = fx.key("acct/a-stuck.jsonl"), fx.key("acct/b-fine.jsonl") + await fx.put_object(stuck) + await fx.put_object(fine) + fx.provider.failing_keys.add(stuck) + result = await fx.collect(bindings=["objects"]) + assert result.deleted == 1 and result.failed == 1 + assert fx.on_disk(stuck) and not fx.on_disk(fine) + assert binding_result(result, "objects").status == "complete" + # It is retried on the next full listing. + fx.provider.failing_keys.clear() + retried = await fx.collect(bindings=["objects"]) + assert retried.deleted == 1 and not fx.on_disk(stuck) + + +async def test_a_listing_failure_keeps_its_cursor_and_names_the_cause(tmp_path: Path) -> None: + async with orphan_fixture(tmp_path) as fx: + for index in range(4): + await fx.put_object(fx.key(f"acct/o{index}.jsonl")) + partial = await fx.collect(bindings=["objects"], limit=2, page_size=2) + assert binding_result(partial, "objects").status == "partial" + cursor = partial.cursors["objects"] + fx.provider.list_failures.append(ReportingRowStoreError("PROVIDER_UNAVAILABLE")) + # The first page of a resumed cursor fails: the stale cursor is dropped. + failed = await fx.collect(bindings=["objects"], cursors={"objects": cursor}) + outcome = binding_result(failed, "objects") + assert outcome.status == "failed" and outcome.error == "PROVIDER_UNAVAILABLE" + assert "objects" not in failed.cursors + # A failure after the first page keeps the position it reached. + calls = 0 + real_list = fx.provider.list + + async def second_page_fails( + request: ReportingRowObjectListInputV1, real: Any + ) -> ReportingRowObjectListPageV1: + nonlocal calls + calls += 1 + if calls == 2: + raise ReportingRowStoreError("DEADLINE_EXCEEDED") + return await real(request) # type: ignore[no-any-return] + + fx.provider.list_override = second_page_fails + mid = await fx.collect(bindings=["objects"], limit=100, page_size=2) + assert binding_result(mid, "objects").error == "DEADLINE_EXCEEDED" + assert "objects" in mid.cursors + assert real_list is not None + + +async def test_a_record_naming_the_key_at_another_version_or_an_unpruned_row_set_keeps_it( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + ref, version = await fx.commit_revision("named_other_version") + # The same key, now at a different version, is still named by a record at the old one. + await fx.provider.inner.delete( + __import__( + "adcp_reporting_storage.ledger.row_storage_object", fromlist=["x"] + ).ReportingRowObjectDeleteInputV1({"root": "rows"}, ref, version), + __import__( + "adcp_reporting_storage.ledger.row_storage_object", fromlist=["x"] + ).ReportingRowObjectContextV1(), + ) + await fx.put_object(ref) # a new version at the recorded key + result = await fx.collect(bindings=["objects"]) + assert result.deleted == 0 + assert binding_result(result, "objects").referenced == 1 + assert fx.on_disk(ref) + + +async def test_a_row_set_that_is_pruning_or_unavailable_keeps_its_object_and_a_pruned_one_does_not( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + ref, _ = await fx.commit_revision("states") + for state in ("pruning", "unavailable"): + async with fx.pool.connection() as connection: + await connection.execute( + "UPDATE adcp_reporting_row_sets SET rows_state = %s", (state,) + ) + kept = await fx.collect(bindings=["objects"]) + assert kept.deleted == 0 and fx.on_disk(ref), state + async with fx.pool.connection() as connection: + await connection.execute("UPDATE adcp_reporting_row_sets SET rows_state = 'pruned'") + freed = await fx.collect(bindings=["objects"]) + assert freed.deleted == 1 and not fx.on_disk(ref) + + +async def test_an_intent_written_by_an_older_sdk_keeps_its_object(tmp_path: Path) -> None: + async with orphan_fixture(tmp_path) as fx: + ref = fx.key("acct/legacy-intent.jsonl.gz") + await fx.add_intent("rrev_old", ref, await fx.put_object(ref), legacy_names=True) + result = await fx.collect(bindings=["objects"]) + assert result.deleted == 0 and fx.on_disk(ref) + for state in ("sweeping",): + async with fx.pool.connection() as connection: + await connection.execute( + "UPDATE adcp_reporting_row_write_intents SET state = %s", (state,) + ) + # The pre-check is blind to the intent state: an intent in any state names its key. + assert (await fx.collect(bindings=["objects"])).deleted == 0 and fx.on_disk(ref) + + +async def test_a_record_under_another_binding_sharing_the_bucket_and_prefix_keeps_the_object( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + ref = fx.key("acct/shared.jsonl.gz") + version = await fx.put_object(ref) + # An open upload under the twin binding (same bucket and prefix) names the object. + await fx.add_intent("rrev_twin", ref, version, binding="twin") + result = await fx.collect(bindings=["objects"]) + assert result.deleted == 0 and fx.on_disk(ref) + + +async def test_a_hostile_provider_cannot_make_it_delete_outside_the_owned_prefix( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + outside = "other-tenant/data.jsonl" + await fx.put_object(outside) + owned = fx.key("acct/owned.jsonl") + await fx.put_object(owned) + old = datetime(2020, 1, 1, tzinfo=timezone.utc) + stamp = old.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + def entry(ref: Any, **changes: Any) -> Any: + base = ReportingRowObjectListEntryV1(ref, "1:2:3:4", 5, stamp) + return replace(base, **changes) + + hostile = [ + entry(outside), + entry(f"{fx.key('x')}/../../{outside}"), + entry(fx.key("a/./b.jsonl")), + entry(fx.key("a//b.jsonl")), + entry(fx.key("bad key.jsonl")), + entry(fx.key("acct/newline.jsonl\n")), + entry(fx.key("acct/no-version.jsonl"), native_version_ref=""), + entry(fx.key("acct/long-version.jsonl"), native_version_ref="v" * 1025), + entry(fx.key("acct/negative.jsonl"), size_bytes=-1), + entry(fx.key("acct/bad-time.jsonl"), created_at="yesterday"), + entry(fx.key("acct/offset-time.jsonl"), created_at="2020-01-01T00:00:00+02:00"), + entry("x" * 1500 + ".jsonl"), + entry(owned), + ] + + async def hostile_page( + request: ReportingRowObjectListInputV1, real: Any + ) -> ReportingRowObjectListPageV1: + return ReportingRowObjectListPageV1(tuple(hostile), None) + + fx.provider.list_override = hostile_page + result = await fx.collect(bindings=["objects"]) + outcome = binding_result(result, "objects") + assert outcome.status == "complete" + assert outcome.foreign == len(hostile) - 1, "every malformed or foreign entry is dropped" + assert fx.on_disk(outside), "nothing outside the owned prefix is ever touched" + assert all(call.startswith(fx.key("")) for call in fx.provider.delete_calls) + assert len(fx.provider.delete_calls) == 1 + # An entry whose version is not the stored one deletes nothing: the provider pins versions. + assert fx.on_disk(owned) + + +async def test_a_provider_that_repeats_a_token_or_returns_too_many_objects_fails_the_binding( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + await fx.put_object(fx.key("acct/a.jsonl")) + + async def repeat( + request: ReportingRowObjectListInputV1, real: Any + ) -> ReportingRowObjectListPageV1: + return ReportingRowObjectListPageV1((), "same-token") + + fx.provider.list_override = repeat + looped = await fx.collect(bindings=["objects"], limit=100, page_size=10) + outcome = binding_result(looped, "objects") + assert outcome.status == "failed" and outcome.error == "PROVIDER_UNAVAILABLE" + assert "objects" not in looped.cursors, "a repeated token is not a cursor to resume" + + async def too_many( + request: ReportingRowObjectListInputV1, real: Any + ) -> ReportingRowObjectListPageV1: + entry = ReportingRowObjectListEntryV1( + fx.key("a/x.jsonl"), "v", 1, "2020-01-01T00:00:00.000Z" + ) + return ReportingRowObjectListPageV1((entry,) * (request.page_size + 1), None) + + fx.provider.list_override = too_many + many = await fx.collect(bindings=["objects"], page_size=3) + assert binding_result(many, "objects").status == "failed" + assert fx.provider.delete_calls == [] + + +async def test_it_validates_its_inputs_and_requires_the_acknowledgement_to_delete( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + store = fx.store + with pytest.raises(ValueError, match="acknowledge_exclusive_namespace"): + await store.collect_orphaned_row_objects() + for bad in (0, 3600, 86_399, 1.5, "7"): + with pytest.raises(ValueError, match="min_age_seconds"): + await store.collect_orphaned_row_objects( + min_age_seconds=bad, acknowledge_exclusive_namespace=True # type: ignore[arg-type] + ) + for name in ("limit", "page_size"): + for bad in (0, -1, 1.5): + with pytest.raises(ValueError, match=name): + await store.collect_orphaned_row_objects( + dry_run=True, **{name: bad} # type: ignore[arg-type] + ) + with pytest.raises(ValueError, match="not a configured object binding"): + await store.collect_orphaned_row_objects(dry_run=True, bindings=["inline"]) + with pytest.raises(ValueError, match="timezone-aware"): + await store.collect_orphaned_row_objects(dry_run=True, now=datetime(2030, 1, 1)) + # A dry run needs no acknowledgement. + assert (await store.collect_orphaned_row_objects(dry_run=True)).dry_run is True + + +async def test_a_store_without_row_storage_refuses_the_collector() -> None: + from ._generation_support import isolated_reporting_pool + + async with isolated_reporting_pool() as pool: + from adcp_reporting_storage import PgReportingLedgerStore + + store = PgReportingLedgerStore(pool=pool) + with pytest.raises(ReportingRowStoreError) as failure: + await store.collect_orphaned_row_objects(dry_run=True) + assert failure.value.code == "INVALID_INPUT" + + +async def test_the_collector_takes_the_per_object_lock_and_rechecks_under_it( + tmp_path: Path, +) -> None: + """An intent committed while the collector waits on the lock keeps the object.""" + async with orphan_fixture(tmp_path) as fx: + ref = fx.key("acct/raced.jsonl.gz") + version = await fx.put_object(ref) + storage = fx.store._row_storage + assert storage is not None + lock_text = storage._object_lock_text("objects", ref) + holder = await fx.pool.getconn() + try: + await holder.execute( + "SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (lock_text,) + ) + task = asyncio.create_task(fx.collect(bindings=["objects"])) + await _until_blocked(fx.pool, task) + assert not task.done(), "the collector waits on the object's lock" + # While it waits, an upload that names the object commits. + await holder.execute( + "INSERT INTO adcp_reporting_row_write_intents (row_binding_id, account_id," + " row_set_id, content_sha256, state, owner_token, created_objects, expires_at)" + " VALUES ('objects', 'acct', 'rrev_raced', %s, 'open', 'o', %s::jsonb," + " clock_timestamp() + INTERVAL '1 hour')", + ( + "e" * 64, + f'[{{"object_ref": "{ref}", "native_version_ref": "{version}"}}]', + ), + ) + await holder.commit() + finally: + await fx.pool.putconn(holder) + result = await asyncio.wait_for(task, 20) + # The pre-check ran before the intent existed; the recheck under the lock saw it. + assert result.deleted == 0 and fx.on_disk(ref) + assert binding_result(result, "objects").referenced == 1 + + +async def test_it_serializes_with_an_adoption_through_another_binding_sharing_the_prefix( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + ref = fx.key("acct/adopted-by-twin.jsonl.gz") + version = await fx.put_object(ref) + storage = fx.store._row_storage + assert storage is not None + # A writer adopting through the twin binding holds the twin's lock for this object. + holder = await fx.pool.getconn() + try: + await holder.execute( + "SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", + (storage._object_lock_text("twin", ref),), + ) + task = asyncio.create_task(fx.collect(bindings=["objects"])) + await _until_blocked(fx.pool, task) + assert not task.done(), "the delete waits for an adoption through any binding" + await holder.execute( + "INSERT INTO adcp_reporting_row_write_intents (row_binding_id, account_id," + " row_set_id, content_sha256, state, owner_token, created_objects, expires_at)" + " VALUES ('twin', 'acct', 'rrev_twin', %s, 'open', 'o', %s::jsonb," + " clock_timestamp() + INTERVAL '1 hour')", + ( + "e" * 64, + f'[{{"object_ref": "{ref}", "native_version_ref": "{version}"}}]', + ), + ) + await holder.commit() + finally: + await fx.pool.putconn(holder) + result = await asyncio.wait_for(task, 20) + assert result.deleted == 0 and fx.on_disk(ref) + + +async def _until_blocked(pool: Any, task: asyncio.Task[Any]) -> None: + """Wait until some backend is blocked on an advisory lock, proven with ``pg_locks``.""" + for _ in range(400): + if task.done(): + return + rows = await fetch_all( + pool, + "SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND NOT granted", + ) + if rows[0][0]: + return + await asyncio.sleep(0.025) + pytest.fail("the collector never blocked on the object lock") + + +async def test_dry_run_through_the_store_option_reports_and_deletes_nothing( + tmp_path: Path, +) -> None: + from adcp_reporting_storage.ledger.orphan_collector import ReportingOrphanCollectionOptionsV1 + + async with orphan_fixture( + tmp_path, orphan_collection=ReportingOrphanCollectionOptionsV1(dry_run=True) + ) as fx: + ref = fx.key("acct/maintained.jsonl") + await fx.put_object(ref) + operations = dict(fx.store.maintenance_operations()) + assert "orphan_collection" in operations + # Too young at the real clock: nothing is a candidate yet. + result = await operations["orphan_collection"]() + assert result.dry_run is True and result.would_delete == 0 + assert fx.on_disk(ref) diff --git a/tests/conformance/reporting/test_reporting_orphan_fence_pg.py b/tests/conformance/reporting/test_reporting_orphan_fence_pg.py new file mode 100644 index 0000000000..4159d9d746 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_orphan_fence_pg.py @@ -0,0 +1,650 @@ +"""The orphan collector's ownership fence against a cloned or restored ledger database. + +Ported from the JavaScript SDK (adcp-client #3593, ``reporting-orphan-collector-fence-pg``). A +clone is a point-in-time copy of every row-storage table in the live schema: the same installation +identity, namespace key, bindings and fence row, sharing the live database's bucket. +""" + +from __future__ import annotations + +import asyncio +import json +import time +import uuid +from pathlib import Path +from typing import Any + +import pytest +from adcp_reporting_storage.ledger.orphan_collector import ( + ReportingOrphanCollectionOptionsV1, + ReportingOrphanCollectionStateV1, + run_orphan_collection_pass, +) +from adcp_reporting_storage.ledger.row_storage import ReportingRowStoreError + +from ._orphan_support import ( + HookedProvider, + binding_result, + clone_schema, + cloned_store, + orphan_fixture, +) +from .test_reporting_row_storage_pg import fetch_all + + +def fenced_cause(result: Any, binding_id: str = "objects") -> str | None: + outcome = binding_result(result, binding_id) + return outcome.fence.cause if outcome.fence else None + + +async def test_the_first_real_pass_claims_epoch_one_and_each_later_pass_the_next( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + assert await fx.fence_row() is None + first = await fx.collect(bindings=["objects"]) + assert first.fenced == () + assert fx.fence_files() == ["0000000000000001.json"] + assert (await fx.fence_row())[:1] == (1,) # type: ignore[index] + await fx.collect(bindings=["objects"]) + # A claim keeps its predecessor (an unfinished claim is recognised by it) and prunes the + # rest. + assert fx.fence_files() == ["0000000000000001.json", "0000000000000002.json"] + await fx.collect(bindings=["objects"]) + assert fx.fence_files() == ["0000000000000002.json", "0000000000000003.json"] + row = await fx.fence_row() + assert row is not None and row[0] == 3 + assert row[2] is not None, "a later claim records the token it superseded" + + +async def test_the_fence_chain_is_not_a_row_object_it_is_neither_listed_foreign_nor_deleted( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + await fx.collect(bindings=["objects"]) + assert fx.fence_files() + storage = fx.store._row_storage + assert storage is not None + listing = await storage.list_objects("objects", page_size=100) + assert listing.objects == () and listing.foreign == 0 + result = await fx.collect(bindings=["objects"]) + assert result.deleted == 0 and result.foreign == 0 + assert fx.fence_files(), "the collector never deletes its own fence" + + +async def test_a_dry_run_checks_the_fence_without_claiming_it(tmp_path: Path) -> None: + async with orphan_fixture(tmp_path) as fx: + orphan = fx.key("acct/orphan.jsonl") + await fx.put_object(orphan) + dry = await fx.collect(dry_run=True, bindings=["objects"]) + assert dry.would_delete == 1 and dry.fenced == () + assert fx.fence_files() == [] and await fx.fence_row() is None, "no claim, no object" + # It reports a fenced namespace the same way a real pass does. + fx.write_fence(4) + refused = await fx.collect(dry_run=True, bindings=["objects"]) + assert refused.fenced == ("objects",) + assert fenced_cause(refused) == "claimed_elsewhere" + assert refused.would_delete == 0 + + +async def test_a_clone_taken_before_the_original_last_collected_is_fenced_and_deletes_nothing( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + await fx.collect(bindings=["objects"]) # epoch 1 + schema = await clone_schema(fx.pool) + committed, _ = await fx.commit_revision("clone_live") # written after the copy + await fx.collect(bindings=["objects"]) # the original claims epoch 2 + + async with cloned_store(fx, schema) as clone: + result = await clone.collect_orphaned_row_objects( + now=_later(), acknowledge_exclusive_namespace=True, bindings=["objects"] + ) + assert result.fenced == ("objects",) and result.deleted == 0 + assert fenced_cause(result) == "fence_advanced" + outcome = binding_result(result, "objects") + assert outcome.fence is not None + assert (outcome.fence.fence_epoch, outcome.fence.recorded_epoch) == (2, 1) + assert "take_over_orphan_collector_fence" in outcome.fence.detail + assert fx.on_disk(committed), "the clone cannot delete an object the original wrote" + assert result.cursors == {} + + # Control: the same clone, once it has deliberately taken over, deletes it. The fence + # is the only thing standing between the clone and the live object. + await clone.take_over_orphan_collector_fence(acknowledge_previous_owner_gone=True) + deleted = await clone.collect_orphaned_row_objects( + now=_later(), acknowledge_exclusive_namespace=True, bindings=["objects"] + ) + assert deleted.deleted >= 1 and not fx.on_disk(committed) + + +async def test_a_clone_that_collects_first_claims_the_namespace_and_fences_the_original( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + schema = await clone_schema(fx.pool) # neither side has claimed yet + async with cloned_store(fx, schema) as clone: + first = await clone.collect_orphaned_row_objects( + now=_later(), acknowledge_exclusive_namespace=True, bindings=["objects"] + ) + assert first.fenced == () + original = await fx.collect(bindings=["objects"]) + assert original.fenced == ("objects",) + assert fenced_cause(original) == "claimed_elsewhere", "first claim wins" + + +async def test_a_restore_to_an_older_database_state_is_fenced_until_an_operator_takes_over( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + await fx.collect(bindings=["objects"]) + await fx.collect(bindings=["objects"]) # bucket epoch 2 + async with fx.pool.connection() as connection: + # The restore: the database's own record goes back to epoch 1. + await connection.execute( + "UPDATE adcp_reporting_orphan_collector_fences SET epoch = 1," + " token = %s, previous_token = NULL", + (str(uuid.uuid4()),), + ) + restored = await fx.collect(bindings=["objects"]) + assert fenced_cause(restored) == "fence_advanced" + taken = await fx.store.take_over_orphan_collector_fence( + acknowledge_previous_owner_gone=True + ) + assert [(t.binding_ids, t.epoch, t.previous_epoch) for t in taken] == [(("objects",), 3, 2)] + after = await fx.collect(bindings=["objects"]) + assert after.fenced == () + + +async def test_a_takeover_fences_out_the_previous_owner(tmp_path: Path) -> None: + async with orphan_fixture(tmp_path) as fx: + await fx.collect(bindings=["objects"]) + schema = await clone_schema(fx.pool) + async with cloned_store(fx, schema) as clone: + await clone.take_over_orphan_collector_fence(acknowledge_previous_owner_gone=True) + out = await fx.collect(bindings=["objects"]) + assert fenced_cause(out) == "fence_advanced" + with pytest.raises(ValueError, match="acknowledge_previous_owner_gone"): + await clone.take_over_orphan_collector_fence(acknowledge_previous_owner_gone=False) + + +async def test_a_takeover_validates_its_bindings_and_covers_a_shared_fence_once( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + store = fx.store + for bad, message in ( + (["inline"], "not a configured object binding"), + (["nolist"], "cannot list"), + (["nope"], "not a configured object binding"), + ): + with pytest.raises(ValueError, match=message): + await store.take_over_orphan_collector_fence( + acknowledge_previous_owner_gone=True, bindings=bad + ) + # ``objects`` and its read-only ``twin`` share one bucket and prefix, so one fence. + taken = await store.take_over_orphan_collector_fence( + acknowledge_previous_owner_gone=True, include_read_only=True + ) + shared = [t for t in taken if "objects" in t.binding_ids] + assert len(shared) == 1 and set(shared[0].binding_ids) == {"objects", "twin"} + # Read-only bindings are skipped unless asked for. + default = await store.take_over_orphan_collector_fence(acknowledge_previous_owner_gone=True) + assert all("legacy" not in t.binding_ids for t in default) + only = await store.take_over_orphan_collector_fence( + acknowledge_previous_owner_gone=True, bindings=["legacy"] + ) + assert [t.binding_ids for t in only] == [("legacy",)] + + +async def test_bindings_that_share_a_bucket_and_prefix_share_one_fence_and_one_claim_per_pass( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + result = await fx.collect(bindings=["objects", "twin"], include_read_only=True) + assert result.fenced == () + assert fx.fence_files() == ["0000000000000001.json"], "one claim for both bindings" + rows = await fetch_all( + fx.pool, "SELECT count(*) FROM adcp_reporting_orphan_collector_fences" + ) + assert rows == [(1,)] + + +async def test_a_database_with_no_recorded_claim_refuses_a_namespace_that_already_has_one( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + fx.write_fence(3) + result = await fx.collect(bindings=["objects"]) + assert fenced_cause(result) == "claimed_elsewhere" + outcome = binding_result(result, "objects") + assert outcome.fence is not None and outcome.fence.fence_epoch == 3 + assert outcome.fence.recorded_epoch is None + assert await fx.fence_row() is None, "a refusal never records a claim" + + +async def test_a_recorded_claim_with_no_fence_in_the_bucket_is_refused_as_missing( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + await fx.collect(bindings=["objects"]) + await fx.collect(bindings=["objects"]) + for name in fx.fence_files(): + (fx.fence_dir() / name).unlink() + result = await fx.collect(bindings=["objects"]) + assert fenced_cause(result) == "fence_missing" + assert fx.fence_files() == [], "never silently re-claimed" + + +async def test_a_database_newer_than_its_bucket_is_refused_as_regressed(tmp_path: Path) -> None: + async with orphan_fixture(tmp_path) as fx: + await fx.collect(bindings=["objects"]) + async with fx.pool.connection() as connection: + await connection.execute( + "UPDATE adcp_reporting_orphan_collector_fences SET epoch = 5, token = %s," + " previous_token = %s", + (str(uuid.uuid4()), str(uuid.uuid4())), + ) + result = await fx.collect(bindings=["objects"]) + assert fenced_cause(result) == "fence_regressed" + outcome = binding_result(result, "objects") + assert outcome.fence is not None + assert (outcome.fence.fence_epoch, outcome.fence.recorded_epoch) == (1, 5) + + +async def test_two_claims_of_one_epoch_the_loser_is_fenced_as_a_token_mismatch( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + await fx.collect(bindings=["objects"]) + (fx.fence_dir() / "0000000000000001.json").unlink() + fx.write_fence(1) # another claim of epoch 1, under another token + result = await fx.collect(bindings=["objects"]) + assert fenced_cause(result) == "token_mismatch" + + +async def test_a_lost_race_for_the_next_epoch_is_a_lost_claim(tmp_path: Path) -> None: + async with orphan_fixture(tmp_path) as fx: + await fx.collect(bindings=["objects"]) + raced = False + + async def other_database_claims_first(put: Any) -> None: + nonlocal raced + if "_collector-fence/0000000000000002" in put.object_ref and not raced: + raced = True + fx.write_fence(2) # another database wins the create-only write + + fx.provider.on_put = other_database_claims_first + result = await fx.collect(bindings=["objects"]) + assert fenced_cause(result) == "claim_lost" + outcome = binding_result(result, "objects") + assert outcome.fence is not None and outcome.fence.fence_epoch == 2 + + +@pytest.mark.parametrize( + "damage", + ["garbage", "other_namespace", "oversized", "extra_field", "bad_token", "wrong_epoch"], +) +async def test_malformed_foreign_namespace_or_oversized_fence_objects_fail_closed( + tmp_path: Path, damage: str +) -> None: + async with orphan_fixture(tmp_path) as fx: + await fx.put_object(fx.key("acct/orphan.jsonl")) + good = str(uuid.uuid4()) + if damage == "garbage": + fx.write_fence(1, raw=b"not json\n") + elif damage == "other_namespace": + fx.write_fence(1, namespace="f" * 64) + elif damage == "oversized": + fx.write_fence(1, raw=b"x" * 5000) + elif damage == "extra_field": + fx.write_fence( + 1, + raw=json.dumps( + { + "v": 1, + "namespace_key": fx.namespace_key, + "epoch": 1, + "token": good, + "previous_token": None, + "claimed_at": "2026-01-01T00:00:00.000Z", + "extra": 1, + } + ).encode(), + ) + elif damage == "bad_token": + fx.write_fence(1, token="not-a-uuid") + else: + fx.write_fence(1) + (fx.fence_dir() / "0000000000000001.json").rename( + fx.fence_dir() / "0000000000000002.json" + ) + result = await fx.collect(bindings=["objects"]) + assert fenced_cause(result) == "fence_unreadable", damage + assert result.deleted == 0 and fx.on_disk(fx.key("acct/orphan.jsonl")) + + +async def test_two_versions_of_one_epoch_are_ambiguous_and_fail_closed(tmp_path: Path) -> None: + from adcp_reporting_storage.ledger.row_storage_object import ( + ReportingRowObjectListEntryV1, + ReportingRowObjectListPageV1, + ) + + async with orphan_fixture(tmp_path) as fx: + await fx.collect(bindings=["objects"]) + key = f"orph/{fx.namespace_key}/_collector-fence/0000000000000001.json" + duplicate = ReportingRowObjectListPageV1( + tuple( + ReportingRowObjectListEntryV1(key, f"{index}:1:1:1", 10, "2026-01-01T00:00:00.000Z") + for index in (1, 2) + ), + None, + ) + fx.provider.fence_list_override = duplicate + result = await fx.collect(bindings=["objects"]) + assert fenced_cause(result) == "fence_unreadable" + + +async def test_an_unfinished_claim_is_completed_by_the_next_pass(tmp_path: Path) -> None: + async with orphan_fixture(tmp_path) as fx: + fx.provider.put_failures.append(RuntimeError("bucket unavailable")) + crashed = await fx.collect(bindings=["objects"]) + outcome = binding_result(crashed, "objects") + assert outcome.status == "failed" and outcome.fence is None, "a failure is not a fence" + assert fx.fence_files() == [], "the object was never written" + row = await fx.fence_row() + assert row is not None and row[0] == 1, "the intent was committed first" + recovered = await fx.collect(bindings=["objects"]) + assert recovered.fenced == () and binding_result(recovered, "objects").status == "complete" + assert fx.fence_files() == ["0000000000000001.json"] + done = await fx.fence_row() + assert done is not None and done[1] != row[1], "a finished claim mints a fresh token" + + +async def test_a_copy_taken_while_a_claim_was_unfinished_cannot_adopt_the_same_token( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + fx.provider.put_failures.append(RuntimeError("bucket unavailable")) + await fx.collect(bindings=["objects"]) # intent committed, object never written + schema = await clone_schema(fx.pool) # the copy carries the same unfinished intent + original = await fx.collect(bindings=["objects"]) + assert original.fenced == () + async with cloned_store(fx, schema) as clone: + result = await clone.collect_orphaned_row_objects( + now=_later(), acknowledge_exclusive_namespace=True, bindings=["objects"] + ) + assert fenced_cause(result) == "token_mismatch" + assert result.deleted == 0 + + +async def test_two_hosts_of_the_same_database_collecting_at_once_do_not_fence_each_other( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + for index in range(6): + await fx.put_object(fx.key(f"acct/o{index}.jsonl")) + first, second = await asyncio.gather( + fx.collect(bindings=["objects"]), fx.collect(bindings=["objects"]) + ) + assert first.fenced == () and second.fenced == () + assert first.deleted + second.deleted >= 6 + row = await fx.fence_row() + assert row is not None and row[0] == 2 + + +async def test_another_database_claiming_mid_pass_stops_the_deletes_at_the_next_check( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + refs = [fx.key(f"acct/o{index:03d}.jsonl") for index in range(60)] + for ref in refs: + await fx.put_object(ref) + deleted = 0 + + async def other_database_claims(object_ref: str) -> None: + nonlocal deleted + deleted += 1 + if deleted == 10: + fx.write_fence(99) # a newer claim appears while this pass is running + + fx.provider.on_delete = other_database_claims + result = await fx.collect(bindings=["objects"]) + outcome = binding_result(result, "objects") + assert outcome.status == "fenced" and fenced_cause(result) == "fence_advanced" + assert outcome.deleted == 25, "a check before the first delete and every 25 after it" + assert sum(fx.on_disk(ref) for ref in refs) == 35 + assert result.deleted == 25, "a mid-pass refusal keeps what went before" + + +async def test_the_fence_is_claimed_before_the_first_delete_of_a_pass(tmp_path: Path) -> None: + async with orphan_fixture(tmp_path) as fx: + await fx.put_object(fx.key("acct/o.jsonl")) + seen: list[list[str]] = [] + + async def look(object_ref: str) -> None: + seen.append(fx.fence_files()) + + fx.provider.on_delete = look + await fx.collect(bindings=["objects"]) + assert seen == [["0000000000000001.json"]] + + +async def test_without_the_fence_table_a_real_pass_and_a_dry_run_refuse_and_delete_nothing( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + orphan = fx.key("acct/o.jsonl") + await fx.put_object(orphan) + async with fx.pool.connection() as connection: + await connection.execute("DROP TABLE adcp_reporting_orphan_collector_fences") + for dry_run in (False, True): + result = await fx.collect(bindings=["objects"], dry_run=dry_run) + assert fenced_cause(result) == "migration_required" + assert result.deleted == 0 and result.would_delete == 0 and fx.on_disk(orphan) + with pytest.raises(ReportingRowStoreError, match="fence table does not exist"): + await fx.store.take_over_orphan_collector_fence(acknowledge_previous_owner_gone=True) + + +async def test_the_fence_table_is_part_of_the_row_storage_ddl_and_reapplying_it_changes_nothing( + tmp_path: Path, +) -> None: + from adcp_reporting_storage.ledger.row_storage import REPORTING_ROW_STORAGE_DDL_PATH + + async with orphan_fixture(tmp_path) as fx: + await fx.collect(bindings=["objects"]) + before = await fx.fence_row() + async with fx.pool.connection() as connection: + await connection.execute(REPORTING_ROW_STORAGE_DDL_PATH.read_text()) + assert await fx.fence_row() == before + columns = await fetch_all( + fx.pool, + "SELECT column_name FROM information_schema.columns WHERE table_schema =" + " current_schema() AND table_name = 'adcp_reporting_orphan_collector_fences'" + " ORDER BY column_name", + ) + assert [row[0] for row in columns] == [ + "claimed_at", + "epoch", + "fence_scope", + "previous_token", + "token", + ] + + +async def test_a_provider_failure_while_reading_or_writing_the_fence_is_failed_not_fenced( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + await fx.put_object(fx.key("acct/o.jsonl")) + fx.provider.fence_list_failures.append(ReportingRowStoreError("PROVIDER_UNAVAILABLE")) + read = await fx.collect(bindings=["objects"]) + outcome = binding_result(read, "objects") + assert outcome.status == "failed" and outcome.error == "PROVIDER_UNAVAILABLE" + assert read.fenced == () and read.deleted == 0 + fx.provider.put_failures.append(RuntimeError("down")) + write = await fx.collect(bindings=["objects"]) + assert binding_result(write, "objects").status == "failed" and write.fenced == () + ok = await fx.collect(bindings=["objects"]) + assert ok.deleted == 1 + + +async def test_a_failed_claim_keeps_the_resume_position_of_a_listing(tmp_path: Path) -> None: + async with orphan_fixture(tmp_path) as fx: + for index in range(4): + await fx.put_object(fx.key(f"acct/o{index}.jsonl")) + partial = await fx.collect(bindings=["objects"], limit=2, page_size=2) + cursor = partial.cursors["objects"] + fx.provider.fence_list_failures.append(ReportingRowStoreError("PROVIDER_UNAVAILABLE")) + again = await fx.collect( + bindings=["objects"], limit=2, page_size=2, cursors={"objects": cursor} + ) + assert binding_result(again, "objects").status == "failed" + assert again.cursors == {"objects": cursor}, "the listing never started" + + +async def test_a_takeover_recovers_every_refusal_and_moves_past_a_database_ahead_of_its_bucket( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + # claimed_elsewhere + fx.write_fence(3) + assert fenced_cause(await fx.collect(bindings=["objects"])) == "claimed_elsewhere" + await fx.store.take_over_orphan_collector_fence(acknowledge_previous_owner_gone=True) + assert (await fx.collect(bindings=["objects"])).fenced == () + row = await fx.fence_row() + assert row is not None and row[0] >= 4 + # fence_regressed: the database is ahead of its bucket + async with fx.pool.connection() as connection: + await connection.execute( + "UPDATE adcp_reporting_orphan_collector_fences SET epoch = 50, token = %s", + (str(uuid.uuid4()),), + ) + assert fenced_cause(await fx.collect(bindings=["objects"])) == "fence_regressed" + await fx.store.take_over_orphan_collector_fence(acknowledge_previous_owner_gone=True) + assert (await fx.collect(bindings=["objects"])).fenced == () + row = await fx.fence_row() + assert row is not None and row[0] > 50, "a takeover moves past a database that is ahead" + # fence_unreadable + fx.write_fence(row[0] + 1, raw=b"damaged") + assert fenced_cause(await fx.collect(bindings=["objects"])) == "fence_unreadable" + await fx.store.take_over_orphan_collector_fence(acknowledge_previous_owner_gone=True) + assert (await fx.collect(bindings=["objects"])).fenced == () + assert len(fx.fence_files()) == 2 or len(fx.fence_files()) == 1 + + +async def test_a_fenced_clone_refuses_every_binding_that_shares_the_fence_with_one_check( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + fx.write_fence(7) + fx.provider.list_calls.clear() + result = await fx.collect(bindings=["objects", "twin"], include_read_only=True) + assert set(result.fenced) == {"objects", "twin"} + assert result.cursors == {} + # The fence chain was read once for the shared scope, and no object listing began. + listings = fx.provider.list_calls + assert all("_collector-fence" in prefix for prefix in listings), listings + + +async def test_a_fenced_namespace_never_rests_a_whole_interval_in_the_maintenance_cycle( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + options = ReportingOrphanCollectionOptionsV1( + acknowledge_exclusive_namespace=True, interval_seconds=86_400 + ) + state = ReportingOrphanCollectionStateV1() + fx.write_fence(4) + first = await run_orphan_collection_pass(fx.store, options, state) + assert first.fenced == ("objects",) + assert state.next_cycle_at is not None + assert state.next_cycle_at - time.monotonic() <= 3_600 + 1, "looks again within the hour" + # The operator fixes it; the next cycle (the hour elapsing) resumes collection. + await fx.store.take_over_orphan_collector_fence(acknowledge_previous_owner_gone=True) + state.next_cycle_at = time.monotonic() - 1 + resumed = await run_orphan_collection_pass(fx.store, options, state) + assert resumed.fenced == () + assert state.next_cycle_at is not None + assert state.next_cycle_at - time.monotonic() > 3_600, "a clean cycle rests the interval" + + +def _later() -> Any: + from ._orphan_support import later + + return later() + + +def test_the_hook_provider_is_a_real_provider_shape() -> None: + assert HookedProvider.name == "filesystem" + + +async def test_a_listing_that_skipped_a_fence_entry_fails_closed_rather_than_reading_no_claim( + tmp_path: Path, +) -> None: + from adcp_reporting_storage.ledger.row_storage_object import ReportingRowObjectListPageV1 + + async with orphan_fixture(tmp_path) as fx: + await fx.put_object(fx.key("acct/orphan.jsonl")) + # The provider could not address an entry in the fence directory: it might be the newest + # claim, and an empty-looking chain would otherwise read as "never claimed". + fx.provider.fence_list_override = ReportingRowObjectListPageV1((), None, skipped=1) + result = await fx.collect(bindings=["objects"]) + assert fenced_cause(result) == "fence_unreadable" + assert result.deleted == 0 and fx.on_disk(fx.key("acct/orphan.jsonl")) + + +async def test_a_cancelled_claim_leaves_no_advisory_lock_on_a_pooled_connection( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + await fx.collect(bindings=["objects"]) + storage = fx.store._row_storage + assert storage is not None + scope, _, _ = await storage.collector_fence_location("objects") + holder = await fx.pool.getconn() + try: + await holder.execute( + "SELECT pg_advisory_lock(hashtextextended(%s, 0))", (f"adcp.orphan_fence:{scope}",) + ) + task = asyncio.create_task(fx.collect(bindings=["objects"])) + for _ in range(200): + rows = await fetch_all( + fx.pool, + "SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND NOT granted", + ) + if rows[0][0]: + break + await asyncio.sleep(0.025) + task.cancel() + (outcome,) = await asyncio.gather(task, return_exceptions=True) + assert isinstance(outcome, asyncio.CancelledError) + await holder.execute( + "SELECT pg_advisory_unlock(hashtextextended(%s, 0))", + (f"adcp.orphan_fence:{scope}",), + ) + await holder.commit() + finally: + await fx.pool.putconn(holder) + held = await fetch_all( + fx.pool, + "SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND granted" + " AND pid <> pg_backend_pid()", + ) + assert held == [(0,)], "no pooled connection kept a session lock" + again = await asyncio.wait_for(fx.collect(bindings=["objects"]), 20) + assert again.fenced == () + + +async def test_binding_name_arguments_must_be_sequences_of_ids_not_strings( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + for name in ("bindings", "skip_bindings"): + for bad in ("objects", "", b"objects", [1], ["objects", None]): + with pytest.raises(ValueError, match=name): + await fx.collect(dry_run=True, **{name: bad}) + with pytest.raises(ValueError, match="bindings"): + await fx.store.take_over_orphan_collector_fence( + acknowledge_previous_owner_gone=True, bindings="objects" # type: ignore[arg-type] + ) diff --git a/tests/conformance/reporting/test_reporting_row_binding_bounds_pg.py b/tests/conformance/reporting/test_reporting_row_binding_bounds_pg.py new file mode 100644 index 0000000000..e4738f2913 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_row_binding_bounds_pg.py @@ -0,0 +1,232 @@ +"""``row_binding_id`` is bounded at 255 characters in the database too (adcp-client-python #1498). + +``tests/fixtures/reporting-row-storage-pre-binding-bounds.sql`` is the row-storage DDL as released +before the bound moved from 128 to 255 (an unnamed ``CHECK`` PostgreSQL named after the column). +Re-applying the current DDL drops it and adds ``adcp_reporting_row_bindings_row_binding_id_bounds``, +the constraint a fresh install carries inline and the JavaScript SDK names the same. Ported from +adcp-client #3595. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +from adcp_reporting_storage import PgReportingLedgerStore, ReportingRowStorageOptionsV1 +from adcp_reporting_storage.ledger.row_storage import ( + REPORTING_ROW_STORAGE_DDL_PATH, + REPORTING_ROW_STORAGE_TABLES, + ReportingObjectRowBindingV1, + ReportingPostgresRowBindingV1, +) +from adcp_reporting_storage.ledger.row_storage_object import ( + create_filesystem_reporting_row_object_provider_v1, +) +from adcp_reporting_storage.outbox._schema import _catalog_objects + +from adcp.reporting.ledger.row_storage_errors import ReportingRowStoreError + +from ._generation_support import isolated_reporting_pool +from .test_reporting_row_storage_pg import fetch_all, many_rows, seeded, typed_revision, walk + +OLD_DDL = ( + Path(__file__).resolve().parents[2] + / "fixtures" + / "reporting-row-storage-pre-binding-bounds.sql" +) +CONSTRAINT = "adcp_reporting_row_bindings_row_binding_id_bounds" +LEGACY_CONSTRAINT = "adcp_reporting_row_bindings_row_binding_id_check" +SHA = "a" * 64 + + +async def insert_binding(pool: Any, binding_id: str) -> None: + async with pool.connection() as connection, connection.transaction(): + await connection.execute( + "INSERT INTO adcp_reporting_row_bindings (row_binding_id, kind, provider," + " identity_config, identity_sha256, namespace_key)" + " VALUES (%s, 'postgres', 'postgres', '{}', %s, %s)", + (binding_id, SHA, "b" * 64), + ) + + +async def binding_constraints(pool: Any) -> list[tuple[Any, ...]]: + return await fetch_all( + pool, + "SELECT conname, oid::text, pg_get_constraintdef(oid) FROM pg_constraint" + " WHERE conrelid = 'adcp_reporting_row_bindings'::regclass AND contype = 'c'" + " AND pg_get_constraintdef(oid) LIKE '%%row_binding_id%%' ORDER BY conname", + ) + + +async def catalog(pool: Any) -> dict[str, Any]: + async with pool.connection() as connection: + return await _catalog_objects( + connection, + table_names=REPORTING_ROW_STORAGE_TABLES, + function_identity=("adcp_reporting_row_storage_immutable", ""), + name_prefix="adcp_", + ) + + +async def apply(pool: Any, ddl: str) -> None: + async with pool.connection() as connection, connection.transaction(): + await connection.execute(ddl) + + +async def test_a_fresh_install_accepts_255_and_refuses_the_rest_under_one_named_constraint() -> ( + None +): + psycopg = pytest.importorskip("psycopg") + async with isolated_reporting_pool() as pool: + await apply(pool, REPORTING_ROW_STORAGE_DDL_PATH.read_text()) + assert [row[0] for row in await binding_constraints(pool)] == [CONSTRAINT] + assert "{1,255}" in (await binding_constraints(pool))[0][2] + await insert_binding(pool, "a" * 255) + await insert_binding(pool, "a" * 128) + for bad in ("a" * 256, "", "has space", "trailing\n", "аcyrillic", "x" * 5000): + with pytest.raises(psycopg.errors.CheckViolation): + await insert_binding(pool, bad) + + +async def test_a_database_with_the_original_128_limit_is_widened_in_place_and_idempotently() -> ( + None +): + psycopg = pytest.importorskip("psycopg") + current = REPORTING_ROW_STORAGE_DDL_PATH.read_text() + async with isolated_reporting_pool() as old, isolated_reporting_pool() as fresh: + await apply(old, OLD_DDL.read_text()) + assert [row[0] for row in await binding_constraints(old)] == [LEGACY_CONSTRAINT] + await insert_binding(old, "k" * 128) + with pytest.raises(psycopg.errors.CheckViolation): + await insert_binding(old, "k" * 129) + + await apply(old, current) + upgraded = await binding_constraints(old) + assert [row[0] for row in upgraded] == [CONSTRAINT], "the original CHECK is gone" + await insert_binding(old, "k" * 255) + with pytest.raises(psycopg.errors.CheckViolation): + await insert_binding(old, "k" * 256) + assert await fetch_all(old, "SELECT count(*) FROM adcp_reporting_row_bindings") == [ + (2,) + ], "no recorded binding was touched" + + # Re-applying changes nothing: the same constraint object survives. + await apply(old, current) + await apply(old, current) + assert await binding_constraints(old) == upgraded + + # The upgraded database is indistinguishable from a fresh install. + await apply(fresh, current) + assert await catalog(old) == await catalog(fresh) + + +async def test_a_current_schema_is_not_rewritten_by_the_guard() -> None: + current = REPORTING_ROW_STORAGE_DDL_PATH.read_text() + async with isolated_reporting_pool() as pool: + await apply(pool, current) + before = await binding_constraints(pool) + await apply(pool, current) + assert await binding_constraints(pool) == before + + +async def test_the_guard_leaves_a_host_customised_constraint_alone() -> None: + """Only the ``{1,128}`` shape this SDK shipped is replaced; anything else is the host's.""" + psycopg = pytest.importorskip("psycopg") + current = REPORTING_ROW_STORAGE_DDL_PATH.read_text() + async with isolated_reporting_pool() as pool: + await apply(pool, OLD_DDL.read_text()) + async with pool.connection() as connection, connection.transaction(): + await connection.execute( + f"ALTER TABLE adcp_reporting_row_bindings DROP CONSTRAINT {LEGACY_CONSTRAINT}" + ) + await connection.execute( + "ALTER TABLE adcp_reporting_row_bindings ADD CONSTRAINT host_row_binding_id_64" + " CHECK (row_binding_id ~ '^[A-Za-z0-9_.:-]{1,64}$')" + ) + await apply(pool, current) + assert [row[0] for row in await binding_constraints(pool)] == [ + CONSTRAINT, + "host_row_binding_id_64", + ] + with pytest.raises(psycopg.errors.CheckViolation): + await insert_binding(pool, "h" * 65) + + +async def test_the_guard_does_nothing_before_the_bindings_table_exists() -> None: + async with isolated_reporting_pool() as pool: + ddl = REPORTING_ROW_STORAGE_DDL_PATH.read_text() + start = ddl.index("DO $binding_id_bound$") + block = ddl[start : ddl.index("$binding_id_bound$;", start) + len("$binding_id_bound$;")] + await apply(pool, block) + + +async def test_a_binding_registered_against_the_original_limit_fails_closed_until_the_ddl_runs( + tmp_path: Path, +) -> None: + """The startup catalog check names the missing bound; it never reaches a bare CHECK failure.""" + async with isolated_reporting_pool() as pool: + store = PgReportingLedgerStore( + pool=pool, + row_storage=ReportingRowStorageOptionsV1( + deployment_namespace="test", + bindings={"x" * 200: ReportingPostgresRowBindingV1()}, + ), + ) + await apply(pool, _ledger_ddl()) + await apply(pool, OLD_DDL.read_text()) + with pytest.raises(ReportingRowStoreError) as failure: + await store.ready_row_storage() + assert failure.value.code == "STATE_UNAVAILABLE" + assert CONSTRAINT in str(failure.value) + + await store.create_schema() # applies the current row-storage DDL in place + await store.ready_row_storage() + assert await fetch_all( + pool, "SELECT length(row_binding_id) FROM adcp_reporting_row_bindings" + ) == [(200,)] + + +def _ledger_ddl() -> str: + from adcp_reporting_storage.ledger.pg import _DDL_PATH + + return _DDL_PATH.read_text() + + +async def test_a_255_character_binding_id_and_credential_ref_write_and_read_back_a_revision( + tmp_path: Path, +) -> None: + binding_id = "r" * 255 + root = tmp_path / "rows" + provider = create_filesystem_reporting_row_object_provider_v1(roots={"rows": root}) + options = ReportingRowStorageOptionsV1( + deployment_namespace="test", + bindings={ + binding_id: ReportingObjectRowBindingV1( + provider="filesystem", + location={"root": "rows"}, + prefix="adcp-rows", + credential_ref="c" * 255, + ), + }, + providers={"filesystem": provider}, + ) + async with isolated_reporting_pool() as pool: + store = PgReportingLedgerStore(pool=pool, row_storage=options) + await store.create_schema() + obligation = await seeded(store) + async with pool.connection() as connection: + await connection.execute( + "UPDATE reporting_configurations SET lease_worker_id = 'worker-1'," + " lease_expires_at = clock_timestamp() + INTERVAL '10 minutes'" + ) + rows = many_rows(50) + revision = typed_revision(obligation, rows, suffix="long") + assert await store.stage_revision_rows(revision, rows) is True + await store.commit_revision(revision, rows) + + assert await fetch_all( + pool, "SELECT row_binding_id, operational_config FROM adcp_reporting_row_bindings" + ) == [(binding_id, {"compression": "gzip", "credentialRef": "c" * 255})] + walked, _ = await walk(store, obligation.account_id, revision.reporting_revision_id, 20) + assert walked == rows diff --git a/tests/conformance/reporting/test_reporting_row_object_lock_contract_pg.py b/tests/conformance/reporting/test_reporting_row_object_lock_contract_pg.py new file mode 100644 index 0000000000..e1d031fc2a --- /dev/null +++ b/tests/conformance/reporting/test_reporting_row_object_lock_contract_pg.py @@ -0,0 +1,293 @@ +"""The per-object advisory lock is kept only where nothing else orders the two sides. + +Ported from the JavaScript SDK (adcp-client #3606, ``reporting-row-shared-object-pg``). Retirement, +the intent sweep and a commit that adopts an object its own intent recorded take no object lock; the +orphan collector's delete and a commit that adopts an object no record names pair on it. +""" + +from __future__ import annotations + +import asyncio +import hashlib +from pathlib import Path +from typing import Any + +import pytest +from adcp_reporting_storage.ledger.row_storage import ( + ReportingRowChunkLocatorV1, + ReportingRowStoreError, + recorded_intent_objects, +) + +from ._orphan_support import Fixture, orphan_fixture +from .test_reporting_row_storage_pg import fetch_all + +BODY = b'{"a":1}\n' + + +async def lock_holder(fx: Fixture, binding: str, object_ref: str) -> Any: + """A connection holding the per-object lock of ``binding`` until it commits.""" + storage = fx.store._row_storage + assert storage is not None + holder = await fx.pool.getconn() + await holder.execute( + "SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", + (storage._object_lock_text(binding, object_ref),), + ) + return holder + + +async def release(fx: Fixture, holder: Any) -> None: + await holder.commit() + await fx.pool.putconn(holder) + + +def locator(object_ref: str, version: str) -> ReportingRowChunkLocatorV1: + return ReportingRowChunkLocatorV1( + object_ref, version, "gzip", hashlib.sha256(BODY).hexdigest(), len(BODY), created=False + ) + + +async def verify( + fx: Fixture, binding: str, loc: ReportingRowChunkLocatorV1, recorded: Any = () +) -> None: + storage = fx.store._row_storage + assert storage is not None + async with fx.pool.connection() as connection, connection.transaction(): + await storage.lock_and_verify_adopted(connection, binding, [loc], recorded) + + +async def blocks( + fx: Fixture, binding: str, loc: ReportingRowChunkLocatorV1, recorded: Any = () +) -> bool: + """Whether the adoption waits on the object lock (and then completes once it is released).""" + task = asyncio.create_task(verify(fx, binding, loc, recorded)) + done, _ = await asyncio.wait({task}, timeout=0.6) + waited = not done + if waited: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + return waited + + +async def idle_in_transaction(fx: Fixture) -> int: + rows = await fetch_all( + fx.pool, + "SELECT count(*) FROM pg_stat_activity WHERE state LIKE 'idle in transaction%%'" + " AND datname = current_database() AND pid <> pg_backend_pid()", + ) + return int(rows[0][0]) + + +async def test_a_delete_of_a_referenced_object_takes_no_lock_and_holds_no_connection_across_the_provider_delete( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + ref, version = await fx.commit_revision("lock_free_retire") + storage = fx.store._row_storage + assert storage is not None + holder = await lock_holder(fx, "objects", ref) # the lock is held elsewhere throughout + seen: list[int] = [] + + async def during_delete(object_ref: str) -> None: + seen.append(await idle_in_transaction(fx)) + + fx.provider.on_delete = during_delete + try: + # Retirement scopes the delete to its own obligation, so its own references do not count. + removed = await asyncio.wait_for( + storage.delete_unreferenced( + fx.pool, + "objects", + [(ref, version)], + obligation_id=fx.obligation.reporting_obligation_id, + ), + 10, + ) + assert removed == 1 and not fx.on_disk(ref) + # Only the lock holder is idle in a transaction: nothing of the deleter's is open. + assert seen == [1] + finally: + await release(fx, holder) + + +async def test_a_referenced_object_is_kept_and_a_pruning_row_set_does_not_count( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + ref, version = await fx.commit_revision("kept") + storage = fx.store._row_storage + assert storage is not None + assert await storage.delete_unreferenced(fx.pool, "objects", [(ref, version)]) == 0 + assert fx.on_disk(ref) + async with fx.pool.connection() as connection: + await connection.execute("UPDATE adcp_reporting_row_sets SET rows_state = 'pruning'") + # Each deleter commits ``pruning`` before its check, so for any two one of them deletes. + assert await storage.delete_unreferenced(fx.pool, "objects", [(ref, version)]) == 1 + assert not fx.on_disk(ref) + + +async def test_the_intent_sweep_takes_no_object_lock_while_the_lock_is_held_elsewhere( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + ref = fx.key("acct/swept.jsonl.gz") + version = await fx.put_object(ref) + await fx.add_intent("rrev_expired", ref, version) + async with fx.pool.connection() as connection: + await connection.execute( + "UPDATE adcp_reporting_row_write_intents" + " SET expires_at = clock_timestamp() - INTERVAL '1 hour'" + ) + holder = await lock_holder(fx, "objects", ref) + try: + swept = await asyncio.wait_for(fx.store.sweep_row_write_intents(), 15) + assert swept.swept == 1 and swept.objects_deleted == 1 and not fx.on_disk(ref) + finally: + await release(fx, holder) + + +async def test_an_adoption_of_an_object_the_intent_recorded_takes_no_lock(tmp_path: Path) -> None: + async with orphan_fixture(tmp_path) as fx: + ref = fx.key("acct/own.jsonl.gz") + version = await fx.put_object(ref, BODY) + holder = await lock_holder(fx, "objects", ref) + try: + recorded = {(ref, version)} + await asyncio.wait_for(verify(fx, "objects", locator(ref, version), recorded), 5) + finally: + await release(fx, holder) + + +async def test_the_pre_alignment_intent_keys_are_read_so_an_older_writers_object_takes_no_lock() -> ( + None +): + pairs = recorded_intent_objects( + [ + {"object_ref": "a", "native_version_ref": "1"}, + {"objectKey": "b", "nativeVersion": "2"}, + {"object_ref": "c", "nativeVersion": "3"}, # mixed spellings are tolerated + {"object_ref": "d"}, + {"unrelated": True}, + "not an object", + {"object_ref": 5, "native_version_ref": "9"}, + ] + ) + assert pairs == {("a", "1"), ("b", "2"), ("c", "3")} + assert recorded_intent_objects(None) == set() and recorded_intent_objects({}) == set() + + +async def test_an_adoption_takes_the_lock_for_a_different_version_or_an_object_the_intent_does_not_name( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + ref = fx.key("acct/adopted.jsonl.gz") + version = await fx.put_object(ref, BODY) + other = fx.key("acct/other.jsonl.gz") + holder = await lock_holder(fx, "objects", ref) + try: + # The intent names the key at another native version: that is not this object. + assert await blocks(fx, "objects", locator(ref, version), {(ref, "9:9:9:9")}) + # The intent names another object, or none. + assert await blocks(fx, "objects", locator(ref, version), {(other, version)}) + assert await blocks(fx, "objects", locator(ref, version)) + finally: + await release(fx, holder) + # Once the lock is free the adoption proves the object and succeeds. + await asyncio.wait_for(verify(fx, "objects", locator(ref, version)), 5) + + +async def test_an_orphan_delete_in_flight_blocks_an_adopter_of_the_unrecorded_object_which_then_fails_cleanly( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + ref = fx.key("acct/raced.jsonl.gz") + version = await fx.put_object(ref, BODY) + storage = fx.store._row_storage + assert storage is not None + deleting = asyncio.Event() + proceed = asyncio.Event() + + async def pause(object_ref: str) -> None: + deleting.set() + await proceed.wait() + + fx.provider.on_delete = pause + orphan = asyncio.create_task( + storage.delete_orphaned_object(fx.pool, "objects", ref, version) + ) + await asyncio.wait_for(deleting.wait(), 10) + adopter = asyncio.create_task(verify(fx, "objects", locator(ref, version))) + done, _ = await asyncio.wait({adopter}, timeout=0.6) + assert not done, "the adoption waits for the delete that holds the object lock" + proceed.set() + assert await asyncio.wait_for(orphan, 10) is True + with pytest.raises(ReportingRowStoreError) as failure: + await asyncio.wait_for(adopter, 10) + assert failure.value.code == "STATE_UNAVAILABLE", "the version-pinned stat fails it cleanly" + + +async def test_an_orphan_delete_takes_every_bindings_lock_for_the_object(tmp_path: Path) -> None: + async with orphan_fixture(tmp_path) as fx: + ref = fx.key("acct/shared.jsonl.gz") + version = await fx.put_object(ref, BODY) + storage = fx.store._row_storage + assert storage is not None + # An adopter through the twin binding (same bucket and prefix) holds only the twin's lock. + holder = await lock_holder(fx, "twin", ref) + task = asyncio.create_task(storage.delete_orphaned_object(fx.pool, "objects", ref, version)) + done, _ = await asyncio.wait({task}, timeout=0.6) + assert not done and fx.on_disk(ref) + await release(fx, holder) + assert await asyncio.wait_for(task, 10) is True and not fx.on_disk(ref) + + +async def test_an_orphan_delete_keeps_an_object_a_record_in_another_binding_names( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + ref = fx.key("acct/named.jsonl.gz") + version = await fx.put_object(ref, BODY) + await fx.add_intent("rrev_twin", ref, version, binding="twin") + storage = fx.store._row_storage + assert storage is not None + assert await storage.delete_orphaned_object(fx.pool, "objects", ref, version) is False + assert fx.on_disk(ref) + + +async def test_a_writer_whose_intent_expired_before_its_commit_cannot_commit_the_swept_objects( + tmp_path: Path, +) -> None: + async with orphan_fixture(tmp_path) as fx: + from .test_reporting_row_storage_pg import many_rows, seeded, typed_revision + + fx.obligation = await seeded(fx.store) + async with fx.pool.connection() as connection: + await connection.execute( + "UPDATE reporting_configurations SET lease_worker_id = 'worker-1'," + " lease_expires_at = clock_timestamp() + INTERVAL '10 minutes'" + ) + rows = many_rows(4) + revision = typed_revision(fx.obligation, rows, suffix="swept_writer") + assert await fx.store.stage_revision_rows(revision, rows) is True + async with fx.pool.connection() as connection: + await connection.execute( + "UPDATE adcp_reporting_row_write_intents" + " SET expires_at = clock_timestamp() - INTERVAL '1 hour'" + ) + swept = await fx.store.sweep_row_write_intents() + assert swept.swept == 1 and swept.objects_deleted == 1 + # Exactly one of the sweep claim and the commit's intent close wins; the writer lost. + with pytest.raises(ReportingRowStoreError) as failure: + await fx.store.commit_revision(revision, rows) + assert failure.value.code == "STATE_UNAVAILABLE" + assert await fetch_all(fx.pool, "SELECT count(*) FROM adcp_reporting_row_sets") == [(0,)] + + +def test_the_lock_text_is_unchanged_and_shared_with_the_javascript_sdk() -> None: + from adcp_reporting_storage.ledger.row_storage import ReportingRowStorageV1 + + assert ( + ReportingRowStorageV1._object_lock_text("reports", "adcp-rows/ns/a/b") + == "adcp-reporting-object:reports:adcp-rows/ns/a/b" + ) diff --git a/tests/fixtures/public_api_snapshot.json b/tests/fixtures/public_api_snapshot.json index 3b09a9cc99..820bea3a8c 100644 --- a/tests/fixtures/public_api_snapshot.json +++ b/tests/fixtures/public_api_snapshot.json @@ -9028,6 +9028,13 @@ "ReportingChangeRecordV1": "adcp_reporting_storage.ledger.change_feed", "ReportingChangesPageV1": "adcp_reporting_storage.ledger.change_feed", "ReportingObjectRowBindingV1": "adcp_reporting_storage.ledger.row_storage", + "ReportingOrphanBindingResultV1": "adcp_reporting_storage.ledger.orphan_collector", + "ReportingOrphanCandidateV1": "adcp_reporting_storage.ledger.orphan_collector", + "ReportingOrphanCollectionOptionsV1": "adcp_reporting_storage.ledger.orphan_collector", + "ReportingOrphanCollectionResultV1": "adcp_reporting_storage.ledger.orphan_collector", + "ReportingOrphanCollectionStateV1": "adcp_reporting_storage.ledger.orphan_collector", + "ReportingOrphanFenceRefusalV1": "adcp_reporting_storage.ledger.orphan_fence", + "ReportingOrphanFenceTakeoverV1": "adcp_reporting_storage.ledger.orphan_collector", "ReportingOwnershipBackfill": "adcp_reporting_storage.migration", "ReportingOwnershipMigrationError": "adcp_reporting_storage.migration", "ReportingPeriodRetirementV1": "adcp_reporting_storage.ledger.period_retirement", diff --git a/tests/fixtures/reporting-row-storage-pre-binding-bounds.sql b/tests/fixtures/reporting-row-storage-pre-binding-bounds.sql new file mode 100644 index 0000000000..0862039adb --- /dev/null +++ b/tests/fixtures/reporting-row-storage-pre-binding-bounds.sql @@ -0,0 +1,334 @@ +-- AdCP Reliable Reporting row storage: bindings, row sets, chunk manifests, +-- PostgreSQL chunk bodies and write intents. +-- +-- These are the shared SDK persistence tables (adcontextprotocol/adcp#7996, +-- section 3.2, reporting-row-storage v1). The JavaScript SDK ships the same +-- tables and columns, so one database can hold chunks written by either SDK: +-- every reader verifies stored bytes against digests committed here. +-- +-- Apply after reporting_ledger.sql (PgReportingLedgerStore.create_schema does +-- this when row storage is enabled). PostgreSQL 13 or later (gen_random_uuid, +-- sha256). Idempotent; reapplying preserves the installation identity and every +-- recorded location. +-- +-- One deliberate difference from the JavaScript DDL: adcp_reporting_row_sets +-- .obligation_id has no foreign key. Until the shared Core layout (persistence +-- Stage 3), Python obligations live in reporting_obligations, not in the +-- JavaScript adcp_reporting_obligations table, so a foreign key to either would +-- reject the other SDK's row sets. The (row_set_id, account_id, obligation_id) +-- composite key still binds every chunk and every shared row set to its own +-- account and obligation. +-- +-- Stored values use AdCP's names and bounds (reporting-file-manifest.json, +-- reporting-file-entry.json): object_ref and native_version_ref are 1 to 1024 +-- characters, size_bytes is the stored object's size, and the row-set format and +-- algorithm are jsonl and adcp_jcs_rows_v1. A database created before this +-- alignment (object_key, native_version, byte_count, physical_byte_count, +-- canonical_byte_count, encoding) is renamed in place by the first block below, +-- which also rewrites open write intents. Recorded object keys are not rewritten. +-- +-- The adcp_ prefix keeps these objects outside the reporting_* catalog contract +-- older binaries verify, so adding them cannot change any retained manifest. + +SELECT pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); +-- The JavaScript SDK serializes its appliers on this key; taking it too serializes the two SDKs' +-- migrations of these shared tables against each other. +SELECT pg_advisory_xact_lock(hashtextextended('adcp-reporting-row-storage-migration', 0)); + +-- In-place upgrade from the pre-AdCP-alignment names. Every step is guarded, so a fresh install +-- (no tables yet) and a reapplied migration are both no-ops here. +DO $align$ +DECLARE + renames TEXT[][] := ARRAY[ + ARRAY['adcp_reporting_row_chunks', 'object_key', 'object_ref'], + ARRAY['adcp_reporting_row_chunks', 'native_version', 'native_version_ref'], + ARRAY['adcp_reporting_row_chunks', 'byte_count', 'canonical_size_bytes'], + ARRAY['adcp_reporting_row_chunks', 'physical_byte_count', 'size_bytes'], + ARRAY['adcp_reporting_row_sets', 'canonical_byte_count', 'content_size_bytes'] + ]; + -- Column CHECK constraints are named after their column when created; keep an upgraded database + -- identical to a fresh one, which the startup catalog check compares by name. + constraint_renames TEXT[][] := ARRAY[ + ARRAY['adcp_reporting_row_chunks', 'adcp_reporting_row_chunks_byte_count_check', + 'adcp_reporting_row_chunks_canonical_size_bytes_check'], + ARRAY['adcp_reporting_row_chunks', 'adcp_reporting_row_chunks_physical_byte_count_check', + 'adcp_reporting_row_chunks_size_bytes_check'], + ARRAY['adcp_reporting_row_sets', 'adcp_reporting_row_sets_canonical_byte_count_check', + 'adcp_reporting_row_sets_content_size_bytes_check'] + ]; + item TEXT[]; +BEGIN + FOREACH item SLICE 1 IN ARRAY renames LOOP + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = item[1] AND column_name = item[2] + ) AND NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = item[1] AND column_name = item[3] + ) THEN + EXECUTE format('ALTER TABLE %I RENAME COLUMN %I TO %I', item[1], item[2], item[3]); + END IF; + END LOOP; + FOREACH item SLICE 1 IN ARRAY constraint_renames LOOP + IF EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = item[2] AND conrelid = to_regclass(quote_ident(current_schema()) || '.' || quote_ident(item[1])) + ) AND NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = item[3] AND conrelid = to_regclass(quote_ident(current_schema()) || '.' || quote_ident(item[1])) + ) THEN + EXECUTE format('ALTER TABLE %I RENAME CONSTRAINT %I TO %I', item[1], item[2], item[3]); + END IF; + END LOOP; + -- encoding = 'adcp_canonical_jsonl_v1' becomes AdCP's format = 'jsonl' plus + -- algorithm = 'adcp_jcs_rows_v1'. The old CHECK admitted one value, so the defaults backfill + -- every row without an UPDATE (the immutability trigger forbids one). + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = 'adcp_reporting_row_sets' AND column_name = 'encoding' + ) AND NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = 'adcp_reporting_row_sets' AND column_name = 'format' + ) THEN + ALTER TABLE adcp_reporting_row_sets + ADD COLUMN format TEXT NOT NULL DEFAULT 'jsonl' CHECK (format = 'jsonl'), + ADD COLUMN algorithm TEXT NOT NULL DEFAULT 'adcp_jcs_rows_v1' CHECK (algorithm = 'adcp_jcs_rows_v1'); + ALTER TABLE adcp_reporting_row_sets ALTER COLUMN format DROP DEFAULT, ALTER COLUMN algorithm DROP DEFAULT; + ALTER TABLE adcp_reporting_row_sets DROP COLUMN encoding; + END IF; +END +$align$; + +CREATE TABLE IF NOT EXISTS adcp_persistence_installation ( + singleton BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (singleton), + installation_id UUID NOT NULL DEFAULT gen_random_uuid(), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); +INSERT INTO adcp_persistence_installation (singleton) VALUES (TRUE) ON CONFLICT DO NOTHING; + +CREATE TABLE IF NOT EXISTS adcp_reporting_row_bindings ( + row_binding_id TEXT COLLATE "C" PRIMARY KEY CHECK (row_binding_id ~ '^[A-Za-z0-9_.:-]{1,128}$'), + kind TEXT NOT NULL CHECK (kind IN ('postgres', 'object')), + provider TEXT NOT NULL CHECK (provider ~ '^[a-z][a-z0-9_]{0,63}$'), + identity_config JSONB NOT NULL CHECK (jsonb_typeof(identity_config) = 'object'), + identity_sha256 TEXT COLLATE "C" NOT NULL CHECK (identity_sha256 ~ '^[0-9a-f]{64}$'), + operational_config JSONB NOT NULL DEFAULT '{}'::jsonb CHECK (jsonb_typeof(operational_config) = 'object'), + namespace_key TEXT COLLATE "C" NOT NULL CHECK (namespace_key ~ '^[0-9a-f]{64}$'), + state TEXT NOT NULL DEFAULT 'active' CHECK (state IN ('active', 'read_only', 'retired')), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + retired_at TIMESTAMPTZ, + CHECK ((state = 'retired') = (retired_at IS NOT NULL)) +); + +CREATE TABLE IF NOT EXISTS adcp_reporting_row_sets ( + row_set_id TEXT COLLATE "C" PRIMARY KEY, + row_set_kind TEXT NOT NULL CHECK (row_set_kind IN ('revision', 'adjustment')), + account_id TEXT COLLATE "C" NOT NULL, + obligation_id TEXT COLLATE "C" NOT NULL, + format TEXT NOT NULL CHECK (format = 'jsonl'), + algorithm TEXT NOT NULL CHECK (algorithm = 'adcp_jcs_rows_v1'), + digest_profile TEXT NOT NULL CHECK (digest_profile IN ('revision_envelope_v1', 'rows_v1')), + content_sha256 TEXT COLLATE "C" NOT NULL CHECK (content_sha256 ~ '^[0-9a-f]{64}$'), + content_size_bytes BIGINT NOT NULL CHECK (content_size_bytes > 0), + row_count BIGINT NOT NULL CHECK (row_count >= 0), + row_manifest_sha256 TEXT COLLATE "C" NOT NULL CHECK (row_manifest_sha256 ~ '^[0-9a-f]{64}$'), + chunk_count INTEGER NOT NULL CHECK (chunk_count >= 0), + rows_shared_from_row_set_id TEXT COLLATE "C", + row_binding_id TEXT COLLATE "C" NOT NULL REFERENCES adcp_reporting_row_bindings(row_binding_id), + row_location_version INTEGER NOT NULL DEFAULT 1 CHECK (row_location_version > 0), + rows_state TEXT NOT NULL DEFAULT 'live' CHECK (rows_state IN ('live', 'pruning', 'pruned', 'unavailable')), + rows_state_changed_at TIMESTAMPTZ, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CHECK ((row_count = 0) = (chunk_count = 0)), + CHECK (rows_shared_from_row_set_id IS NULL OR rows_shared_from_row_set_id <> row_set_id), + -- Tenant columns are the composite key children reference, so a chunk or a + -- shared row set can never name another account's or obligation's rows. + UNIQUE (row_set_id, account_id, obligation_id), + FOREIGN KEY (rows_shared_from_row_set_id, account_id, obligation_id) + REFERENCES adcp_reporting_row_sets (row_set_id, account_id, obligation_id) +); +CREATE INDEX IF NOT EXISTS adcp_reporting_row_sets_obligation + ON adcp_reporting_row_sets (obligation_id, recorded_at, row_set_id); +CREATE INDEX IF NOT EXISTS adcp_reporting_row_sets_state + ON adcp_reporting_row_sets (account_id, rows_state, recorded_at) + WHERE rows_state <> 'pruned'; +CREATE INDEX IF NOT EXISTS adcp_reporting_row_sets_shared + ON adcp_reporting_row_sets (rows_shared_from_row_set_id) + WHERE rows_shared_from_row_set_id IS NOT NULL; + +CREATE TABLE IF NOT EXISTS adcp_reporting_row_chunks ( + row_set_id TEXT COLLATE "C" NOT NULL, + chunk_index INTEGER NOT NULL CHECK (chunk_index >= 0), + account_id TEXT COLLATE "C" NOT NULL, + obligation_id TEXT COLLATE "C" NOT NULL, + first_ordinal BIGINT NOT NULL CHECK (first_ordinal >= 0), + row_count INTEGER NOT NULL CHECK (row_count > 0), + canonical_size_bytes BIGINT NOT NULL CHECK (canonical_size_bytes > 0), + sha256 TEXT COLLATE "C" NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + segments JSONB NOT NULL CHECK (jsonb_typeof(segments) = 'array'), + object_ref TEXT COLLATE "C" CONSTRAINT adcp_reporting_row_chunks_object_ref_bounds + CHECK (object_ref IS NULL OR char_length(object_ref) BETWEEN 1 AND 1024), + native_version_ref TEXT CONSTRAINT adcp_reporting_row_chunks_native_version_ref_bounds + CHECK (native_version_ref IS NULL OR char_length(native_version_ref) BETWEEN 1 AND 1024), + compression TEXT NOT NULL DEFAULT 'none' CHECK (compression IN ('none', 'gzip')), + physical_sha256 TEXT COLLATE "C" CHECK (physical_sha256 IS NULL OR physical_sha256 ~ '^[0-9a-f]{64}$'), + size_bytes BIGINT CHECK (size_bytes IS NULL OR size_bytes > 0), + PRIMARY KEY (row_set_id, chunk_index), + FOREIGN KEY (row_set_id, account_id, obligation_id) + REFERENCES adcp_reporting_row_sets (row_set_id, account_id, obligation_id), + CHECK ((object_ref IS NULL) = (native_version_ref IS NULL)), + CHECK ((object_ref IS NULL) = (physical_sha256 IS NULL)), + CHECK ((object_ref IS NULL) = (size_bytes IS NULL)) +); +CREATE INDEX IF NOT EXISTS adcp_reporting_row_chunks_object + ON adcp_reporting_row_chunks (object_ref, native_version_ref) + WHERE object_ref IS NOT NULL; +CREATE INDEX IF NOT EXISTS adcp_reporting_row_chunks_body + ON adcp_reporting_row_chunks (account_id, obligation_id, sha256) + WHERE object_ref IS NULL; + +CREATE TABLE IF NOT EXISTS adcp_reporting_chunk_bodies ( + account_id TEXT COLLATE "C" NOT NULL, + obligation_id TEXT COLLATE "C" NOT NULL, + sha256 TEXT COLLATE "C" NOT NULL, + body BYTEA NOT NULL, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (account_id, obligation_id, sha256), + CHECK (sha256 = encode(sha256(body), 'hex')) +); + +CREATE TABLE IF NOT EXISTS adcp_reporting_row_write_intents ( + row_binding_id TEXT COLLATE "C" NOT NULL REFERENCES adcp_reporting_row_bindings(row_binding_id), + account_id TEXT COLLATE "C" NOT NULL, + row_set_id TEXT COLLATE "C" NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL CHECK (content_sha256 ~ '^[0-9a-f]{64}$'), + state TEXT NOT NULL DEFAULT 'open' CHECK (state IN ('open', 'sweeping')), + owner_token TEXT COLLATE "C" NOT NULL, + created_objects JSONB NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(created_objects) = 'array'), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + expires_at TIMESTAMPTZ NOT NULL, + sweep_attempts INTEGER NOT NULL DEFAULT 0, + next_sweep_at TIMESTAMPTZ, + PRIMARY KEY (row_binding_id, row_set_id, content_sha256) +); +CREATE INDEX IF NOT EXISTS adcp_reporting_row_write_intents_expiry + ON adcp_reporting_row_write_intents (expires_at, account_id) + WHERE state = 'open'; + +CREATE OR REPLACE FUNCTION adcp_reporting_row_storage_immutable() RETURNS trigger +LANGUAGE plpgsql AS $immutable$ +BEGIN + IF TG_TABLE_NAME = 'adcp_reporting_row_bindings' THEN + IF (NEW.row_binding_id, NEW.kind, NEW.provider, NEW.identity_config, NEW.identity_sha256, NEW.namespace_key) + IS DISTINCT FROM + (OLD.row_binding_id, OLD.kind, OLD.provider, OLD.identity_config, OLD.identity_sha256, OLD.namespace_key) THEN + RAISE EXCEPTION 'reporting row binding identity is immutable' USING ERRCODE = '23514'; + END IF; + ELSIF TG_TABLE_NAME = 'adcp_reporting_row_sets' THEN + IF (NEW.row_set_id, NEW.row_set_kind, NEW.account_id, NEW.obligation_id, NEW.format, NEW.algorithm, + NEW.digest_profile, NEW.content_sha256, NEW.content_size_bytes, NEW.row_count, NEW.row_manifest_sha256, + NEW.chunk_count, NEW.rows_shared_from_row_set_id, NEW.recorded_at) + IS DISTINCT FROM + (OLD.row_set_id, OLD.row_set_kind, OLD.account_id, OLD.obligation_id, OLD.format, OLD.algorithm, + OLD.digest_profile, OLD.content_sha256, OLD.content_size_bytes, OLD.row_count, OLD.row_manifest_sha256, + OLD.chunk_count, OLD.rows_shared_from_row_set_id, OLD.recorded_at) THEN + RAISE EXCEPTION 'reporting row set content is immutable' USING ERRCODE = '23514'; + END IF; + IF NEW.row_binding_id IS DISTINCT FROM OLD.row_binding_id + AND NEW.row_location_version <> OLD.row_location_version + 1 THEN + RAISE EXCEPTION 'reporting row set moves must advance row_location_version' USING ERRCODE = '23514'; + END IF; + ELSIF TG_TABLE_NAME = 'adcp_reporting_row_chunks' THEN + IF (NEW.row_set_id, NEW.chunk_index, NEW.account_id, NEW.obligation_id, NEW.first_ordinal, NEW.row_count, + NEW.canonical_size_bytes, NEW.sha256, NEW.segments) + IS DISTINCT FROM + (OLD.row_set_id, OLD.chunk_index, OLD.account_id, OLD.obligation_id, OLD.first_ordinal, OLD.row_count, + OLD.canonical_size_bytes, OLD.sha256, OLD.segments) THEN + RAISE EXCEPTION 'reporting row chunk content is immutable' USING ERRCODE = '23514'; + END IF; + ELSIF TG_TABLE_NAME = 'adcp_reporting_chunk_bodies' THEN + RAISE EXCEPTION 'reporting chunk bodies are immutable' USING ERRCODE = '23514'; + END IF; + RETURN NEW; +END +$immutable$; + +DO $triggers$ +DECLARE + target TEXT; +BEGIN + FOREACH target IN ARRAY ARRAY[ + 'adcp_reporting_row_bindings', + 'adcp_reporting_row_sets', + 'adcp_reporting_row_chunks', + 'adcp_reporting_chunk_bodies' + ] LOOP + IF NOT EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgname = target || '_immutable' + AND tgrelid = to_regclass(quote_ident(current_schema()) || '.' || quote_ident(target)) + ) THEN + EXECUTE format( + 'CREATE TRIGGER %I BEFORE UPDATE ON %I FOR EACH ROW EXECUTE FUNCTION adcp_reporting_row_storage_immutable()', + target || '_immutable', + target + ); + END IF; + END LOOP; +END +$triggers$; + +-- Per-intent sweep backoff (1 minute doubling to 6 hours), so one failing intent cannot hold the +-- head of every pass. The same columns the JavaScript SDK adds, for installs created before. +ALTER TABLE adcp_reporting_row_write_intents + ADD COLUMN IF NOT EXISTS sweep_attempts INTEGER NOT NULL DEFAULT 0; +ALTER TABLE adcp_reporting_row_write_intents ADD COLUMN IF NOT EXISTS next_sweep_at TIMESTAMPTZ; + +-- AdCP bounds on installs created before they existed (a fresh install has them inline). +DO $bounds$ +DECLARE + offending BIGINT; +BEGIN + SELECT count(*) INTO offending FROM adcp_reporting_row_chunks + WHERE object_ref IS NOT NULL + AND (char_length(object_ref) NOT BETWEEN 1 AND 1024 + OR char_length(native_version_ref) NOT BETWEEN 1 AND 1024); + IF offending > 0 THEN + RAISE EXCEPTION 'reporting row storage: % chunk(s) hold an object_ref or native_version_ref outside AdCP 1 to 1024 characters; re-sync them before applying this migration', offending + USING ERRCODE = '23514'; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'adcp_reporting_row_chunks_object_ref_bounds' + AND conrelid = to_regclass(quote_ident(current_schema()) || '.adcp_reporting_row_chunks') + ) THEN + ALTER TABLE adcp_reporting_row_chunks ADD CONSTRAINT adcp_reporting_row_chunks_object_ref_bounds + CHECK (object_ref IS NULL OR char_length(object_ref) BETWEEN 1 AND 1024); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'adcp_reporting_row_chunks_native_version_ref_bounds' + AND conrelid = to_regclass(quote_ident(current_schema()) || '.adcp_reporting_row_chunks') + ) THEN + ALTER TABLE adcp_reporting_row_chunks ADD CONSTRAINT adcp_reporting_row_chunks_native_version_ref_bounds + CHECK (native_version_ref IS NULL OR char_length(native_version_ref) BETWEEN 1 AND 1024); + END IF; +END +$bounds$; + +-- Open write intents written before the rename name objects objectKey / nativeVersion. +UPDATE adcp_reporting_row_write_intents + SET created_objects = ( + SELECT COALESCE(jsonb_agg( + CASE WHEN jsonb_typeof(created) = 'object' AND created ? 'objectKey' + THEN jsonb_build_object('object_ref', created -> 'objectKey', + 'native_version_ref', created -> 'nativeVersion') + ELSE created END + ORDER BY ordinality + ), '[]'::jsonb) + FROM jsonb_array_elements(created_objects) WITH ORDINALITY AS element(created, ordinality) + ) + WHERE EXISTS ( + SELECT 1 FROM jsonb_array_elements(created_objects) AS element(created) + WHERE jsonb_typeof(created) = 'object' AND created ? 'objectKey' + ); diff --git a/tests/test_main_release_policies.py b/tests/test_main_release_policies.py index 59b3b9c8c3..235ba8c44a 100644 --- a/tests/test_main_release_policies.py +++ b/tests/test_main_release_policies.py @@ -19,6 +19,8 @@ LEDGER_SHA = "d" * 40 BREAKING_SUBJECT = "fix(reporting)!: scope configuration generations by account (#1174)" +# A PR title carries no PR number; the squash subject appends it. +BREAKING_PR_TITLE = BREAKING_SUBJECT.removesuffix(" (#1174)") BREAKING_FOOTER = ( "BREAKING CHANGE: generation_key returns ReportingConfigurationGenerationKey " "instead of a two-tuple. Use its named fields." @@ -107,7 +109,7 @@ def test_conventional_rerun_reads_the_corrected_pr_body(history: list[str]) -> N "number": 1174, "head": {"sha": history[-1]}, "base": {"sha": history[0]}, - "title": BREAKING_SUBJECT, + "title": BREAKING_PR_TITLE, "body": "Missing the breaking footer in the original webhook.", } api.data["pulls/1174"] = {**pr, "body": BREAKING_FOOTER} @@ -124,7 +126,7 @@ def test_conventional_rerun_rejects_a_newly_invalid_pr_message(history: list[str "number": 1174, "head": {"sha": history[-1]}, "base": {"sha": history[0]}, - "title": BREAKING_SUBJECT, + "title": BREAKING_PR_TITLE, "body": BREAKING_FOOTER, } api.data["pulls/1174"] = {**pr, "body": None} @@ -133,6 +135,67 @@ def test_conventional_rerun_rejects_a_newly_invalid_pr_message(history: list[str policies.conventional("pull_request", history[-1], {"pull_request": pr}, api=api) +def test_conventional_pr_title_keeps_a_trailing_issue_reference(history: list[str]) -> None: + # A PR title has no PR number yet; the squash appends one. A title ending + # in "(#1780)" would integrate as "... (#1780) (#1810)", which main rejects. + api = PolicyAPI(history) + pr = { + "number": 1810, + "head": {"sha": history[-1]}, + "base": {"sha": history[0]}, + "title": "feat(decisioning): reject a request with every error (#1780)", + "body": "", + } + api.data["pulls/1810"] = pr + + with pytest.raises(ReleaseRejectedError, match="unsafe release description"): + policies.conventional("pull_request", history[-1], {"pull_request": pr}, api=api) + + +@pytest.mark.parametrize( + ("message", "integrated", "accepted"), + [ + ("fix: handle the empty case (#1810)", True, True), + ("fix: handle the empty case (#1810)", False, False), + ("fix: handle the empty case (#1780) (#1810)", True, False), + ("fix: handle the empty case", False, True), + ], +) +def test_pr_number_is_stripped_only_from_integrated_subjects( + message: str, integrated: bool, accepted: bool +) -> None: + if accepted: + policies.validate_message(message, integrated=integrated) + else: + with pytest.raises(ReleaseRejectedError, match="unsafe release description"): + policies.validate_message(message, integrated=integrated) + + +HISTORICAL_DESCRIPTION_SHA = "272fced52d57632dedf32f2c175cf294100bff63" +HISTORICAL_DESCRIPTION_SUBJECT = ( + "feat(decisioning): let a raised AdcpError reject a request with every error (#1780) (#1810)" +) + + +@pytest.mark.parametrize( + ("commit_sha", "subject", "accepted"), + [ + (HISTORICAL_DESCRIPTION_SHA, HISTORICAL_DESCRIPTION_SUBJECT, True), + ("a" * 40, HISTORICAL_DESCRIPTION_SUBJECT, False), + (None, HISTORICAL_DESCRIPTION_SUBJECT, False), + (HISTORICAL_DESCRIPTION_SHA, "fix: bad (description) (#1810)", False), + ], +) +def test_historical_description_exception_requires_exact_commit_and_subject( + commit_sha: str | None, subject: str, accepted: bool +) -> None: + if accepted: + policies.validate_message(subject, commit_sha=commit_sha) + else: + with pytest.raises(ReleaseRejectedError, match="unsafe release description"): + policies.validate_message(subject, commit_sha=commit_sha) + + def test_conventional_rerun_cannot_validate_a_different_pr_head(history: list[str]) -> None: api = PolicyAPI(history) pr = { diff --git a/tests/test_release_configuration.py b/tests/test_release_configuration.py index 9a6eaf4ade..a1de38de80 100644 --- a/tests/test_release_configuration.py +++ b/tests/test_release_configuration.py @@ -122,9 +122,22 @@ def test_release_please_keeps_next_on_the_beta_channel() -> None: assert "release-publish.yml --ref next -f package=adcp-reporting-storage" in workflow manifest = json.loads((ROOT / ".release-please-manifest.next.json").read_text()) assert set(manifest) == {".", "packages/adcp-reporting-storage"} - # The companion gate (#1605, #1763) runs for every publish of either - # package, on next too, after the build: the wheel just built is installed - # beside the published companion and smoked, with PostgreSQL available. + # next's publish gate also accepts the storage component's tags. + publisher = yaml.load( + (ROOT / ".github/workflows/release-publish.yml").read_text(), Loader=yaml.BaseLoader + ) + gate = publisher["jobs"]["build"]["steps"][1]["run"] + assert 'adcp-reporting-storage) [[ "$RELEASE_TAG" =~ ^adcp-reporting-storage-v[0-9] ]]' in gate + + +def test_release_publish_gates_both_packages_on_a_working_companion() -> None: + """The companion gate (#1605, #1763) runs for every publish of either package. + + It runs after the build, so the wheel just built is installed beside the + published companion and smoked, with a PostgreSQL service available: a + pair whose version ranges admit each other but whose storage contracts + differ never reaches PyPI. + """ publisher = yaml.load( (ROOT / ".github/workflows/release-publish.yml").read_text(), Loader=yaml.BaseLoader ) @@ -145,8 +158,6 @@ def test_release_please_keeps_next_on_the_beta_channel() -> None: ] assert build_steps and build_steps[0] < companion_steps[0], "gate the built wheel" assert publisher["jobs"]["build"]["services"]["postgres"]["image"].startswith("postgres:") - gate = steps[1]["run"] - assert 'adcp-reporting-storage) [[ "$RELEASE_TAG" =~ ^adcp-reporting-storage-v[0-9] ]]' in gate def test_storage_distribution_pins_one_adcp_major() -> None: diff --git a/tests/test_reporting_orphan_fence.py b/tests/test_reporting_orphan_fence.py new file mode 100644 index 0000000000..6f818b456d --- /dev/null +++ b/tests/test_reporting_orphan_fence.py @@ -0,0 +1,207 @@ +"""The orphan collector's fence decision table, formats and scope, without a database. + +Ported from the JavaScript SDK (adcp-client #3593, ``reporting-orphan-fence``). +""" + +from __future__ import annotations + +import json +import uuid + +import pytest +from adcp_reporting_storage.ledger.orphan_fence import ( + ORPHAN_FENCE_MAX_BYTES, + OrphanFenceRecord, + OrphanFenceRow, + decide_orphan_fence, + encode_orphan_fence_record, + is_orphan_fence_key, + orphan_fence_directory, + orphan_fence_epoch_of_key, + orphan_fence_key, + orphan_fence_scope, + parse_orphan_fence_record, +) + +NAMESPACE = "a" * 64 +OWNED = f"orph/{NAMESPACE}/" +T1, T2, T3 = (str(uuid.uuid4()) for _ in range(3)) + + +def row(epoch: int, token: str = T1, previous: str | None = None) -> OrphanFenceRow: + return OrphanFenceRow(epoch, token, previous) + + +def test_it_claims_epoch_one_when_neither_side_has_a_claim_and_the_next_when_they_match() -> None: + first = decide_orphan_fence(None, None) + assert (first.action, first.epoch, first.previous_token) == ("claim", 1, None) + nxt = decide_orphan_fence(row(4, T1), (4, T1)) + assert (nxt.action, nxt.epoch, nxt.previous_token) == ("claim", 5, T1) + + +def test_it_refuses_a_namespace_claimed_by_a_database_it_has_no_record_of() -> None: + decision = decide_orphan_fence(None, (3, T1)) + assert decision.action == "fenced" and decision.refusal is not None + assert decision.refusal.cause == "claimed_elsewhere" + assert (decision.refusal.fence_epoch, decision.refusal.recorded_epoch) == (3, None) + assert "take_over_orphan_collector_fence" in decision.refusal.detail + + +def test_it_finishes_its_own_unfinished_claim() -> None: + # The object is missing: the first claim was recorded and never written. + assert decide_orphan_fence(row(1, T1, None), None).action == "complete" + # One epoch behind with the recorded previous token: the intent committed, the write did not. + assert decide_orphan_fence(row(5, T2, T1), (4, T1)).action == "complete" + + +@pytest.mark.parametrize( + ("recorded", "latest", "cause"), + [ + (row(3, T1, T2), None, "fence_missing"), + (row(1, T1, T2), None, "fence_missing"), + (row(4, T1), (4, T2), "token_mismatch"), + (row(4, T1), (6, T1), "fence_advanced"), + (row(4, T1), (2, T1), "fence_regressed"), + (row(5, T2, T1), (4, T3), "fence_regressed"), + (row(5, T2, None), (4, T1), "fence_regressed"), + ], +) +def test_it_refuses_every_other_state_with_the_epochs_in_the_refusal( + recorded: OrphanFenceRow, latest: tuple[int, str] | None, cause: str +) -> None: + decision = decide_orphan_fence(recorded, latest) + assert decision.action == "fenced" and decision.refusal is not None + assert decision.refusal.cause == cause + assert decision.refusal.recorded_epoch == recorded.epoch + assert decision.refusal.fence_epoch == (latest[0] if latest else None) + + +def test_no_input_makes_it_delete_when_the_bucket_is_ahead_of_the_record() -> None: + for recorded in (None, row(1), row(7, T1, T2)): + for epoch in range(1, 12): + floor = recorded.epoch if recorded else 0 + if epoch <= floor: + continue + for token in (T1, T2, T3): + assert decide_orphan_fence(recorded, (epoch, token)).action == "fenced" + + +def test_a_record_round_trips_and_a_first_claim_carries_a_null_previous_token() -> None: + record = OrphanFenceRecord(1, T1, None, "2026-10-01T00:00:00.000Z") + data = encode_orphan_fence_record(NAMESPACE, record) + assert data.endswith(b"\n") and json.loads(data)["previous_token"] is None + assert list(json.loads(data)) == [ + "v", + "namespace_key", + "epoch", + "token", + "previous_token", + "claimed_at", + ], "the key order the JavaScript SDK writes" + assert parse_orphan_fence_record(data, namespace_key=NAMESPACE, epoch=1) == record + chained = OrphanFenceRecord(2, T2, T1, "2026-10-02T00:00:00.000Z") + assert ( + parse_orphan_fence_record( + encode_orphan_fence_record(NAMESPACE, chained), namespace_key=NAMESPACE, epoch=2 + ) + == chained + ) + + +def test_it_parses_strictly_and_rejects_anything_that_is_not_exactly_a_fence_of_this_epoch() -> ( + None +): + good = { + "v": 1, + "namespace_key": NAMESPACE, + "epoch": 3, + "token": T1, + "previous_token": T2, + "claimed_at": "2026-10-01T00:00:00.000Z", + } + + def parse(value: object, epoch: int = 3, namespace: str = NAMESPACE) -> object: + data = value if isinstance(value, bytes) else json.dumps(value).encode() + return parse_orphan_fence_record(data, namespace_key=namespace, epoch=epoch) + + assert parse(good) is not None + assert parse(good, epoch=4) is None, "another epoch" + assert parse(good, namespace="b" * 64) is None, "another namespace" + for change in ( + {"v": 2}, + {"v": True}, + {"epoch": 4}, + {"epoch": "3"}, + {"epoch": True}, + {"token": "nope"}, + {"token": T1.upper()}, + {"previous_token": "nope"}, + {"previous_token": 5}, + {"claimed_at": "yesterday"}, + {"claimed_at": "2026-10-01"}, + {"claimed_at": 5}, + {"extra": 1}, + ): + assert parse({**good, **change}) is None, change + missing = dict(good) + del missing["previous_token"] + assert parse(missing) is None + for bad in (b"", b"not json", b"[]", b"null", b"3", b"\xff\xfe"): + assert parse(bad) is None + assert parse(b" " * (ORPHAN_FENCE_MAX_BYTES + 1)) is None + + +def test_keys_sort_by_epoch_and_only_exact_epoch_names_under_the_fence_directory_count() -> None: + assert orphan_fence_directory(OWNED) == f"{OWNED}_collector-fence/" + keys = [orphan_fence_key(OWNED, epoch) for epoch in (2, 10, 1, 9)] + assert sorted(keys) == [orphan_fence_key(OWNED, epoch) for epoch in (1, 2, 9, 10)] + assert orphan_fence_key(OWNED, 7).endswith("/0000000000000007.json") + assert orphan_fence_epoch_of_key(OWNED, orphan_fence_key(OWNED, 12)) == 12 + directory = orphan_fence_directory(OWNED) + for other in ( + f"{directory}7.json", + f"{directory}00000000000000007.json", + f"{directory}0000000000000000.json", + f"{directory}0000000000000007.json.tmp", + f"{directory}0000000000000007.JSON", + f"{directory}٣000000000000007.json", + f"{directory}sub/0000000000000007.json", + f"{OWNED}0000000000000007.json", + ): + assert orphan_fence_epoch_of_key(OWNED, other) is None, other + assert is_orphan_fence_key(OWNED, f"{directory}anything") + assert not is_orphan_fence_key(OWNED, f"{OWNED}acct/row.jsonl") + + +def test_a_fence_is_one_physical_place_provider_location_prefix_and_namespace() -> None: + base = { + "provider": "s3", + "location": {"bucket": "rows", "region": "us-east-1"}, + "prefix": "adcp-rows", + "namespace_key": NAMESPACE, + } + scope = orphan_fence_scope(**base) # type: ignore[arg-type] + assert len(scope) == 64 and int(scope, 16) >= 0 + assert orphan_fence_scope(**{**base, "location": {"region": "us-east-1", "bucket": "rows"}}) == scope # type: ignore[arg-type] + for change in ( + {"provider": "gcs"}, + {"location": {"bucket": "other", "region": "us-east-1"}}, + {"prefix": "other"}, + {"namespace_key": "b" * 64}, + ): + assert orphan_fence_scope(**{**base, **change}) != scope, change # type: ignore[arg-type] + + +def test_the_scope_hash_is_the_one_the_javascript_sdk_computes() -> None: + """sha256 of the RFC 8785 form of ["adcp.orphan_fence.v1", provider, location, prefix, ns].""" + import hashlib + + canonical = ( + '["adcp.orphan_fence.v1","filesystem",{"root":"rows"},"orph","' + NAMESPACE + '"]' + ).encode() + assert ( + orphan_fence_scope( + provider="filesystem", location={"root": "rows"}, prefix="orph", namespace_key=NAMESPACE + ) + == hashlib.sha256(canonical).hexdigest() + ) diff --git a/tests/test_reporting_orphan_maintenance.py b/tests/test_reporting_orphan_maintenance.py new file mode 100644 index 0000000000..6e8034254a --- /dev/null +++ b/tests/test_reporting_orphan_maintenance.py @@ -0,0 +1,201 @@ +"""Scheduling the orphan collector: options, the store option and the cycle state. + +Ported from the JavaScript SDK (adcp-client #3291 and #3593, ``reporting-maintenance-warnings`` and +``reliable-reporting-service``). No database: the collector itself is replaced. +""" + +from __future__ import annotations + +import time +from dataclasses import replace +from typing import Any + +import pytest +from adcp_reporting_storage import PgReportingLedgerStore +from adcp_reporting_storage.ledger import orphan_collector +from adcp_reporting_storage.ledger.orphan_collector import ( + ReportingOrphanBindingResultV1, + ReportingOrphanCollectionOptionsV1, + ReportingOrphanCollectionResultV1, + ReportingOrphanCollectionStateV1, + run_orphan_collection_pass, +) +from adcp_reporting_storage.ledger.orphan_fence import ReportingOrphanFenceRefusalV1 + + +def options(**overrides: Any) -> ReportingOrphanCollectionOptionsV1: + return ReportingOrphanCollectionOptionsV1( + **{"acknowledge_exclusive_namespace": True, **overrides} + ) + + +def test_deleting_needs_the_acknowledgement_and_every_bound_is_checked() -> None: + with pytest.raises(ValueError, match="acknowledge_exclusive_namespace"): + ReportingOrphanCollectionOptionsV1() + assert ReportingOrphanCollectionOptionsV1(dry_run=True).dry_run is True + assert options().interval_seconds == 86_400 and options().min_age_seconds == 7 * 86_400 + assert options(min_age_seconds=86_400).min_age_seconds == 86_400 + for bad in ( + {"min_age_seconds": 86_399}, + {"min_age_seconds": 0}, + {"min_age_seconds": 1.5}, + {"limit": 0}, + {"limit": -1}, + {"limit": 1.5}, + {"interval_seconds": 0}, + {"interval_seconds": True}, + {"dry_run": "yes"}, + {"include_read_only": 1}, + {"acknowledge_exclusive_namespace": "true"}, + ): + with pytest.raises(ValueError): + options(**bad) + + +def test_the_store_option_needs_row_storage_and_adds_the_maintenance_operation() -> None: + pytest.importorskip("psycopg_pool") + pool = object() + with pytest.raises(ValueError, match="requires row_storage"): + PgReportingLedgerStore(pool=pool, orphan_collection=options()) # type: ignore[arg-type] + with pytest.raises(TypeError): + PgReportingLedgerStore( + pool=pool, # type: ignore[arg-type] + row_storage=True, + orphan_collection=True, # type: ignore[arg-type] + ) + plain = PgReportingLedgerStore(pool=pool, row_storage=True) # type: ignore[arg-type] + assert "orphan_collection" not in [name for name, _ in plain.maintenance_operations()] + scheduled = PgReportingLedgerStore( + pool=pool, # type: ignore[arg-type] + row_storage=True, + orphan_collection=options(), + ) + names = [name for name, _ in scheduled.maintenance_operations()] + assert names[-1] == "orphan_collection" and "intent_sweep" in names + + +def result(**changes: Any) -> ReportingOrphanCollectionResultV1: + return replace(ReportingOrphanCollectionResultV1(dry_run=False), **changes) + + +def binding(binding_id: str, status: Any = "complete", **changes: Any) -> Any: + return ReportingOrphanBindingResultV1(binding_id, status, **changes) + + +class Recorder: + def __init__(self, *outcomes: ReportingOrphanCollectionResultV1) -> None: + self.outcomes = list(outcomes) + self.calls: list[dict[str, Any]] = [] + + async def __call__(self, store: Any, **kwargs: Any) -> ReportingOrphanCollectionResultV1: + self.calls.append(kwargs) + return self.outcomes.pop(0) + + +async def test_a_pass_forwards_its_options_replaces_the_saved_state_and_rests_after_a_clean_listing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorder = Recorder( + result( + cursors={"objects": "page-2"}, + completed=("legacy",), + bindings=(binding("objects", "partial"),), + ), + result(cursors={}, completed=("objects",), bindings=(binding("objects"),)), + ) + monkeypatch.setattr(orphan_collector, "collect_orphaned_row_objects", recorder) + configured = options( + dry_run=False, + limit=50, + min_age_seconds=3 * 86_400, + include_read_only=True, + interval_seconds=600, + ) + state = ReportingOrphanCollectionStateV1(cursors={"stale": "x"}, completed=["done"]) + await run_orphan_collection_pass(object(), configured, state) # type: ignore[arg-type] + assert recorder.calls[0] == { + "dry_run": False, + "acknowledge_exclusive_namespace": True, + "min_age_seconds": 3 * 86_400, + "limit": 50, + "include_read_only": True, + "cursors": {"stale": "x"}, + "skip_bindings": ["done"], + } + # An unfinished listing carries on next pass, remembering what already finished. + assert state.cursors == {"objects": "page-2"} + assert state.completed == ["done", "legacy"] and state.next_cycle_at is None + await run_orphan_collection_pass(object(), configured, state) # type: ignore[arg-type] + assert recorder.calls[1]["cursors"] == {"objects": "page-2"} + assert state.cursors == {} and state.completed == [] + assert state.next_cycle_at is not None + assert 590 < state.next_cycle_at - time.monotonic() <= 600, "a clean cycle rests the interval" + # Between full listings the pass does nothing, and does not call the collector. + idle = await run_orphan_collection_pass(object(), configured, state) # type: ignore[arg-type] + assert idle.listed == 0 and idle.dry_run is False and len(recorder.calls) == 2 + + +async def test_a_binding_that_finished_is_not_listed_again_while_another_needs_more_passes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorder = Recorder( + result( + cursors={"big": "t1"}, + completed=("small",), + bindings=(binding("small"), binding("big", "partial")), + ), + result(cursors={"big": "t2"}, completed=(), bindings=(binding("big", "partial"),)), + result(cursors={}, completed=("big",), bindings=(binding("big"),)), + ) + monkeypatch.setattr(orphan_collector, "collect_orphaned_row_objects", recorder) + state = ReportingOrphanCollectionStateV1() + for _ in range(3): + await run_orphan_collection_pass(object(), options(), state) # type: ignore[arg-type] + assert [call["skip_bindings"] for call in recorder.calls] == [[], ["small"], ["small"]] + assert state.completed == [] and state.cursors == {} and state.next_cycle_at is not None + + +async def test_a_failed_binding_keeps_the_task_retrying_every_pass( + monkeypatch: pytest.MonkeyPatch, +) -> None: + failing = result( + cursors={}, failed=1, bindings=(binding("objects", "failed", error="PROVIDER_UNAVAILABLE"),) + ) + recorder = Recorder(failing, failing) + monkeypatch.setattr(orphan_collector, "collect_orphaned_row_objects", recorder) + state = ReportingOrphanCollectionStateV1() + for _ in range(2): + await run_orphan_collection_pass(object(), options(), state) # type: ignore[arg-type] + assert state.next_cycle_at is None, "a failure does not rest the interval" + assert len(recorder.calls) == 2 + + +async def test_a_fenced_binding_rests_at_most_an_hour_whatever_the_interval( + monkeypatch: pytest.MonkeyPatch, +) -> None: + refusal = ReportingOrphanFenceRefusalV1("fence_advanced", "take over", 5, 3) + fenced = result(fenced=("objects",), bindings=(binding("objects", "fenced", fence=refusal),)) + recorder = Recorder(fenced, fenced) + monkeypatch.setattr(orphan_collector, "collect_orphaned_row_objects", recorder) + state = ReportingOrphanCollectionStateV1() + await run_orphan_collection_pass(object(), options(interval_seconds=86_400), state) # type: ignore[arg-type] + assert state.next_cycle_at is not None + assert 3_590 < state.next_cycle_at - time.monotonic() <= 3_600 + # A short interval is kept as it is. + short = ReportingOrphanCollectionStateV1() + await run_orphan_collection_pass(object(), options(interval_seconds=120), short) # type: ignore[arg-type] + assert short.next_cycle_at is not None + assert 110 < short.next_cycle_at - time.monotonic() <= 120 + + +async def test_a_rejected_collector_propagates_so_the_worker_counts_it_as_a_failed_task( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def explode(store: Any, **kwargs: Any) -> Any: + raise RuntimeError("boom") + + monkeypatch.setattr(orphan_collector, "collect_orphaned_row_objects", explode) + state = ReportingOrphanCollectionStateV1(cursors={"a": "b"}) + with pytest.raises(RuntimeError): + await run_orphan_collection_pass(object(), options(), state) # type: ignore[arg-type] + assert state.cursors == {"a": "b"} and state.next_cycle_at is None diff --git a/tests/test_reporting_production_maintenance.py b/tests/test_reporting_production_maintenance.py index b410f3e844..459249e065 100644 --- a/tests/test_reporting_production_maintenance.py +++ b/tests/test_reporting_production_maintenance.py @@ -269,3 +269,51 @@ async def run_configuration(self, configuration: str) -> None: for r in caplog.records if getattr(r, "code", "") == "REPORTING_PRODUCTION_OWED_SET_FAILED" ] + + +async def test_orphan_collection_reports_fenced_and_unsupported_bindings_as_integer_counters( + caplog: pytest.LogCaptureFixture, +) -> None: + """A fenced namespace is stalled work; a binding that cannot list is pending work. + + Only the closed operation name and counters are logged, never a binding or bucket name. + """ + from types import SimpleNamespace + + class OrphanStore: + def maintenance_operations(self) -> tuple[tuple[str, Any], ...]: + async def run() -> Any: + return SimpleNamespace( + failed=2, + fenced=("secret-binding", "other-binding"), + unsupported=("nolist-binding",), + deleted=9, + ) + + return (("orphan_collection", run),) + + worker = service(OrphanStore()) + with caplog.at_level(logging.INFO, logger="adcp.reporting.production"): + await worker._maintain() + by_code = { + code: [r for r in caplog.records if getattr(r, "code", "") == code] + for code in ( + "REPORTING_MAINTENANCE_FAILED", + "REPORTING_MAINTENANCE_PENDING", + "REPORTING_MAINTENANCE_STALLED", + ) + } + assert [r.operation for r in by_code["REPORTING_MAINTENANCE_FAILED"]] == ["orphan_collection"] + (pending,) = by_code["REPORTING_MAINTENANCE_PENDING"] + assert pending.operation == "orphan_collection" and pending.unsupported == 1 + (stalled,) = by_code["REPORTING_MAINTENANCE_STALLED"] + assert stalled.operation == "orphan_collection" and stalled.fenced == 2 + assert stalled.levelno == logging.WARNING + assert all( + "binding" not in r.getMessage() and "secret" not in r.getMessage() for r in caplog.records + ) + assert worker.maintenance_report["orphan_collection"] == { + "failed": 2, + "fenced": 2, + "unsupported": 1, + } diff --git a/tests/test_reporting_row_binding_bounds.py b/tests/test_reporting_row_binding_bounds.py new file mode 100644 index 0000000000..39d72eb1d3 --- /dev/null +++ b/tests/test_reporting_row_binding_bounds.py @@ -0,0 +1,164 @@ +"""``row_binding_id`` and ``credential_ref`` accept 1 to 255 characters (adcp-client-python #1498). + +Both were capped at 128, tighter than every comparable AdCP identifier (``reporting_revision_id``, +``resource_ref`` and ``destination_ref`` use ``^[A-Za-z0-9_.:-]{1,255}$``). Ported from the +JavaScript SDK (adcp-client #3595, ``reporting-row-binding-bounds``). No database needed: this is +the configuration check and the provider ``clients`` map. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +from adcp_reporting_storage.ledger.row_storage import ( + ReportingObjectRowBindingV1, + ReportingPostgresRowBindingV1, + ReportingRowStorageOptionsV1, + ReportingRowStorageV1, +) +from adcp_reporting_storage.ledger.row_storage_object import ( + REPORTING_ROW_BINDING_ID_MAX_LENGTH, + REPORTING_ROW_CREDENTIAL_REF_MAX_LENGTH, + create_filesystem_reporting_row_object_provider_v1, +) +from adcp_reporting_storage.ledger.row_storage_provider_io import client_map, is_row_identifier + +from adcp.reporting.ledger.row_storage_errors import ReportingRowStoreError + +HOSTILE = ( + "", + "has space", + "tab\there", + "trailing-newline\n", + "nul\x00byte", + "ａfullwidth", # fullwidth "a" + "аcyrillic", # Cyrillic "a" next to the Latin one + "٣arabic-indic-digit", + "semi;colon", + "slash/er", +) + + +def test_the_bounds_are_the_adcp_identifier_bound() -> None: + assert REPORTING_ROW_BINDING_ID_MAX_LENGTH == 255 + assert REPORTING_ROW_CREDENTIAL_REF_MAX_LENGTH == 255 + + +@pytest.mark.parametrize("length", [1, 128, 129, 254, 255]) +def test_an_identifier_up_to_255_characters_is_accepted(length: int) -> None: + assert is_row_identifier("a" * length) + + +@pytest.mark.parametrize("length", [256, 300, 5000]) +def test_an_identifier_beyond_255_characters_is_refused(length: int) -> None: + assert not is_row_identifier("a" * length) + + +def test_every_character_of_the_identifier_alphabet_is_accepted() -> None: + assert is_row_identifier("Az09_.:-") + + +@pytest.mark.parametrize("value", [*HOSTILE, None, 7, b"abc", ["a"]]) +def test_hostile_and_non_string_identifiers_are_refused(value: Any) -> None: + assert not is_row_identifier(value) + + +def storage(tmp_path: Path, binding_id: str = "reports", **binding: Any) -> ReportingRowStorageV1: + providers = { + "filesystem": create_filesystem_reporting_row_object_provider_v1( + roots={"rows": tmp_path / "rows"} + ) + } + defaults: dict[str, Any] = { + "provider": "filesystem", + "location": {"root": "rows"}, + "prefix": "adcp-rows", + } + return ReportingRowStorageV1( + ReportingRowStorageOptionsV1( + bindings={binding_id: ReportingObjectRowBindingV1(**{**defaults, **binding})}, + providers=providers, + ) + ) + + +@pytest.mark.parametrize("length", [128, 129, 255]) +def test_a_binding_id_up_to_255_characters_passes_the_configuration_check( + tmp_path: Path, length: int +) -> None: + storage(tmp_path, "b" * length) + ReportingRowStorageV1( + ReportingRowStorageOptionsV1(bindings={"p" * length: ReportingPostgresRowBindingV1()}) + ) + + +def test_a_binding_id_beyond_255_characters_is_refused_and_not_echoed(tmp_path: Path) -> None: + with pytest.raises(ReportingRowStoreError) as failure: + storage(tmp_path, "x" * 5000) + assert failure.value.code == "INVALID_INPUT" + message = str(failure.value) + assert "1 to 255 characters of A-Za-z0-9_.:-" in message + assert len(message) < 300, "a hostile identifier is not echoed whole" + assert "x" * 100 not in message + + +def test_a_short_invalid_binding_id_is_quoted_with_the_bound(tmp_path: Path) -> None: + with pytest.raises(ReportingRowStoreError) as failure: + storage(tmp_path, "bad id") + assert '"bad id"' in str(failure.value) + assert "1 to 255 characters" in str(failure.value) + + +@pytest.mark.parametrize("binding_id", HOSTILE) +def test_hostile_binding_ids_are_refused(tmp_path: Path, binding_id: str) -> None: + with pytest.raises(ReportingRowStoreError) as failure: + storage(tmp_path, binding_id) + assert failure.value.code == "INVALID_INPUT" + + +@pytest.mark.parametrize("key", [7, b"abc", ("a", "b"), None, 1.5, b"x" * 5000, object]) +def test_a_non_string_binding_id_is_refused_not_a_type_error(key: Any) -> None: + with pytest.raises(ReportingRowStoreError) as failure: + ReportingRowStorageV1( + ReportingRowStorageOptionsV1( + bindings={key: ReportingPostgresRowBindingV1()} # type: ignore[dict-item] + ) + ) + assert failure.value.code == "INVALID_INPUT" + assert len(str(failure.value)) < 300, "a hostile key is not echoed whole" + + +@pytest.mark.parametrize("length", [128, 129, 255]) +def test_a_credential_ref_up_to_255_characters_passes_the_configuration_check( + tmp_path: Path, length: int +) -> None: + storage(tmp_path, credential_ref="c" * length) + + +def test_a_credential_ref_beyond_255_characters_is_refused(tmp_path: Path) -> None: + with pytest.raises(ReportingRowStoreError) as failure: + storage(tmp_path, credential_ref="c" * 256) + assert failure.value.code == "INVALID_INPUT" + assert "1 to 255 characters of A-Za-z0-9_.:-" in str(failure.value) + assert "c" * 100 not in str(failure.value) + + +@pytest.mark.parametrize("credential_ref", HOSTILE) +def test_hostile_credential_refs_are_refused(tmp_path: Path, credential_ref: str) -> None: + with pytest.raises(ReportingRowStoreError) as failure: + storage(tmp_path, credential_ref=credential_ref) + assert failure.value.code == "INVALID_INPUT" + + +def test_a_provider_clients_map_holds_every_reference_a_binding_can_name() -> None: + client = object() + for length in (128, 129, 255): + assert client_map({"r" * length: client}) == {"r" * length: client} + for bad in ("r" * 256, *HOSTILE): + with pytest.raises(ReportingRowStoreError) as failure: + client_map({bad: client}) + assert failure.value.code == "INVALID_INPUT" + with pytest.raises(ReportingRowStoreError): + client_map({"ok": None}) diff --git a/tests/test_reporting_row_storage_azure.py b/tests/test_reporting_row_storage_azure.py index c42c2b5af2..2bf22c0685 100644 --- a/tests/test_reporting_row_storage_azure.py +++ b/tests/test_reporting_row_storage_azure.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from datetime import datetime, timezone from types import SimpleNamespace from typing import Any @@ -18,6 +19,7 @@ ReportingRowObjectContextV1, ReportingRowObjectDeleteInputV1, ReportingRowObjectGetInputV1, + ReportingRowObjectListInputV1, ReportingRowObjectPutInputV1, run_reporting_row_object_provider_conformance_v1, ) @@ -43,6 +45,7 @@ class _Version: content_type: str | None cache_control: str | None current: bool = True + created: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) @property def etag(self) -> str: @@ -73,6 +76,55 @@ def get_container_properties(self, **transport: Any) -> Any: def get_blob_client(self, key: str) -> Any: return Blob(name, key) + def list_blobs( + self, + name_starts_with: str | None = None, + include: list[str] | None = None, + results_per_page: int | None = None, + **transport: Any, + ) -> Any: + service.transport.append(transport) + entries = [ + SimpleNamespace( + name=key, + version_id=version.version_id, + size=len(version.data), + creation_time=version.created, + last_modified=version.created, + ) + for (container, key), history in sorted(service.blobs.items()) + if container == name and key.startswith(name_starts_with or "") + for version in history + ] + + class Pages: + continuation_token: str | None = None + + def __init__(self, start: int) -> None: + self.start = start + self.served = False + + def __iter__(self) -> Pages: + return self + + def __next__(self) -> Any: + if self.served: + raise StopIteration + self.served = True + size = results_per_page or len(entries) + chunk = entries[self.start : self.start + size] + following = self.start + len(chunk) + self.continuation_token = ( + str(following) if following < len(entries) else None + ) + return iter(chunk) + + class Paged: + def by_page(self, continuation_token: str | None = None) -> Pages: + return Pages(int(continuation_token) if continuation_token else 0) + + return Paged() + class Blob: def __init__(self, container: str, key: str) -> None: self.container, self.key = container, key @@ -216,7 +268,7 @@ async def test_passes_the_shared_provider_conformance_suite() -> None: passed = await run_reporting_row_object_provider_conformance_v1( provider, location=LOCATION, prefix="adcp-rows" ) - assert passed[-1] == "write-unique-version" and "ranged-read" in passed + assert "write-unique-version" in passed and "ranged-read" in passed and passed[-1] == "list" async def test_create_only_upload_sets_headers_and_identifier_safe_metadata() -> None: @@ -395,3 +447,60 @@ def test_the_official_client_exposes_everything_the_provider_calls() -> None: from azure.core import MatchConditions assert hasattr(MatchConditions, "IfNotModified") + + +# -- list --------------------------------------------------------------------------------------- + + +async def test_list_pages_every_version_and_carries_the_call_budget() -> None: + service = FakeBlobService() + provider = create_azure_blob_reporting_row_object_provider_v1(client=service) + keys = [f"adcp-rows/ns/o{index}.jsonl" for index in range(5)] + versions = {} + for key in keys: + versions[key] = (await provider.put_if_absent(put(key, b"abc"), CONTEXT)).native_version_ref + found = [] + token = None + context = ReportingRowObjectContextV1(7.5) + for _ in range(20): + page = await provider.list( + ReportingRowObjectListInputV1(LOCATION, "adcp-rows/ns/", 2, token), context + ) + assert len(page.objects) <= 2 + found.extend(page.objects) + token = page.next_page_token + if token is None: + break + assert {(e.object_ref, e.native_version_ref, e.size_bytes) for e in found} == { + (key, versions[key], 3) for key in keys + } + assert service.transport[-1]["timeout"] == 8 and service.transport[-1]["read_timeout"] == 7.5 + + +async def test_list_skips_a_blob_without_a_version_id_size_or_time_and_counts_it() -> None: + service = FakeBlobService() + provider = create_azure_blob_reporting_row_object_provider_v1(client=service) + await provider.put_if_absent(put("adcp-rows/ns/good.jsonl"), CONTEXT) + # A blob written before versioning was enabled has no version ID. + service.blobs[("rows", "adcp-rows/ns/before-versioning.jsonl")] = [ + type(service.blobs[("rows", "adcp-rows/ns/good.jsonl")][0])( + "not a version id!", b"x", {}, None, None + ) + ] + page = await provider.list( + ReportingRowObjectListInputV1(LOCATION, "adcp-rows/ns/", 10), CONTEXT + ) + assert [e.object_ref for e in page.objects] == ["adcp-rows/ns/good.jsonl"] + assert page.skipped == 1 + + +async def test_list_refuses_a_bad_page_size_and_an_oversized_prefix() -> None: + provider = create_azure_blob_reporting_row_object_provider_v1(client=FakeBlobService()) + for size in (0, -1, 1.5): + with pytest.raises(ReportingRowStoreError) as failure: + await provider.list( + ReportingRowObjectListInputV1(LOCATION, "adcp-rows/", size), CONTEXT # type: ignore[arg-type] + ) + assert failure.value.code == "INVALID_INPUT" + with pytest.raises(ReportingRowStoreError): + await provider.list(ReportingRowObjectListInputV1(LOCATION, "p" * 1025, 5), CONTEXT) diff --git a/tests/test_reporting_row_storage_filesystem.py b/tests/test_reporting_row_storage_filesystem.py index 099f0e0828..cfb4d2a622 100644 --- a/tests/test_reporting_row_storage_filesystem.py +++ b/tests/test_reporting_row_storage_filesystem.py @@ -3,7 +3,10 @@ from __future__ import annotations import os +from dataclasses import replace +from datetime import datetime, timezone from pathlib import Path +from typing import Any import pytest from adcp_reporting_storage.ledger.row_storage_object import ( @@ -11,10 +14,14 @@ ReportingRowObjectContextV1, ReportingRowObjectDeleteInputV1, ReportingRowObjectGetInputV1, + ReportingRowObjectListInputV1, ReportingRowObjectPutInputV1, + ReportingRowObjectStatInputV1, assert_reporting_row_key_template_v1, assert_reporting_row_prefix_v1, create_filesystem_reporting_row_object_provider_v1, + is_row_object_key_shape, + parse_row_instant, render_reporting_row_object_key_v1, reporting_row_account_key_v1, run_reporting_row_object_provider_conformance_v1, @@ -43,6 +50,7 @@ async def test_the_provider_passes_the_shared_conformance_suite(tmp_path: Path) "version-pinned-stat", "exact-version-delete", "write-unique-version", + "list", ] @@ -321,3 +329,252 @@ def key(row_set_id: str) -> str: ) assert key("rrev_1") != key("rrev_2") + + +# -- list (the orphan collector's capability) ------------------------------------------------- + +LIST_NAMESPACE = f"adcp-rows/{NAMESPACE}" + + +async def _put_all(provider: Any, keys: list[str]) -> dict[str, str]: + versions = {} + for key in keys: + created = await provider.put_if_absent( + ReportingRowObjectPutInputV1( + {"root": "rows"}, key, f"{key}\n".encode(), "application/x-ndjson", {} + ), + CONTEXT, + ) + versions[key] = created.native_version_ref + return versions + + +async def _list_all(provider: Any, prefix: str, page_size: int) -> list[Any]: + found: list[Any] = [] + token = None + for _ in range(200): + page = await provider.list( + ReportingRowObjectListInputV1({"root": "rows"}, prefix, page_size, token), CONTEXT + ) + assert len(page.objects) <= page_size + found.extend(page.objects) + token = page.next_page_token + if token is None: + return found + raise AssertionError("listing did not finish") + + +async def test_list_returns_every_object_under_the_prefix_once_in_a_stable_order( + tmp_path: Path, +) -> None: + provider = provider_in(tmp_path) + keys = [ + f"{LIST_NAMESPACE}/a/1.jsonl", + f"{LIST_NAMESPACE}/a/2.jsonl.gz", + f"{LIST_NAMESPACE}/a/b/3.jsonl", + f"{LIST_NAMESPACE}/a.jsonl", + f"{LIST_NAMESPACE}/b.jsonl", + f"{LIST_NAMESPACE}/z/9.jsonl", + ] + await _put_all(provider, keys) + expected = [entry.object_ref for entry in await _list_all(provider, f"{LIST_NAMESPACE}/", 100)] + assert sorted(expected) == sorted(keys) and len(set(expected)) == len(keys) + for page_size in (1, 2, 3, 5, 100): + seen = [e.object_ref for e in await _list_all(provider, f"{LIST_NAMESPACE}/", page_size)] + assert seen == expected, page_size + assert [e.object_ref for e in await _list_all(provider, f"{LIST_NAMESPACE}/a/", 2)] == [ + f"{LIST_NAMESPACE}/a/1.jsonl", + f"{LIST_NAMESPACE}/a/2.jsonl.gz", + f"{LIST_NAMESPACE}/a/b/3.jsonl", + ] + assert await _list_all(provider, f"{LIST_NAMESPACE}/zz", 5) == [] + + +async def test_list_reports_the_version_size_and_write_time_the_other_operations_use( + tmp_path: Path, +) -> None: + provider = provider_in(tmp_path) + key = f"{LIST_NAMESPACE}/acct/x.jsonl" + created = await provider.put_if_absent( + ReportingRowObjectPutInputV1({"root": "rows"}, key, b"0123456789", "application/gzip", {}), + CONTEXT, + ) + (entry,) = await _list_all(provider, f"{LIST_NAMESPACE}/", 10) + assert entry.native_version_ref == created.native_version_ref + stat = await provider.stat( + ReportingRowObjectStatInputV1({"root": "rows"}, key, created.native_version_ref), CONTEXT + ) + assert stat is not None and entry.size_bytes == stat.size_bytes == 10 + written = parse_row_instant(entry.created_at) + assert written is not None and abs((written - datetime.now(timezone.utc)).total_seconds()) < 60 + assert entry.created_at.endswith("Z") + + +async def test_list_never_returns_a_temporary_file_or_a_key_outside_the_prefix( + tmp_path: Path, +) -> None: + provider = provider_in(tmp_path) + await _put_all(provider, [f"{LIST_NAMESPACE}/ok.jsonl", "other/prefix.jsonl"]) + leftover = tmp_path / "rows" / LIST_NAMESPACE / "half-written.jsonl.123.tmp" + leftover.write_bytes(b"x") + keys = [e.object_ref for e in await _list_all(provider, f"{LIST_NAMESPACE}/", 10)] + assert keys == [f"{LIST_NAMESPACE}/ok.jsonl"] + + +async def test_list_resumes_after_the_object_a_previous_page_ended_on_was_deleted( + tmp_path: Path, +) -> None: + provider = provider_in(tmp_path) + keys = [f"{LIST_NAMESPACE}/o{index}.jsonl" for index in range(5)] + versions = await _put_all(provider, keys) + first = await provider.list( + ReportingRowObjectListInputV1({"root": "rows"}, f"{LIST_NAMESPACE}/", 2), CONTEXT + ) + assert first.next_page_token == keys[1] + for key in keys[:2]: + await provider.delete( + ReportingRowObjectDeleteInputV1({"root": "rows"}, key, versions[key]), CONTEXT + ) + rest = await _list_all_from(provider, f"{LIST_NAMESPACE}/", first.next_page_token) + assert rest == keys[2:] + + +async def _list_all_from(provider: Any, prefix: str, token: str | None) -> list[str]: + found: list[str] = [] + while True: + page = await provider.list( + ReportingRowObjectListInputV1({"root": "rows"}, prefix, 2, token), CONTEXT + ) + found.extend(entry.object_ref for entry in page.objects) + token = page.next_page_token + if token is None: + return found + + +async def test_list_refuses_a_bad_page_size_an_unknown_root_and_a_prefix_that_climbs_out( + tmp_path: Path, +) -> None: + provider = provider_in(tmp_path) + for bad in (0, -1, 1.5, "5"): + with pytest.raises(ReportingRowStoreError) as failure: + await provider.list( + ReportingRowObjectListInputV1({"root": "rows"}, "adcp-rows/", bad), CONTEXT # type: ignore[arg-type] + ) + assert failure.value.code == "INVALID_INPUT" + with pytest.raises(ReportingRowStoreError) as unknown: + await provider.list( + ReportingRowObjectListInputV1({"root": "nope"}, "adcp-rows/", 5), CONTEXT + ) + assert unknown.value.code == "UNSAFE_BINDING" + with pytest.raises(ReportingRowStoreError) as climbing: + await provider.list( + ReportingRowObjectListInputV1({"root": "rows"}, "adcp-rows/../x/", 5), CONTEXT + ) + assert climbing.value.code == "INVALID_INPUT" + + +class _BrokenLister: + """A filesystem provider whose ``list`` breaks one rule the collector relies on.""" + + name = "filesystem" + ranged_reads = True + + def __init__(self, inner: Any, defect: str) -> None: + self._inner, self._defect = inner, defect + for method in ("validate_location", "probe", "put_if_absent", "get", "stat", "delete"): + setattr(self, method, getattr(inner, method)) + + async def list(self, request: Any, context: Any) -> Any: + page = await self._inner.list(request, context) + objects = list(page.objects) + if self._defect == "duplicate" and objects: + objects.append(objects[0]) + elif self._defect == "wrong_version" and objects: + objects[0] = replace(objects[0], native_version_ref="9:9:9:9") + elif self._defect == "wrong_size" and objects: + objects[0] = replace(objects[0], size_bytes=objects[0].size_bytes + 1) + elif self._defect == "no_time" and objects: + objects[0] = replace(objects[0], created_at="2000-01-01T00:00:00.000Z") + elif self._defect == "ignores_prefix": + return await self._inner.list(replace(request, prefix=request.prefix[:-1]), context) + elif self._defect == "too_many" and objects: + objects = objects * (request.page_size + 1) + elif self._defect == "repeat_token": + return replace(page, next_page_token="same") + elif self._defect == "drops_last" and len(objects) > 1: + objects.pop() + return replace(page, objects=tuple(objects)) + + +@pytest.mark.parametrize( + "defect", + [ + "duplicate", + "wrong_version", + "wrong_size", + "no_time", + "too_many", + "repeat_token", + "drops_last", + ], +) +async def test_the_conformance_runner_rejects_a_list_that_breaks_the_collectors_contract( + tmp_path: Path, defect: str +) -> None: + provider = _BrokenLister(provider_in(tmp_path), defect) + with pytest.raises(AssertionError): + await run_reporting_row_object_provider_conformance_v1( + provider, location={"root": "rows"}, prefix="adcp-rows" # type: ignore[arg-type] + ) + + +async def test_a_provider_without_list_still_passes_conformance_and_reports_no_list_check( + tmp_path: Path, +) -> None: + inner = provider_in(tmp_path) + + class WithoutList: + name = "filesystem" + ranged_reads = True + + def __init__(self) -> None: + for method in ("validate_location", "probe", "put_if_absent", "get", "stat", "delete"): + setattr(self, method, getattr(inner, method)) + + passed = await run_reporting_row_object_provider_conformance_v1( + WithoutList(), location={"root": "rows"}, prefix="adcp-rows" # type: ignore[arg-type] + ) + assert "list" not in passed and passed[-1] == "write-unique-version" + + +def test_strict_instants_and_key_shapes() -> None: + assert parse_row_instant("2026-10-01T00:00:00Z") is not None + assert parse_row_instant("2026-10-01T00:00:00.123456789Z") is not None + for bad in ( + "2026-10-01", + "2026-10-01T00:00:00", + "2026-10-01T00:00:00+00:00", + "2026-10-01 00:00:00Z", + "2026-13-01T00:00:00Z", + "2026-02-30T00:00:00Z", + "", + None, + 7, + ): + assert parse_row_instant(bad) is None, bad + assert is_row_object_key_shape("adcp-rows/ns/acct/0001.jsonl.gz") + for bad_key in ("", "a//b", "a/./b", "a/../b", "a b", "a\n", "/a", "a/", "x" * 1025): + assert not is_row_object_key_shape(bad_key), bad_key + + +async def test_list_does_not_follow_symlinks_out_of_the_root(tmp_path: Path) -> None: + provider = provider_in(tmp_path) + await _put_all(provider, [f"{LIST_NAMESPACE}/real.jsonl"]) + outside = tmp_path / "outside" + outside.mkdir() + (outside / "secret.jsonl").write_bytes(b"x") + owned = tmp_path / "rows" / LIST_NAMESPACE + (owned / "linked-dir").symlink_to(outside, target_is_directory=True) + (owned / "linked-file.jsonl").symlink_to(outside / "secret.jsonl") + keys = [e.object_ref for e in await _list_all(provider, f"{LIST_NAMESPACE}/", 10)] + assert keys == [f"{LIST_NAMESPACE}/real.jsonl"] diff --git a/tests/test_reporting_row_storage_gcs.py b/tests/test_reporting_row_storage_gcs.py index 59ecb68f4c..05c3502e89 100644 --- a/tests/test_reporting_row_storage_gcs.py +++ b/tests/test_reporting_row_storage_gcs.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from datetime import datetime, timezone from types import SimpleNamespace from typing import Any @@ -18,6 +19,7 @@ ReportingRowObjectContextV1, ReportingRowObjectDeleteInputV1, ReportingRowObjectGetInputV1, + ReportingRowObjectListInputV1, ReportingRowObjectPutInputV1, run_reporting_row_object_provider_conformance_v1, ) @@ -35,6 +37,7 @@ class _Version: metadata: dict[str, str] content_type: str cache_control: str | None + created: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) @dataclass @@ -82,6 +85,39 @@ def lifecycle_rules(self) -> Any: def blob(self, key: str, generation: int | None = None) -> Any: return Blob(name, key, generation) + def list_blobs( + self, + prefix: str | None = None, + versions: bool | None = None, + max_results: int | None = None, + page_token: str | None = None, + timeout: Any = None, + ) -> Any: + storage.calls.append(("list", {"prefix": prefix, "versions": versions})) + entries = [ + (key, version) + for (bucket, key), history in sorted(storage.objects.items()) + if bucket == name and key.startswith(prefix or "") + for version in (history if versions else history[-1:]) + ] + start = int(page_token) if page_token else 0 + chunk = entries[start : start + (max_results or len(entries))] + following = start + len(chunk) + blobs = [ + SimpleNamespace( + name=key, + generation=version.generation, + size=len(version.data), + time_created=version.created, + updated=version.created, + ) + for key, version in chunk + ] + return SimpleNamespace( + pages=iter([blobs]), + next_page_token=str(following) if following < len(entries) else None, + ) + def get_blob(self, key: str, timeout: Any = None) -> Any: live = storage.objects.get((name, key)) if not live: @@ -175,6 +211,7 @@ async def test_passes_the_shared_provider_conformance_suite() -> None: "version-pinned-stat", "exact-version-delete", "write-unique-version", + "list", ] @@ -454,3 +491,113 @@ def bucket(self, name: str) -> Any: ReportingRowObjectDeleteInputV1(LOCATION, "adcp-rows/ns/x", "123"), context ) assert seen == [("reload", 4.5), ("delete", 4.5)] + + +# -- list --------------------------------------------------------------------------------------- + + +async def test_list_lists_every_generation_in_pages_and_leaves_out_what_it_cannot_address() -> None: + storage = FakeStorage().add_bucket() + provider = create_gcs_reporting_row_object_provider_v1(storage=storage) + keys = [f"adcp-rows/ns/o{index}.jsonl" for index in range(5)] + versions = {} + for key in keys: + versions[key] = (await provider.put_if_absent(put(key, b"abc"), CONTEXT)).native_version_ref + # A noncurrent generation of the first key is listed too: it is what a delete could name. + storage.objects[("rows-bucket", keys[0])].append( + type(storage.objects[("rows-bucket", keys[0])][0])( + storage.next_generation, b"later", {}, "application/gzip", None + ) + ) + found = [] + token = None + for _ in range(20): + page = await provider.list( + ReportingRowObjectListInputV1(LOCATION, "adcp-rows/ns/", 2, token), CONTEXT + ) + assert len(page.objects) <= 2 + found.extend(page.objects) + token = page.next_page_token + if token is None: + break + assert [(e.object_ref, e.size_bytes) for e in found].count((keys[0], 3)) == 1 + assert len(found) == 6 and {e.object_ref for e in found} == set(keys) + call = next(options for name, options in storage.calls if name == "list") + assert call["versions"] is True and call["prefix"] == "adcp-rows/ns/" + + +async def test_list_skips_an_entry_without_a_usable_generation_size_or_time_and_counts_it() -> None: + from types import SimpleNamespace + + storage = FakeStorage().add_bucket() + await create_gcs_reporting_row_object_provider_v1(storage=storage).put_if_absent( + put("adcp-rows/ns/good.jsonl"), CONTEXT + ) + + class Wrapped: + def bucket(self, name: str) -> Any: + real = storage.bucket(name) + + class Bucket: + def __getattr__(self, attribute: str) -> Any: + return getattr(real, attribute) + + def list_blobs(self, **options: Any) -> Any: + iterator = real.list_blobs(**options) + blobs = list(next(iter(iterator.pages))) + now = datetime.now(timezone.utc) + prefix = options["prefix"] + blobs += [ + SimpleNamespace( + name=f"{prefix}no-generation.jsonl", + generation=None, + size=1, + time_created=now, + updated=now, + ), + SimpleNamespace( + name=f"{prefix}bad-size.jsonl", + generation=5, + size=-1, + time_created=now, + updated=now, + ), + SimpleNamespace( + name=f"{prefix}no-time.jsonl", + generation=6, + size=1, + time_created=None, + updated=None, + ), + SimpleNamespace( + name="elsewhere/x.jsonl", + generation=7, + size=1, + time_created=now, + updated=now, + ), + ] + return SimpleNamespace( + pages=iter([blobs]), next_page_token=iterator.next_page_token + ) + + return Bucket() + + provider = create_gcs_reporting_row_object_provider_v1(storage=Wrapped()) + page = await provider.list( + ReportingRowObjectListInputV1(LOCATION, "adcp-rows/ns/", 50), CONTEXT + ) + assert [e.object_ref for e in page.objects] == ["adcp-rows/ns/good.jsonl"] + assert page.skipped == 4 + + +async def test_list_refuses_a_bad_page_size_and_an_oversized_prefix() -> None: + provider = create_gcs_reporting_row_object_provider_v1(storage=FakeStorage().add_bucket()) + for size in (0, -1, 1.5): + with pytest.raises(ReportingRowStoreError) as failure: + await provider.list( + ReportingRowObjectListInputV1(LOCATION, "adcp-rows/", size), CONTEXT # type: ignore[arg-type] + ) + assert failure.value.code == "INVALID_INPUT" + with pytest.raises(ReportingRowStoreError): + await provider.list(ReportingRowObjectListInputV1(LOCATION, "p" * 1025, 5), CONTEXT) diff --git a/tests/test_reporting_row_storage_s3.py b/tests/test_reporting_row_storage_s3.py index 0a0ff48abe..913ca84673 100644 --- a/tests/test_reporting_row_storage_s3.py +++ b/tests/test_reporting_row_storage_s3.py @@ -14,6 +14,7 @@ ReportingRowObjectContextV1, ReportingRowObjectDeleteInputV1, ReportingRowObjectGetInputV1, + ReportingRowObjectListInputV1, ReportingRowObjectPutInputV1, run_reporting_row_object_provider_conformance_v1, ) @@ -51,7 +52,7 @@ async def test_passes_the_shared_provider_conformance_suite(s3: Any) -> None: passed = await run_reporting_row_object_provider_conformance_v1( provider, location=LOCATION, prefix="adcp-rows" ) - assert passed[-1] == "write-unique-version" and "ranged-read" in passed + assert "write-unique-version" in passed and "ranged-read" in passed and passed[-1] == "list" async def test_create_only_writes_record_the_version_id_and_adopt_on_conflict(s3: Any) -> None: @@ -278,3 +279,91 @@ async def test_a_client_without_finite_socket_timeouts_is_refused_at_the_probe() with pytest.raises(ReportingRowStoreError) as refused: await provider.probe(LOCATION, "adcp-rows", CONTEXT) assert refused.value.code == "UNSAFE_BINDING" and "read_timeout" in str(refused.value) + + +# -- list --------------------------------------------------------------------------------------- + + +async def _list_pages(provider: Any, prefix: str, page_size: int) -> tuple[list[Any], int]: + found: list[Any] = [] + skipped = 0 + token = None + for _ in range(100): + page = await provider.list( + ReportingRowObjectListInputV1(LOCATION, prefix, page_size, token), CONTEXT + ) + found.extend(page.objects) + skipped += page.skipped + token = page.next_page_token + if token is None: + return found, skipped + raise AssertionError("listing did not finish") + + +async def test_list_returns_noncurrent_versions_and_never_a_delete_marker(s3: Any) -> None: + provider = create_s3_reporting_row_object_provider_v1(client=s3) + key = "adcp-rows/ns/versions.jsonl" + first = s3.put_object(Bucket="rows-bucket", Key=key, Body=b"one\n")["VersionId"] + second = s3.put_object(Bucket="rows-bucket", Key=key, Body=b"three\n")["VersionId"] + s3.delete_object(Bucket="rows-bucket", Key=key) # a delete marker, not an object + other = ( + await provider.put_if_absent(put("adcp-rows/ns/other.jsonl"), CONTEXT) + ).native_version_ref + for page_size in (1, 2, 10): + found, skipped = await _list_pages(provider, "adcp-rows/ns/", page_size) + assert skipped == 0 + assert sorted((e.object_ref, e.native_version_ref) for e in found) == sorted( + [(key, first), (key, second), ("adcp-rows/ns/other.jsonl", other)] + ), page_size + sizes = { + e.native_version_ref: e.size_bytes + for e in (await _list_pages(provider, "adcp-rows/", 5))[0] + } + assert sizes[first] == 4 and sizes[second] == 6 + + +async def test_list_leaves_out_a_version_written_before_versioning_and_counts_it(s3: Any) -> None: + provider = create_s3_reporting_row_object_provider_v1(client=s3) + s3.create_bucket(Bucket="young-bucket") + s3.put_object(Bucket="young-bucket", Key="adcp-rows/ns/before.jsonl", Body=b"x\n") + s3.put_bucket_versioning(Bucket="young-bucket", VersioningConfiguration={"Status": "Enabled"}) + s3.put_object(Bucket="young-bucket", Key="adcp-rows/ns/after.jsonl", Body=b"y\n") + found, skipped = await _list_in(provider, "young-bucket") + assert [e.object_ref for e in found] == ["adcp-rows/ns/after.jsonl"] + assert skipped == 1, "a null-version object is never addressable by a recorded version" + + +async def _list_in(provider: Any, bucket: str) -> tuple[list[Any], int]: + page = await provider.list( + ReportingRowObjectListInputV1({"bucket": bucket}, "adcp-rows/ns/", 10), CONTEXT + ) + return list(page.objects), page.skipped + + +async def test_list_refuses_a_bad_token_a_bad_page_size_and_a_truncation_without_markers( + s3: Any, +) -> None: + from types import SimpleNamespace + + provider = create_s3_reporting_row_object_provider_v1(client=s3) + for token in ("not base64!", "e30", "bm90LWpzb24"): + with pytest.raises(ReportingRowStoreError) as failure: + await provider.list( + ReportingRowObjectListInputV1(LOCATION, "adcp-rows/", 5, token), CONTEXT + ) + assert failure.value.code == "INVALID_INPUT", token + for size in (0, -3, 1.5): + with pytest.raises(ReportingRowStoreError): + await provider.list( + ReportingRowObjectListInputV1(LOCATION, "adcp-rows/", size), CONTEXT # type: ignore[arg-type] + ) + truncated = SimpleNamespace( + put_object=lambda **_: {}, + list_object_versions=lambda **_: {"Versions": [], "IsTruncated": True}, + ) + broken = create_s3_reporting_row_object_provider_v1(client=truncated) + with pytest.raises(ReportingRowStoreError) as partial: + await broken.list(ReportingRowObjectListInputV1(LOCATION, "adcp-rows/", 5), CONTEXT) + assert ( + partial.value.code == "PROVIDER_UNAVAILABLE" + ), "a partial listing is never reported complete"