From c9c818fa0d64ee6559eaa1e0191acc79a35a6623 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sun, 8 Nov 2026 03:16:59 +0000 Subject: [PATCH 1/3] feat(reporting-storage): make the contract-2 ledger upgrade rolling-deploy safe with expand and contract A ledger created by adcp-reporting-storage 0.1.x/0.2.0b0 is now upgraded in two halves, so a default rolling update works and a rollback keeps working until the operator contracts (#1856, #1831). Expand (0002, 0006; online-safe): offering_id is added with CHECK (offering_id = ''), the keys and references the earlier release uses keep their names and definitions (four-column reporting_configurations_pkey, six-column period, checkpoint and event keys, four/six-column references), and the widened ones are added beside them (reporting_configurations_generation_key, reporting_obligations_offering_period_key, reporting_status_scope_checkpoints_offering_key, reporting_status_notification_events_offering_fkey/_offering_key). While uncontracted, both releases install the earlier release's bodies of the three reconciliation functions and two status guards, so an old pod's restart changes nothing either release validates. The validators accept the expanded catalog through outbox/required_expand_schema.json, derived from a real expanded ledger. Contract (0007, "-- adcp-upgrade: contract"): drops the expand-only objects, renames the widened keys back, re-creates the offering-aware references and restores this release's function bodies; the result is catalog-identical to a new ledger. Only "python -m adcp migrate --contract" (migrate(contract=True)) applies it to an expanded ledger; plain migrate, create_schema() and apply_pending() stop before it, and record it unprompted on a ledger that was never expanded. postgres_storage().check_schema() ignores a pending contract step. A new ledger is created contracted. A subscription generation written before contract raises ReportingUpgradeNotContractedError (a LedgerConflictError, code REPORTING_UPGRADE_NOT_CONTRACTED) naming the contract command. The published-wheel tests drive the full rolling sequence for 0.1.0b4/9.3.1 and 0.2.0b0/10.0.0b2, with and without notifications, plus rollback, concurrent old/new boots, the CLI, a 0.2.0b1-widened ledger and the expand manifest. Co-Authored-By: Claude Opus 5.5 --- .../src/adcp_reporting_storage/__init__.py | 5 + .../src/adcp_reporting_storage/ledger/pg.py | 33 +- .../ledger/reporting_ledger.sql | 29 +- .../reporting_ledger_account_generations.sql | 76 ++- .../ledger/reporting_ledger_contract.sql | 168 +++++ .../reporting_ledger_reconciliation.sql | 385 ++++++++++++ .../reporting_status_checkpoint_offerings.sql | 280 +++++++-- .../materializer/schema.py | 5 +- ...002_reporting_subscription_generations.sql | 11 +- ..._reporting_status_checkpoint_offerings.sql | 10 +- ...007_reporting_contract_generation_keys.sql | 19 + .../migrations/__init__.py | 154 ++++- .../adcp_reporting_storage/migrations/cli.py | 66 +- .../adcp_reporting_storage/outbox/_schema.py | 52 +- .../outbox/required_expand_schema.json | 154 +++++ .../outbox/status_schema.py | 12 +- .../production/schema.py | 4 +- .../projection/schema.py | 4 +- .../src/adcp_reporting_storage/storage.py | 10 +- .../conformance/reporting/_expand_manifest.py | 63 ++ .../reporting/_published_ledger_seed.py | 147 ++++- .../test_reporting_storage_migrations_pg.py | 89 ++- ..._reporting_storage_published_upgrade_pg.py | 572 +++++++++++++++++- .../test_reporting_subscription_ledger_pg.py | 60 +- tests/fixtures/public_api_snapshot.json | 1 + tests/test_reporting_storage_facade.py | 73 ++- 26 files changed, 2272 insertions(+), 210 deletions(-) create mode 100644 packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger_contract.sql create mode 100644 packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0007_reporting_contract_generation_keys.sql create mode 100644 packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/required_expand_schema.json create mode 100644 tests/conformance/reporting/_expand_manifest.py diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/__init__.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/__init__.py index 1b8bad7eca..5e7a7f8660 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/__init__.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/__init__.py @@ -196,6 +196,9 @@ from adcp_reporting_storage.migration import ( require_owned_schema_or_empty as require_owned_schema_or_empty, ) + from adcp_reporting_storage.migrations import ( + ReportingUpgradeNotContractedError as ReportingUpgradeNotContractedError, + ) from adcp_reporting_storage.outbox.pg import PgReportingOutbox as PgReportingOutbox from adcp_reporting_storage.outbox.status_activity_pg import ( PgReportingActivityUnionStore as PgReportingActivityUnionStore, @@ -270,6 +273,8 @@ # Caller-ownership migration "ReportingOwnershipBackfill": ("migration", "ReportingOwnershipBackfill"), "ReportingOwnershipMigrationError": ("migration", "ReportingOwnershipMigrationError"), + # Expand/contract upgrade of an earlier release's ledger (#1856) + "ReportingUpgradeNotContractedError": ("migrations", "ReportingUpgradeNotContractedError"), "backfill_legacy_reporting": ("migration", "backfill_legacy_reporting"), "legacy_generation_digest": ("migration", "legacy_generation_digest"), "migrate_legacy_reporting": ("migration", "migrate_legacy_reporting"), diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index 3047418979..9a58e82cb2 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -161,6 +161,7 @@ ReportingRowStorageV1, ReportingRowStoreError, ) +from adcp_reporting_storage.migrations import upgrade_not_contracted_error if TYPE_CHECKING: from psycopg_pool import AsyncConnectionPool @@ -511,12 +512,15 @@ async def create_schema(self) -> None: yet recorded in ``adcp_schema_migrations``, then this store's optional DDL (change feed, row storage), all in one transaction under the reporting schema lock. A ledger created by adcp-reporting-storage - 0.1.0b4 or 0.2.0b0 is upgraded in place: ``offering_id`` is added with - ``''`` for every existing account-level generation, and the applied - versions are recorded, so ``python -m adcp migrate --status`` reads - them. Re-running is a no-op. The later migrations (``0004`` onward: subscription - state, inline storage, status checkpoint offerings) are applied and - recorded by ``python -m adcp migrate``, or by the stores that own them. + 0.1.0b4 or 0.2.0b0 is expanded: ``offering_id`` is added with ``''`` + for every existing account-level generation, beside the keys that + release uses, so its processes keep running through a rolling deploy. + The applied versions are recorded, so ``python -m adcp migrate + --status`` reads them. Re-running is a no-op. A boot never applies the + contract step (``python -m adcp migrate --contract``). The later + migrations (``0004`` onward: subscription state, inline storage, status + checkpoint offerings, the contract step) are applied and recorded by + ``python -m adcp migrate``, or by the stores that own them. The bootstrap takes a transaction-scoped schema lock before any DDL. Keep the migration in that transaction, including with an autocommit @@ -1057,6 +1061,17 @@ async def _lock_account(connection: Any, account_id: str) -> None: # -- configurations --------------------------------------------------- + @staticmethod + async def _insert_configuration(connection: Any, query: str, params: tuple[Any, ...]) -> Any: + try: + return await connection.execute(query, params) + except Exception as error: + # An expanded ledger refuses a subscription generation until contract. + not_contracted = upgrade_not_contracted_error(error) + if not_contracted is not None: + raise not_contracted from error + raise + async def put_configuration(self, configuration: ReportingConfiguration) -> None: reject_reserved_authoritative_party(configuration) key = configuration.generation_key @@ -1064,7 +1079,8 @@ async def put_configuration(self, configuration: ReportingConfiguration) -> None digest = _fingerprint(payload) async with self._connection() as connection, connection.transaction(): await self._lock_account(connection, key.account_id) - inserted_cursor = await connection.execute( + inserted_cursor = await self._insert_configuration( + connection, "INSERT INTO reporting_configurations" " (delivery_config_id, delivery_config_version, account_id," " report_definition_id, reporting_profile, feed_purpose, required_finality," @@ -3627,6 +3643,9 @@ def _translate_integrity_error(error: Exception) -> LedgerConflictError: Matching the index name rather than the driver's prose: the message text varies by Postgres version and locale, the index name does not. """ + not_contracted = upgrade_not_contracted_error(error) + if not_contracted is not None: + return not_contracted text = str(error) if "reporting_revisions_one_official" in text: return LedgerConflictError( diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger.sql index cb57362cc1..af582bd717 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger.sql +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger.sql @@ -51,7 +51,12 @@ CREATE TABLE IF NOT EXISTS reporting_configurations ( -- expiry instead of wedging the period forever. lease_worker_id TEXT COLLATE "C", lease_expires_at TIMESTAMPTZ, - PRIMARY KEY (account_id, consumer_id, delivery_config_id, delivery_config_version) + -- '' for an account-level generation; a reporting subscription keeps one + -- generation per resolved offering of a version. A new ledger is created + -- on the widened key; reporting_ledger_account_generations.sql upgrades + -- one created by an earlier release. + offering_id TEXT COLLATE "C" NOT NULL DEFAULT '', + PRIMARY KEY (account_id, consumer_id, delivery_config_id, delivery_config_version, offering_id) ); CREATE INDEX IF NOT EXISTS reporting_configurations_account_idx @@ -129,15 +134,28 @@ CREATE TABLE IF NOT EXISTS reporting_obligations ( created_at TIMESTAMPTZ NOT NULL, -- NULL is unknown legacy currency, never an implicit USD default. The -- currency migration installs validation and immutability on all schemas. - currency TEXT COLLATE "C" + currency TEXT COLLATE "C", + offering_id TEXT COLLATE "C" NOT NULL DEFAULT '' ); -- One obligation per logical period. Without this, two workers racing a period -- close could commit two obligations and a seller could quietly publish twice -- and pick a winner. -CREATE UNIQUE INDEX IF NOT EXISTS reporting_obligations_period_key - ON reporting_obligations - (account_id, consumer_id, delivery_config_id, delivery_config_version, period_start, period_end); +-- +-- A new ledger keys it by offering_id too. An earlier release's ledger keeps +-- its six-column key here (PostgreSQL resolves the columns of CREATE INDEX IF +-- NOT EXISTS before the name): reporting_ledger_account_generations.sql +-- upgrades it. +DO $obligation_period_key$ +BEGIN + IF to_regclass(quote_ident(current_schema()) || '.reporting_obligations_period_key') IS NULL THEN + CREATE UNIQUE INDEX reporting_obligations_period_key + ON reporting_obligations + (account_id, consumer_id, delivery_config_id, delivery_config_version, offering_id, + period_start, period_end); + END IF; +END +$obligation_period_key$; CREATE INDEX IF NOT EXISTS reporting_obligations_account_idx ON reporting_obligations (account_id, period_end DESC); @@ -274,6 +292,7 @@ CREATE TABLE IF NOT EXISTS reporting_consumer_statuses ( seller_ledger_as_of TIMESTAMPTZ, superseded BOOLEAN NOT NULL DEFAULT FALSE, content_sha256 TEXT COLLATE "C" NOT NULL, + offering_id TEXT COLLATE "C" NOT NULL DEFAULT '', PRIMARY KEY (account_id, consumer_id, reporting_status_id) ); diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger_account_generations.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger_account_generations.sql index 69e23bff58..a2503bd4fb 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger_account_generations.sql +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger_account_generations.sql @@ -6,16 +6,34 @@ -- AdCP 3.3 reporting subscriptions keep one generation per resolved offering -- of a subscription version, so the owned key is (account_id, consumer_id, -- delivery_config_id, delivery_config_version, offering_id), with --- offering_id '' for every account-level generation. An owned schema still on --- the four-column key is widened here, losslessly and under the schema lock: --- existing rows keep offering_id '' and their uniqueness. This is a --- maintenance upgrade: stop every reporting worker first, because workers of --- earlier releases refuse the widened schema. A foreign key onto the key that --- the SDK does not own refuses the upgrade instead of being dropped. +-- offering_id '' for every account-level generation. reporting_ledger.sql +-- creates a new ledger on that key, so this file changes nothing there. +-- +-- A ledger created by an earlier release (adcp-reporting-storage 0.1.x or +-- 0.2.0b0, storage contract 1) is upgraded by EXPANDING it, so processes of +-- that release keep booting, reading and writing account-level generations +-- beside this one through a rolling deploy, and a rollback keeps working: +-- +-- * everything the earlier release needs keeps its name and definition: the +-- four-column reporting_configurations_pkey, the six-column +-- reporting_obligations_period_key and the four-column references onto the +-- configuration key; +-- * the widened objects take new names: reporting_configurations_generation_key +-- (UNIQUE on the five columns, the target of this release's ON CONFLICT and +-- of the offering-aware references) and the seven-column +-- reporting_obligations_offering_period_key; +-- * CHECK (offering_id = '') holds on the widened tables, because an earlier +-- release would read a subscription generation as an account-level one. +-- +-- Subscription generations need the CONTRACT step (migration 0007, +-- python -m adcp migrate --contract), run once every earlier-release +-- reporting process has stopped. It drops the expand-only objects and leaves +-- exactly the catalog of a new ledger. Expanding is lossless, under the schema +-- lock: existing rows keep offering_id '' and their uniqueness. A foreign key +-- onto the key that the SDK does not own refuses the upgrade. DO $account_generations$ DECLARE - primary_key_name TEXT; primary_key_columns TEXT[]; reference RECORD; BEGIN @@ -25,21 +43,21 @@ BEGIN -- while the unique index is rebuilt. Other ledger rows are untouched. LOCK TABLE reporting_configurations IN ACCESS EXCLUSIVE MODE; - SELECT pk.conname::TEXT, ARRAY( + SELECT ARRAY( SELECT attribute.attname::TEXT FROM unnest(pk.conkey) WITH ORDINALITY AS key_column(attnum, position) JOIN pg_attribute attribute ON attribute.attrelid = pk.conrelid AND attribute.attnum = key_column.attnum ORDER BY key_column.position ) - INTO primary_key_name, primary_key_columns + INTO primary_key_columns FROM pg_constraint pk WHERE pk.conrelid = 'reporting_configurations'::regclass AND pk.contype = 'p'; IF primary_key_columns = ARRAY[ 'account_id', 'consumer_id', 'delivery_config_id', 'delivery_config_version', 'offering_id' ] THEN - RETURN; + RETURN; -- contracted: a new ledger, or one already widened in place END IF; IF primary_key_columns IS DISTINCT FROM ARRAY[ 'account_id', 'consumer_id', 'delivery_config_id', 'delivery_config_version' @@ -47,8 +65,13 @@ BEGIN RAISE EXCEPTION 'Reporting caller ownership requires a stopped-worker maintenance migration' USING HINT = 'Use migrate_legacy_reporting; retain legacy state in quarantine and backfill authoritative ownership.'; END IF; + IF EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'reporting_configurations'::regclass + AND conname = 'reporting_configurations_generation_key') THEN + RETURN; -- expanded and not yet contracted + END IF; - -- Only the SDK's own references are re-pointed at the widened key. + -- Only the SDK's own tables reference the key. FOR reference IN SELECT referencing.relname::TEXT AS table_name FROM pg_constraint fk @@ -66,35 +89,30 @@ BEGIN END IF; END LOOP; LOCK TABLE reporting_obligations, reporting_consumer_statuses IN ACCESS EXCLUSIVE MODE; - FOR reference IN - SELECT fk.conrelid::regclass AS table_name, fk.conname::TEXT AS constraint_name - FROM pg_constraint fk - WHERE fk.contype = 'f' AND fk.confrelid = 'reporting_configurations'::regclass - LOOP - EXECUTE format( - 'ALTER TABLE %s DROP CONSTRAINT %I', reference.table_name, reference.constraint_name - ); - END LOOP; - ALTER TABLE reporting_configurations ADD COLUMN offering_id TEXT COLLATE "C" NOT NULL DEFAULT ''; - EXECUTE format('ALTER TABLE reporting_configurations DROP CONSTRAINT %I', primary_key_name); - ALTER TABLE reporting_configurations ADD CONSTRAINT reporting_configurations_pkey - PRIMARY KEY (account_id, consumer_id, delivery_config_id, delivery_config_version, offering_id); + ALTER TABLE reporting_configurations + ADD COLUMN offering_id TEXT COLLATE "C" NOT NULL DEFAULT '', + ADD CONSTRAINT reporting_configurations_generation_key + UNIQUE (account_id, consumer_id, delivery_config_id, delivery_config_version, offering_id), + ADD CONSTRAINT reporting_configurations_uncontracted CHECK (offering_id = ''); - ALTER TABLE reporting_obligations ADD COLUMN offering_id TEXT COLLATE "C" NOT NULL DEFAULT ''; - DROP INDEX reporting_obligations_period_key; - CREATE UNIQUE INDEX reporting_obligations_period_key + ALTER TABLE reporting_obligations + ADD COLUMN offering_id TEXT COLLATE "C" NOT NULL DEFAULT '', + ADD CONSTRAINT reporting_obligations_uncontracted CHECK (offering_id = ''); + CREATE UNIQUE INDEX reporting_obligations_offering_period_key ON reporting_obligations (account_id, consumer_id, delivery_config_id, delivery_config_version, offering_id, period_start, period_end); ALTER TABLE reporting_consumer_statuses - ADD COLUMN offering_id TEXT COLLATE "C" NOT NULL DEFAULT ''; + ADD COLUMN offering_id TEXT COLLATE "C" NOT NULL DEFAULT '', + ADD CONSTRAINT reporting_consumer_statuses_uncontracted CHECK (offering_id = ''); -- Managed delivery records subscription generations under their offering -- (reconciliation records, materializer discovery and status checkpoints). -- Production generations serve account-level generations only; a - -- subscription generation is refused by their CHECK. + -- subscription generation is refused by their CHECK. Each table keeps its + -- four-column reference beside the offering-aware one until contract. IF to_regclass(quote_ident(current_schema()) || '.reporting_reconciliation_records') IS NOT NULL THEN ALTER TABLE reporting_reconciliation_records ADD COLUMN offering_id TEXT COLLATE "C" NOT NULL DEFAULT '', diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger_contract.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger_contract.sql new file mode 100644 index 0000000000..d1f756cd3b --- /dev/null +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger_contract.sql @@ -0,0 +1,168 @@ +-- CONTRACT the generation keys of a ledger that +-- reporting_ledger_account_generations.sql and +-- reporting_status_checkpoint_offerings.sql EXPANDED. +-- +-- An expanded ledger keeps everything adcp-reporting-storage 0.1.x/0.2.0b0 +-- (storage contract 1) needs under its original names, so processes of that +-- release can keep running and restarting through a rolling deploy. This file +-- removes it, after which the ledger is exactly a new ledger's catalog: +-- +-- * reporting_configurations: drops the four-column primary key, the +-- references onto it and CHECK (offering_id = ''); the five-column +-- reporting_configurations_generation_key becomes reporting_configurations_pkey +-- and the offering-aware references are re-created onto it, by name; +-- * reporting_obligations: drops the six-column period key and CHECK +-- (offering_id = ''); reporting_obligations_offering_period_key takes the name +-- reporting_obligations_period_key; +-- * reporting_consumer_statuses: drops CHECK (offering_id = ''); +-- * status checkpoints (where installed): the seven-column +-- reporting_status_scope_checkpoints_offering_key becomes the primary key, each +-- status event table's widened reference and key take the names of the +-- six-column ones they replace. +-- +-- This is a STOPPED-WORKER step: earlier-release processes cannot write the +-- contracted keys, and swapping primary keys takes exclusive locks. Apply it +-- with python -m adcp migrate --contract once every earlier-release reporting +-- process has stopped. It changes nothing on a contracted ledger. Migration +-- 0007 then re-runs reporting_ledger_reconciliation.sql and +-- reporting_status_checkpoint_offerings.sql, which restore this release's +-- function bodies. + +DO $contract$ +DECLARE + reference RECORD; + restored TEXT[] := ARRAY[]::TEXT[]; + canonical JSONB := '{}'::JSONB; + item TEXT; +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + + -- (Nested tests: PostgreSQL resolves a ::regclass literal while planning.) + IF to_regclass(quote_ident(current_schema()) || '.reporting_configurations') IS NULL THEN + RETURN; + END IF; + IF EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'reporting_configurations'::regclass + AND conname = 'reporting_configurations_generation_key') THEN + LOCK TABLE reporting_configurations, reporting_obligations, reporting_consumer_statuses + IN ACCESS EXCLUSIVE MODE; + -- Every reference onto the configuration key goes: the four-column ones + -- for good, the offering-aware ones to be re-created onto the new key. + FOR reference IN + SELECT fk.conrelid::regclass::TEXT AS table_name, fk.conname::TEXT AS constraint_name, + pg_get_constraintdef(fk.oid) AS definition, array_length(fk.conkey, 1) AS width + FROM pg_constraint fk + WHERE fk.contype = 'f' AND fk.confrelid = 'reporting_configurations'::regclass + ORDER BY 1, 2 + LOOP + IF reference.table_name NOT IN ( + 'reporting_reconciliation_records', + 'reporting_materializer_discovery', + 'reporting_status_scope_checkpoints', + 'reporting_production_generations' + ) THEN + RAISE EXCEPTION 'Contracting the reporting generation key requires a stopped-worker maintenance migration' + USING HINT = 'A foreign key outside the SDK references reporting_configurations; drop it first.'; + END IF; + EXECUTE format('LOCK TABLE %s IN ACCESS EXCLUSIVE MODE', reference.table_name); + EXECUTE format('ALTER TABLE %s DROP CONSTRAINT %I', + reference.table_name, reference.constraint_name); + IF reference.width = 5 THEN + restored := restored || format('ALTER TABLE %s ADD CONSTRAINT %I %s', + reference.table_name, reference.constraint_name, reference.definition); + END IF; + END LOOP; + ALTER TABLE reporting_configurations + DROP CONSTRAINT reporting_configurations_uncontracted, + DROP CONSTRAINT reporting_configurations_pkey, + DROP CONSTRAINT reporting_configurations_generation_key, + ADD CONSTRAINT reporting_configurations_pkey PRIMARY KEY + (account_id, consumer_id, delivery_config_id, delivery_config_version, offering_id); + FOREACH item IN ARRAY restored LOOP + EXECUTE item; + END LOOP; + + ALTER TABLE reporting_obligations DROP CONSTRAINT reporting_obligations_uncontracted; + DROP INDEX reporting_obligations_period_key; + ALTER INDEX reporting_obligations_offering_period_key RENAME TO reporting_obligations_period_key; + ALTER TABLE reporting_consumer_statuses + DROP CONSTRAINT reporting_consumer_statuses_uncontracted; + END IF; + + IF to_regclass(quote_ident(current_schema()) || '.reporting_status_scope_checkpoints') IS NULL THEN + RETURN; + END IF; + IF EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'reporting_status_scope_checkpoints'::regclass + AND conname = 'reporting_status_scope_checkpoints_offering_key') THEN + LOCK TABLE reporting_status_scope_checkpoints IN ACCESS EXCLUSIVE MODE; + restored := ARRAY[]::TEXT[]; + -- A table's six-column reference names the widened one that replaces it. + FOR reference IN + SELECT fk.conrelid::regclass::TEXT AS table_name, fk.conname::TEXT AS constraint_name + FROM pg_constraint fk + WHERE fk.contype = 'f' AND fk.confrelid = 'reporting_status_scope_checkpoints'::regclass + AND array_length(fk.conkey, 1) = 6 + LOOP + canonical := canonical || jsonb_build_object(reference.table_name, reference.constraint_name); + END LOOP; + FOR reference IN + SELECT fk.conrelid::regclass::TEXT AS table_name, fk.conname::TEXT AS constraint_name, + pg_get_constraintdef(fk.oid) AS definition, array_length(fk.conkey, 1) AS width + FROM pg_constraint fk + WHERE fk.contype = 'f' AND fk.confrelid = 'reporting_status_scope_checkpoints'::regclass + ORDER BY 1, 2 + LOOP + IF reference.table_name NOT IN ( + 'reporting_status_notification_events', + 'reporting_projection_notification_events' + ) THEN + RAISE EXCEPTION 'Contracting the status checkpoint key requires a stopped-worker maintenance migration' + USING HINT = 'A foreign key outside the SDK references reporting_status_scope_checkpoints; drop it first.'; + END IF; + EXECUTE format('LOCK TABLE %s IN ACCESS EXCLUSIVE MODE', reference.table_name); + EXECUTE format('ALTER TABLE %s DROP CONSTRAINT %I', + reference.table_name, reference.constraint_name); + IF reference.width = 7 THEN + restored := restored || format('ALTER TABLE %s ADD CONSTRAINT %I %s', + reference.table_name, + coalesce(canonical ->> reference.table_name, reference.constraint_name), + reference.definition); + END IF; + END LOOP; + ALTER TABLE reporting_status_scope_checkpoints + DROP CONSTRAINT reporting_status_scope_checkpoints_pkey, + DROP CONSTRAINT reporting_status_scope_checkpoints_offering_key, + ADD CONSTRAINT reporting_status_scope_checkpoints_pkey PRIMARY KEY + (account_id, consumer_namespace, delivery_config_id, version, + scope_kind, obligation_namespace, offering_id); + FOREACH item IN ARRAY restored LOOP + EXECUTE item; + END LOOP; + + -- The widened event key takes the name of the seven-column one. + FOR reference IN + SELECT widened.conrelid::regclass::TEXT AS table_name, + widened.conname::TEXT AS widened_name, earlier.conname::TEXT AS earlier_name + FROM pg_constraint widened + JOIN pg_constraint earlier ON earlier.conrelid = widened.conrelid + AND earlier.contype = 'u' AND earlier.conname <> widened.conname + WHERE widened.contype = 'u' + AND widened.conname = widened.conrelid::regclass::TEXT || '_offering_key' + AND ARRAY( + SELECT attribute.attname::TEXT + FROM unnest(earlier.conkey) WITH ORDINALITY AS key_column(attnum, position) + JOIN pg_attribute attribute ON attribute.attrelid = earlier.conrelid + AND attribute.attnum = key_column.attnum + ORDER BY key_column.position) = ARRAY[ + 'account_id', 'consumer_namespace', 'delivery_config_id', 'version', + 'scope_kind', 'obligation_namespace', 'cause_generation'] + LOOP + EXECUTE format('ALTER TABLE %s DROP CONSTRAINT %I', + reference.table_name, reference.earlier_name); + EXECUTE format('ALTER TABLE %s RENAME CONSTRAINT %I TO %I', + reference.table_name, reference.widened_name, reference.earlier_name); + END LOOP; + END IF; +END +$contract$; diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger_reconciliation.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger_reconciliation.sql index 3641a36a73..1a1a3fa7e5 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger_reconciliation.sql +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger_reconciliation.sql @@ -964,3 +964,388 @@ BEGIN END IF; END $reconciliation$; + +-- While a ledger upgraded from adcp-reporting-storage 0.1.x/0.2.0b0 is +-- uncontracted (reporting_ledger_account_generations.sql expanded it and the +-- contract step, migration 0007, has not run), processes of that release may +-- still be running and restarting beside this one. Each of their boots +-- re-creates the three reconciliation functions above with their own bodies, +-- and both releases' schema validators compare function bodies. So while +-- uncontracted, this release installs exactly the earlier release's bodies +-- too, and neither release's boot changes what the other validates. +-- +-- The earlier bodies are correct for this release's writes while +-- uncontracted: every generation is account-level (CHECK (offering_id = '') +-- on the configuration key), so offering_id is '' wherever they omit it, and +-- an account-level record's payload carries neither offering_id nor +-- path_template. The text below is the earlier release's, byte for byte; +-- the contract step re-runs this file, which then keeps the bodies above. +DO $uncontracted_reconciliation$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'reporting_configurations'::regclass + AND conname = 'reporting_configurations_generation_key') THEN + RETURN; + END IF; + CREATE OR REPLACE FUNCTION reporting_reconciliation_evidence(r reporting_reconciliation_records) + RETURNS VOID LANGUAGE plpgsql AS $function$ + DECLARE + binding JSONB; + delivery JSONB; + outcome JSONB; + expected_rows BIGINT; + expected_totals JSONB; + expected_digest JSONB; + observed_digest JSONB; + adjustment reporting_adjustments; + BEGIN + IF r.record_kind NOT IN ('materialization', 'revision_receipt', 'adjustment_receipt') + OR (r.record_kind = 'materialization' + AND r.payload->>'status' NOT IN ('available', 'delivered')) + OR (r.receipt_status IS NOT NULL AND r.receipt_status <> 'accepted') THEN + RETURN; + END IF; + SELECT v.row_count, v.managed_control_totals, v.canonical_content_digest + INTO expected_rows, expected_totals, expected_digest + FROM reporting_revisions v + WHERE (v.account_id, v.reporting_obligation_id, v.reporting_revision_id) + = (r.account_id, r.reporting_obligation_id, r.reporting_revision_id); + IF r.record_kind = 'adjustment_receipt' THEN + SELECT a.* INTO adjustment FROM reporting_adjustments a + WHERE (a.account_id, a.adjusts_reporting_revision_id, a.reporting_adjustment_id) + = (r.account_id, r.reporting_revision_id, r.reporting_adjustment_id); + IF adjustment.reporting_adjustment_id IS NULL + OR adjustment.managed_control_total_deltas IS NULL + OR jsonb_array_length(adjustment.managed_control_total_deltas) = 0 + OR r.payload->>'observed_adjustment_sha256' IS DISTINCT FROM reporting_payload_sha256( + jsonb_build_object( + 'reporting_adjustment_id', adjustment.reporting_adjustment_id, + 'adjusts_reporting_revision_id', adjustment.adjusts_reporting_revision_id, + 'reason_code', adjustment.reason_code, + 'accounting_period', jsonb_build_object( + 'start', reporting_iso_utc(adjustment.accounting_period_start), + 'end', reporting_iso_utc(adjustment.accounting_period_end)), + 'control_total_deltas', adjustment.managed_control_total_deltas, + 'correction_observed_at', reporting_iso_utc(adjustment.correction_observed_at), + 'created_at', reporting_iso_utc(adjustment.created_at)) + || CASE WHEN adjustment.reason_detail IS NOT NULL + THEN jsonb_build_object('reason_detail', adjustment.reason_detail) + ELSE '{}'::jsonb END) THEN + RAISE EXCEPTION 'reporting adjustment acceptance is inconsistent' USING ERRCODE = '23514'; + END IF; + RETURN; + END IF; + IF expected_totals IS NULL OR expected_rows IS NULL THEN + RAISE EXCEPTION 'reporting revision evidence is unavailable' USING ERRCODE = '23514'; + END IF; + IF r.record_kind = 'revision_receipt' THEN + SELECT m.payload INTO outcome FROM reporting_reconciliation_records m + WHERE (m.account_id, m.consumer_id, m.delivery_config_id, m.delivery_config_version, + m.reporting_obligation_id, m.reporting_materialization_id, m.reporting_revision_id) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version, + r.reporting_obligation_id, r.reporting_materialization_id, r.reporting_revision_id) + AND m.record_kind = 'materialization' + AND m.payload->>'status' IN ('available', 'delivered'); + observed_digest := reporting_wire_digest(r.payload->'observed_canonical_content_digest'); + expected_digest := reporting_wire_digest(expected_digest); + IF outcome IS NULL + OR r.payload->>'verification_profile' + IS DISTINCT FROM outcome#>>'{verification,verification_profile}' + OR (r.payload->>'observed_row_count')::bigint IS DISTINCT FROM expected_rows + OR reporting_sorted_totals(r.payload->'observed_control_totals') + IS DISTINCT FROM reporting_sorted_totals(outcome#>'{verification,control_totals}') + OR (observed_digest IS NOT NULL AND observed_digest IS DISTINCT FROM expected_digest) + OR (nullif(r.payload->>'observed_manifest_sha256', '') IS NOT NULL + AND r.payload->>'observed_manifest_sha256' + IS DISTINCT FROM outcome#>>'{resource,manifest_sha256}') + OR (nullif(r.payload->>'observed_native_version_ref', '') IS NOT NULL + AND r.payload->>'observed_native_version_ref' + IS DISTINCT FROM outcome#>>'{resource,native_version_ref}') + OR (r.payload->>'verification_profile' = 'canonical_digest' + AND (observed_digest IS NULL OR observed_digest IS DISTINCT FROM expected_digest)) + OR (r.payload->>'verification_profile' = 'manifest_checksums' + AND r.payload->>'observed_manifest_sha256' + IS DISTINCT FROM outcome#>>'{resource,manifest_sha256}') + OR (r.payload->>'verification_profile' = 'native_commit' + AND r.payload->>'observed_native_version_ref' + IS DISTINCT FROM outcome#>>'{resource,native_version_ref}') + OR (r.payload->>'observed_at')::timestamptz < (outcome->>'completed_at')::timestamptz + OR (r.payload->>'observed_at')::timestamptz + >= (outcome#>>'{resource,expires_at}')::timestamptz THEN + RAISE EXCEPTION 'reporting receipt acceptance is inconsistent' USING ERRCODE = '23514'; + END IF; + RETURN; + END IF; + SELECT b.payload INTO binding FROM reporting_reconciliation_records b + WHERE (b.account_id, b.consumer_id, b.delivery_config_id, b.delivery_config_version) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version) + AND b.record_kind = 'destination_binding'; + SELECT d.payload INTO delivery FROM reporting_reconciliation_records d + WHERE (d.account_id, d.consumer_id, d.delivery_config_id, d.delivery_config_version, + d.reporting_obligation_id) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version, + r.reporting_obligation_id) + AND d.record_kind = 'obligation_delivery'; + observed_digest := reporting_wire_digest(r.payload#>'{verification,canonical_content_digest}'); + expected_digest := reporting_wire_digest(expected_digest); + IF binding IS NULL OR delivery IS NULL + OR (r.payload#>>'{verification,row_count}')::bigint IS DISTINCT FROM expected_rows + OR reporting_sorted_totals(r.payload#>'{verification,control_totals}') + IS DISTINCT FROM reporting_sorted_totals(expected_totals) + OR (observed_digest IS NOT NULL AND observed_digest IS DISTINCT FROM expected_digest) + OR (r.payload#>>'{verification,verification_profile}' = 'canonical_digest' + AND (observed_digest IS NULL OR observed_digest IS DISTINCT FROM expected_digest)) + OR r.payload#>>'{verification,verified_at}' IS DISTINCT FROM r.payload->>'completed_at' + OR nullif(r.payload#>'{verification,verified_format}', 'null'::jsonb) + IS DISTINCT FROM nullif(binding->'format', 'null'::jsonb) + OR r.payload#>'{resource,reader_compatibility}' + IS DISTINCT FROM coalesce(binding->'reader_compatibility', '[]'::jsonb) + OR r.payload#>>'{resource,kind}' IS DISTINCT FROM (CASE binding->>'method' + WHEN 'file_transfer' THEN 'manifest' WHEN 'dataset_share' THEN 'dataset' + ELSE 'warehouse_relation' END) + OR (binding->>'method' = 'dataset_share' + AND r.payload#>>'{verification,verification_path}' <> 'representative_consumer') + OR (binding->>'method' = 'warehouse_materialization' + AND (r.payload#>>'{verification,verification_path}' <> 'destination' + OR r.payload->>'status' <> 'delivered')) + OR (r.payload->>'status' = 'delivered' + AND r.payload#>>'{verification,verification_path}' <> 'destination') + OR (binding->>'method' = 'file_transfer' AND ( + jsonb_array_length(coalesce(r.payload#>'{resource,object_refs}', '[]'::jsonb)) = 0 + OR jsonb_array_length( + coalesce(r.payload#>'{verification,physical_checksums}', '[]'::jsonb)) = 0)) + OR (r.payload#>>'{resource,expires_at}')::timestamptz < greatest( + (delivery->>'resource_retained_until')::timestamptz, + (r.payload->>'completed_at')::timestamptz + + ((binding->>'resource_retention_days') || ' days')::interval) THEN + RAISE EXCEPTION 'reporting materialization evidence is inconsistent' USING ERRCODE = '23514'; + END IF; + -- Every checksum names a retained object, whatever the method; file + -- transfer must additionally cover all of them. + IF EXISTS ( + SELECT 1 FROM jsonb_array_elements( + coalesce(r.payload#>'{verification,physical_checksums}', '[]'::jsonb)) AS c + WHERE NOT coalesce(r.payload#>'{resource,object_refs}', '[]'::jsonb) + @> jsonb_build_array(c->'object_ref')) + OR (binding->>'method' = 'file_transfer' AND reporting_sorted_strings( + (SELECT jsonb_agg(c->'object_ref') + FROM jsonb_array_elements(r.payload#>'{verification,physical_checksums}') AS c)) + IS DISTINCT FROM reporting_sorted_strings(r.payload#>'{resource,object_refs}')) THEN + RAISE EXCEPTION 'reporting physical checksum binding is inconsistent' USING ERRCODE = '23514'; + END IF; + END + $function$; + CREATE OR REPLACE FUNCTION reporting_reconciliation_validate(r reporting_reconciliation_records) + RETURNS VOID LANGUAGE plpgsql AS $function$ + DECLARE + generation JSONB; + identity TEXT; + chain TEXT; + expected_namespace TEXT; + parent_kind TEXT; + BEGIN + -- The retained fingerprint must be a fact about the stored bytes, and the + -- payload must be closed. Without both, ordinary direct SQL can persist a + -- provider blob or credential under an attacker-chosen digest and leave the + -- principal's whole feed unreadable. Never echo the offending value. + IF r.content_sha256 IS DISTINCT FROM reporting_payload_sha256(r.payload) + OR EXISTS (SELECT 1 FROM reporting_payload_keys(r.payload) AS field + WHERE field <> ALL (ARRAY[ + 'account_id', 'adjusts_reporting_revision_id', 'algorithm', 'attempt', + 'canonical_content_digest', 'canonicalization_id', 'canonicalization_sha256', + 'canonicalization_uri', 'check_id', 'checked_at', 'completed_at', + 'consumer_commit_ref', 'consumer_id', 'control_totals', 'created_at', 'currency', + 'delivery_config_id', 'delivery_config_version', 'destination_ref', 'expires_at', + 'failure_code', 'feed_purpose', 'format', 'generation_key', 'immutability', 'kind', + 'location', 'manifest_sha256', 'method', 'name', 'native_observed_through', + 'native_version_ref', 'object_ref', 'object_refs', 'observed_adjustment_sha256', + 'observed_at', 'observed_canonical_content_digest', 'observed_control_totals', + 'observed_manifest_sha256', 'observed_native_version_ref', 'observed_row_count', + 'physical_checksums', 'reader_compatibility', 'received_at', 'reconciliation_mode', + 'rejection_codes', 'reporting_adjustment_id', 'reporting_materialization_id', + 'reporting_obligation_id', 'reporting_receipt_id', 'reporting_revision_id', 'resource', + 'resource_ref', 'resource_retained_until', 'resource_retention_days', 'row_count', + 'scope', 'state', 'status', 'success_status', 'supersedes_reporting_receipt_id', + 'transport', 'trusted_binding_ref', 'unit', 'value', 'value_type', 'verification', + 'verification_path', 'verification_profile', 'verified_at', 'verified_format'])) THEN + RAISE EXCEPTION 'reporting payload is not closed retained evidence' USING ERRCODE = '23514'; + END IF; + generation := jsonb_build_object('account_id', r.account_id, 'consumer_id', r.consumer_id, + 'delivery_config_id', r.delivery_config_id, + 'delivery_config_version', r.delivery_config_version); + IF NOT EXISTS (SELECT 1 FROM reporting_configurations c + WHERE (c.account_id, c.consumer_id, c.delivery_config_id, c.delivery_config_version) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version)) + OR (r.reporting_obligation_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM reporting_obligations o + WHERE (o.account_id, o.consumer_id, o.delivery_config_id, o.delivery_config_version, o.reporting_obligation_id) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version, r.reporting_obligation_id))) + OR (r.reporting_revision_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM reporting_revisions v + WHERE (v.account_id, v.reporting_obligation_id, v.reporting_revision_id) + = (r.account_id, r.reporting_obligation_id, r.reporting_revision_id))) + OR (r.reporting_adjustment_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM reporting_adjustments a + WHERE (a.account_id, a.adjusts_reporting_revision_id, a.reporting_adjustment_id) + = (r.account_id, r.reporting_revision_id, r.reporting_adjustment_id))) THEN + RAISE EXCEPTION 'reporting exact publication graph is unavailable' USING ERRCODE = '23514'; + END IF; + expected_namespace := CASE WHEN r.receipt_status IS NOT NULL THEN 'receipt' ELSE r.record_kind END; + identity := CASE r.record_kind + WHEN 'destination_binding' THEN reporting_identity_sha256( + '{"account_id":' || to_json(r.account_id)::text || ',"consumer_id":' || to_json(r.consumer_id)::text || ',"delivery_config_id":' || + to_json(r.delivery_config_id)::text || ',"delivery_config_version":' || + r.delivery_config_version::text || '}') + WHEN 'obligation_delivery' THEN r.reporting_obligation_id + WHEN 'materialization_check' THEN r.payload->>'check_id' + WHEN 'revision_receipt' THEN r.payload->>'reporting_receipt_id' + WHEN 'adjustment_receipt' THEN r.payload->>'reporting_receipt_id' + ELSE r.reporting_materialization_id END; + chain := CASE r.record_kind + WHEN 'revision_receipt' THEN reporting_identity_sha256( + '["revision",' || to_json(r.reporting_obligation_id)::text || ',' || + to_json(r.reporting_revision_id)::text || ']') + WHEN 'adjustment_receipt' THEN reporting_identity_sha256( + '["adjustment",' || to_json(r.reporting_adjustment_id)::text || ']') END; + IF r.payload->>'kind' IS DISTINCT FROM r.record_kind + OR r.namespace IS DISTINCT FROM expected_namespace + OR r.record_id IS DISTINCT FROM identity + OR r.receipt_chain_key IS DISTINCT FROM chain + OR r.change_id IS DISTINCT FROM reporting_identity_sha256( + '[' || to_json(r.account_id)::text || ',' || to_json(r.consumer_id)::text || ',' || + to_json(expected_namespace)::text || ',' || to_json(identity)::text || ']') + OR r.reporting_revision_id IS DISTINCT FROM (CASE WHEN r.record_kind = 'adjustment_receipt' + THEN r.payload->>'adjusts_reporting_revision_id' ELSE r.payload->>'reporting_revision_id' END) + OR r.reporting_materialization_id IS DISTINCT FROM r.payload->>'reporting_materialization_id' + OR r.reporting_adjustment_id IS DISTINCT FROM r.payload->>'reporting_adjustment_id' + OR coalesce(to_jsonb(r.attempt_number), 'null'::jsonb) IS DISTINCT FROM coalesce(r.payload->'attempt', 'null'::jsonb) + OR r.receipt_status IS DISTINCT FROM (CASE WHEN r.record_kind IN ('revision_receipt', 'adjustment_receipt') + THEN r.payload->>'status' END) + OR r.supersedes_receipt_id IS DISTINCT FROM r.payload->>'supersedes_reporting_receipt_id' THEN + RAISE EXCEPTION 'reporting payload identity is inconsistent' USING ERRCODE = '23514'; + END IF; + IF r.record_kind = 'destination_binding' THEN + IF r.payload->'generation_key' IS DISTINCT FROM generation + OR r.payload->>'consumer_id' IS DISTINCT FROM r.consumer_id + OR (r.payload->>'verification_profile' = 'manifest_checksums' + AND r.payload->>'method' <> 'file_transfer') THEN + RAISE EXCEPTION 'reporting destination binding is inconsistent' USING ERRCODE = '23514'; + END IF; + RETURN; + END IF; + IF r.payload->'scope' IS DISTINCT FROM jsonb_build_object('generation_key', generation, + 'consumer_id', r.consumer_id, 'reporting_obligation_id', r.reporting_obligation_id) + OR NOT EXISTS ( + SELECT 1 FROM reporting_reconciliation_records b + WHERE (b.account_id, b.consumer_id, b.delivery_config_id, b.delivery_config_version) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version) + AND b.record_kind = 'destination_binding') THEN + RAISE EXCEPTION 'reporting frozen binding is unavailable' USING ERRCODE = '23514'; + END IF; + IF r.record_kind = 'obligation_delivery' THEN RETURN; END IF; + IF NOT EXISTS ( + SELECT 1 FROM reporting_reconciliation_records d + WHERE (d.account_id, d.consumer_id, d.delivery_config_id, d.delivery_config_version, d.reporting_obligation_id) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version, r.reporting_obligation_id) + AND d.record_kind = 'obligation_delivery') THEN + RAISE EXCEPTION 'reporting obligation delivery is unavailable' USING ERRCODE = '23514'; + END IF; + parent_kind := CASE r.record_kind WHEN 'materialization' THEN 'materialization_attempt' + WHEN 'materialization_check' THEN 'materialization' WHEN 'revision_receipt' THEN 'materialization' END; + IF parent_kind IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM reporting_reconciliation_records p + WHERE (p.account_id, p.consumer_id, p.delivery_config_id, p.delivery_config_version, + p.reporting_obligation_id, p.reporting_materialization_id) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version, + r.reporting_obligation_id, r.reporting_materialization_id) + AND p.record_kind = parent_kind + AND (r.record_kind = 'materialization_check' OR p.reporting_revision_id = r.reporting_revision_id) + AND (parent_kind = 'materialization_attempt' OR p.payload->>'status' IN ('available', 'delivered'))) THEN + RAISE EXCEPTION 'reporting exact materialization graph is unavailable' USING ERRCODE = '23514'; + END IF; + IF r.record_kind = 'materialization' AND r.payload->>'status' IN ('available', 'delivered') THEN + IF EXISTS (SELECT 1 FROM reporting_reconciliation_records b + WHERE (b.account_id, b.consumer_id, b.delivery_config_id, b.delivery_config_version) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version) + AND b.record_kind = 'destination_binding' + AND (r.payload->>'status' IS DISTINCT FROM b.payload->>'success_status' + OR r.payload#>>'{verification,verification_profile}' IS DISTINCT FROM b.payload->>'verification_profile')) THEN + RAISE EXCEPTION 'reporting materialization binding is inconsistent' USING ERRCODE = '23514'; + END IF; + IF r.payload#>>'{verification,verification_profile}' = 'native_commit' AND ( + r.payload#>>'{resource,immutability}' IS DISTINCT FROM 'native_version' + OR r.payload#>>'{resource,native_version_ref}' IS NULL + OR r.payload#>>'{verification,native_version_ref}' IS DISTINCT FROM r.payload#>>'{resource,native_version_ref}' + OR r.payload#>>'{verification,native_observed_through}' IS DISTINCT FROM r.payload#>>'{verification,verification_path}' + OR coalesce(r.payload#>>'{verification,verification_path}', '') NOT IN ('representative_consumer', 'destination')) THEN + RAISE EXCEPTION 'reporting native commit is inconsistent' USING ERRCODE = '23514'; + END IF; + END IF; + IF r.record_kind = 'adjustment_receipt' AND NOT EXISTS ( + SELECT 1 FROM reporting_revisions v WHERE v.account_id = r.account_id + AND v.reporting_obligation_id = r.reporting_obligation_id + AND v.reporting_revision_id = r.reporting_revision_id + AND v.finality = 'official' AND v.finalized_at IS NOT NULL) THEN + RAISE EXCEPTION 'reporting adjustment requires its official revision' USING ERRCODE = '23514'; + END IF; + IF r.supersedes_receipt_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM reporting_reconciliation_records p + WHERE (p.account_id, p.consumer_id, p.record_kind, p.delivery_config_id, + p.delivery_config_version, p.reporting_obligation_id, p.reporting_revision_id, p.receipt_chain_key) + = (r.account_id, r.consumer_id, r.record_kind, r.delivery_config_id, + r.delivery_config_version, r.reporting_obligation_id, r.reporting_revision_id, r.receipt_chain_key) + AND p.record_id = r.supersedes_receipt_id AND p.receipt_status = 'rejected') THEN + RAISE EXCEPTION 'reporting receipt predecessor is unavailable' USING ERRCODE = '23514'; + END IF; + PERFORM reporting_reconciliation_evidence(r); + END + $function$; + CREATE OR REPLACE FUNCTION reporting_reconciliation_reference_immutable() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + DECLARE + previous JSONB := to_jsonb(OLD); + proposed JSONB := to_jsonb(NEW); + referenced BOOLEAN; + -- Period-close leasing plus the rc.3 lifecycle state that evolves over + -- one immutable generation: reporting-delivery-config-state.json moves + -- the same generation through ready -> inactive, requiring + -- deactivated_at, and its operational recovery/retention windows are + -- likewise state rather than published content. Reconciliation records + -- bind the generation key and its content, none of which these carry. + mutable TEXT[] := ARRAY['lease_worker_id', 'lease_expires_at', 'activated_at', + 'deactivated_at', 'automated_recovery_seconds', + 'status_retention_days']; + BEGIN + IF proposed = previous + OR (TG_TABLE_NAME = 'reporting_configurations' AND + proposed - mutable = previous - mutable) + OR (TG_TABLE_NAME = 'reporting_revisions' AND proposed - 'readable' = previous - 'readable') THEN + RETURN NEW; + END IF; + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || OLD.account_id)); + IF TG_TABLE_NAME = 'reporting_configurations' THEN + previous := previous - mutable; + proposed := proposed - mutable; + SELECT EXISTS (SELECT 1 FROM reporting_reconciliation_records r + WHERE r.account_id=OLD.account_id AND r.consumer_id=OLD.consumer_id AND r.delivery_config_id = OLD.delivery_config_id + AND r.delivery_config_version = OLD.delivery_config_version) INTO referenced; + ELSIF TG_TABLE_NAME = 'reporting_obligations' THEN + SELECT EXISTS (SELECT 1 FROM reporting_reconciliation_records r + WHERE r.account_id = OLD.account_id AND r.reporting_obligation_id = OLD.reporting_obligation_id) INTO referenced; + ELSIF TG_TABLE_NAME = 'reporting_revisions' THEN + previous := previous - 'readable'; + proposed := proposed - 'readable'; + SELECT EXISTS (SELECT 1 FROM reporting_reconciliation_records r + WHERE r.account_id = OLD.account_id AND r.reporting_revision_id = OLD.reporting_revision_id) INTO referenced; + ELSE + SELECT EXISTS (SELECT 1 FROM reporting_reconciliation_records r + WHERE r.account_id = OLD.account_id AND r.reporting_adjustment_id = OLD.reporting_adjustment_id) INTO referenced; + END IF; + IF referenced AND proposed IS DISTINCT FROM previous THEN + RAISE EXCEPTION 'reporting referenced publication is immutable' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END + $function$; +END +$uncontracted_reconciliation$; diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_status_checkpoint_offerings.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_status_checkpoint_offerings.sql index 33e82d1f5f..f8fa57dd0a 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_status_checkpoint_offerings.sql +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_status_checkpoint_offerings.sql @@ -17,19 +17,36 @@ -- widened key. reporting_status_notifications.sql creates the checkpoint and -- event tables on their published six-column key; this file widens them, an -- earlier release's projection event table too, and defines the checkpoint and --- event guards, which compare the offering. Constraints keep their names. +-- event guards, which compare the offering. -- --- A schema still on the six-column key is widened under the schema lock. This --- is a maintenance upgrade: stop every reporting worker first, because workers --- of earlier releases write the six-column key. A foreign key onto the --- checkpoint key that the SDK does not own refuses the upgrade instead of --- being dropped. +-- How a six-column key is widened follows the configuration key +-- (reporting_ledger_account_generations.sql): +-- +-- * While the ledger is EXPANDED and not yet contracted (it was created by +-- adcp-reporting-storage 0.1.x/0.2.0b0, whose processes may still run +-- beside this release), the checkpoint and status event tables are expanded +-- the same way: the six-column reporting_status_scope_checkpoints_pkey, the +-- six-column event reference and the seven-column event key keep their names +-- and definitions, and the widened ones are added beside them as +-- reporting_status_scope_checkpoints_offering_key, +-- reporting_status_notification_events_offering_fkey and +-- reporting_status_notification_events_offering_key. The contract step +-- (migration 0007) renames them back. The status projection's event table, +-- an extension the earlier release's status workers do not write, is widened +-- in place. +-- * Otherwise (a contracted ledger), the key is widened in place under the +-- schema lock and constraints keep their names. On a contracted ledger only +-- a status table this release has just created is still on six columns. +-- +-- A foreign key onto the checkpoint key that the SDK does not own refuses the +-- upgrade instead of being dropped. DO $checkpoint_offerings$ DECLARE key_columns TEXT[]; reference RECORD; events TEXT; + expand BOOLEAN; BEGIN PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); IF to_regclass(quote_ident(current_schema()) || '.reporting_status_scope_checkpoints') IS NULL THEN @@ -48,13 +65,41 @@ BEGIN IF key_columns IS DISTINCT FROM ARRAY[ 'account_id', 'consumer_namespace', 'delivery_config_id', 'version', - 'scope_kind', 'obligation_namespace', 'offering_id'] THEN - IF key_columns IS DISTINCT FROM ARRAY[ - 'account_id', 'consumer_namespace', 'delivery_config_id', 'version', - 'scope_kind', 'obligation_namespace'] THEN - RAISE EXCEPTION 'Keying status checkpoints by offering requires a stopped-worker maintenance migration' - USING HINT = 'reporting_status_scope_checkpoints has an unexpected primary key.'; + 'scope_kind', 'obligation_namespace', 'offering_id'] + AND key_columns IS DISTINCT FROM ARRAY[ + 'account_id', 'consumer_namespace', 'delivery_config_id', 'version', + 'scope_kind', 'obligation_namespace'] THEN + RAISE EXCEPTION 'Keying status checkpoints by offering requires a stopped-worker maintenance migration' + USING HINT = 'reporting_status_scope_checkpoints has an unexpected primary key.'; + END IF; + -- Expanded and not yet contracted: see reporting_ledger_account_generations.sql. + expand := array_length(key_columns, 1) = 6 AND EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'reporting_configurations'::regclass + AND conname = 'reporting_configurations_generation_key'); + + IF expand THEN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'reporting_status_scope_checkpoints'::regclass + AND conname = 'reporting_status_scope_checkpoints_offering_key') THEN + LOCK TABLE reporting_status_scope_checkpoints IN ACCESS EXCLUSIVE MODE; + ALTER TABLE reporting_status_scope_checkpoints + ADD CONSTRAINT reporting_status_scope_checkpoints_offering_key UNIQUE + (account_id, consumer_namespace, delivery_config_id, version, + scope_kind, obligation_namespace, offering_id); + END IF; + -- The earlier release references the four-column configuration key; + -- a checkpoint table this release created while expanded lacks it. + IF NOT EXISTS (SELECT 1 FROM pg_constraint fk + WHERE fk.conrelid = 'reporting_status_scope_checkpoints'::regclass + AND fk.contype = 'f' AND fk.confrelid = 'reporting_configurations'::regclass + AND array_length(fk.conkey, 1) = 4) THEN + -- PostgreSQL names it as the earlier release's CREATE TABLE did. + ALTER TABLE reporting_status_scope_checkpoints + ADD FOREIGN KEY (account_id, consumer_namespace, delivery_config_id, version) + REFERENCES reporting_configurations + (account_id, consumer_id, delivery_config_id, delivery_config_version); END IF; + ELSIF array_length(key_columns, 1) = 6 THEN LOCK TABLE reporting_status_scope_checkpoints IN ACCESS EXCLUSIVE MODE; -- Only the SDK's own status event tables reference the key; each is -- re-pointed at the widened key below. @@ -91,58 +136,90 @@ BEGIN FOREACH events IN ARRAY ARRAY[ 'reporting_status_notification_events', 'reporting_projection_notification_events'] LOOP - CONTINUE WHEN to_regclass(events) IS NULL; - CONTINUE WHEN EXISTS (SELECT 1 FROM pg_constraint fk + CONTINUE WHEN to_regclass(quote_ident(current_schema()) || '.' || events) IS NULL; + IF expand AND events = 'reporting_status_notification_events' THEN + -- Expanded: the widened reference and key beside the earlier ones. + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = events::regclass AND conname = events || '_offering_fkey') THEN + EXECUTE format('LOCK TABLE %I IN ACCESS EXCLUSIVE MODE', events); + EXECUTE format( + 'ALTER TABLE %I ADD COLUMN IF NOT EXISTS offering_id TEXT COLLATE "C"' + ' NOT NULL DEFAULT %L', events, ''); + EXECUTE format( + 'ALTER TABLE %I ADD CONSTRAINT %I UNIQUE (account_id, consumer_namespace,' + ' delivery_config_id, version, scope_kind, obligation_namespace, offering_id,' + ' cause_generation)', events, events || '_offering_key'); + EXECUTE format( + 'ALTER TABLE %I ADD CONSTRAINT %I FOREIGN KEY (account_id, consumer_namespace,' + ' delivery_config_id, version, scope_kind, obligation_namespace, offering_id)' + ' REFERENCES reporting_status_scope_checkpoints (account_id,' + ' consumer_namespace, delivery_config_id, version, scope_kind,' + ' obligation_namespace, offering_id)', events, events || '_offering_fkey'); + END IF; + ELSIF NOT EXISTS (SELECT 1 FROM pg_constraint fk WHERE fk.conrelid = events::regclass AND fk.contype = 'f' - AND fk.confrelid = 'reporting_status_scope_checkpoints'::regclass); - EXECUTE format('LOCK TABLE %I IN ACCESS EXCLUSIVE MODE', events); - IF NOT EXISTS (SELECT 1 FROM pg_attribute - WHERE attrelid = events::regclass AND attname = 'offering_id' AND NOT attisdropped) THEN - EXECUTE format( - 'ALTER TABLE %I ADD COLUMN offering_id TEXT COLLATE "C" NOT NULL DEFAULT %L', - events, ''); - -- An event of an offering's checkpoint takes that offering. Events - -- are otherwise immutable: only this new column is filled, once. - IF EXISTS (SELECT 1 FROM reporting_status_scope_checkpoints WHERE offering_id <> '') THEN - EXECUTE format('ALTER TABLE %I DISABLE TRIGGER USER', events); + AND fk.confrelid = 'reporting_status_scope_checkpoints'::regclass + AND array_length(fk.conkey, 1) = 7) THEN + EXECUTE format('LOCK TABLE %I IN ACCESS EXCLUSIVE MODE', events); + -- While expanded, an earlier release's projection event table still + -- references the six-column key; widened in place, it drops that. + FOR reference IN + SELECT fk.conname::TEXT AS constraint_name FROM pg_constraint fk + WHERE fk.conrelid = events::regclass AND fk.contype = 'f' + AND fk.confrelid = 'reporting_status_scope_checkpoints'::regclass + LOOP + EXECUTE format('ALTER TABLE %I DROP CONSTRAINT %I', events, reference.constraint_name); + END LOOP; + IF NOT EXISTS (SELECT 1 FROM pg_attribute + WHERE attrelid = events::regclass AND attname = 'offering_id' AND NOT attisdropped) THEN EXECUTE format( - 'UPDATE %I e SET offering_id = c.offering_id' - ' FROM reporting_status_scope_checkpoints c' - ' WHERE c.account_id = e.account_id' - ' AND c.consumer_namespace = e.consumer_namespace' - ' AND c.delivery_config_id = e.delivery_config_id AND c.version = e.version' - ' AND c.scope_kind = e.scope_kind' - ' AND c.obligation_namespace = e.obligation_namespace' - ' AND c.offering_id <> %L', events, ''); - EXECUTE format('ALTER TABLE %I ENABLE TRIGGER USER', events); + 'ALTER TABLE %I ADD COLUMN offering_id TEXT COLLATE "C" NOT NULL DEFAULT %L', + events, ''); + -- An event of an offering's checkpoint takes that offering. Events + -- are otherwise immutable: only this new column is filled, once. + IF EXISTS (SELECT 1 FROM reporting_status_scope_checkpoints WHERE offering_id <> '') THEN + EXECUTE format('ALTER TABLE %I DISABLE TRIGGER USER', events); + EXECUTE format( + 'UPDATE %I e SET offering_id = c.offering_id' + ' FROM reporting_status_scope_checkpoints c' + ' WHERE c.account_id = e.account_id' + ' AND c.consumer_namespace = e.consumer_namespace' + ' AND c.delivery_config_id = e.delivery_config_id AND c.version = e.version' + ' AND c.scope_kind = e.scope_kind' + ' AND c.obligation_namespace = e.obligation_namespace' + ' AND c.offering_id <> %L', events, ''); + EXECUTE format('ALTER TABLE %I ENABLE TRIGGER USER', events); + END IF; END IF; - END IF; - -- One event per checkpoint generation, under the widened key. The - -- replacement takes the name PostgreSQL gave the six-column one. - FOR reference IN - SELECT unique_key.conname::TEXT AS constraint_name - FROM pg_constraint unique_key - WHERE unique_key.conrelid = events::regclass AND unique_key.contype = 'u' - AND ARRAY( - SELECT attribute.attname::TEXT - FROM unnest(unique_key.conkey) WITH ORDINALITY AS key_column(attnum, position) - JOIN pg_attribute attribute ON attribute.attrelid = unique_key.conrelid - AND attribute.attnum = key_column.attnum - ORDER BY key_column.position) = ARRAY[ - 'account_id', 'consumer_namespace', 'delivery_config_id', 'version', - 'scope_kind', 'obligation_namespace', 'cause_generation'] - LOOP - EXECUTE format('ALTER TABLE %I DROP CONSTRAINT %I', events, reference.constraint_name); + -- One event per checkpoint generation, under the widened key. The + -- replacement takes the name PostgreSQL gave the six-column one. + FOR reference IN + SELECT unique_key.conname::TEXT AS constraint_name + FROM pg_constraint unique_key + WHERE unique_key.conrelid = events::regclass AND unique_key.contype = 'u' + AND ARRAY( + SELECT attribute.attname::TEXT + FROM unnest(unique_key.conkey) WITH ORDINALITY AS key_column(attnum, position) + JOIN pg_attribute attribute ON attribute.attrelid = unique_key.conrelid + AND attribute.attnum = key_column.attnum + ORDER BY key_column.position) = ARRAY[ + 'account_id', 'consumer_namespace', 'delivery_config_id', 'version', + 'scope_kind', 'obligation_namespace', 'cause_generation'] + LOOP + EXECUTE format('ALTER TABLE %I DROP CONSTRAINT %I', events, reference.constraint_name); + EXECUTE format( + 'ALTER TABLE %I ADD CONSTRAINT %I UNIQUE (account_id, consumer_namespace,' + ' delivery_config_id, version, scope_kind, obligation_namespace, offering_id,' + ' cause_generation)', events, reference.constraint_name); + END LOOP; + -- PostgreSQL names the reference as it named the six-column one. EXECUTE format( - 'ALTER TABLE %I ADD CONSTRAINT %I UNIQUE (account_id, consumer_namespace,' - ' delivery_config_id, version, scope_kind, obligation_namespace, offering_id,' - ' cause_generation)', events, reference.constraint_name); - END LOOP; - -- PostgreSQL names the reference as it named the six-column one. - EXECUTE format( - 'ALTER TABLE %I ADD FOREIGN KEY (account_id, consumer_namespace,' - ' delivery_config_id, version, scope_kind, obligation_namespace, offering_id)' - ' REFERENCES reporting_status_scope_checkpoints', events); + 'ALTER TABLE %I ADD FOREIGN KEY (account_id, consumer_namespace,' + ' delivery_config_id, version, scope_kind, obligation_namespace, offering_id)' + ' REFERENCES reporting_status_scope_checkpoints (account_id, consumer_namespace,' + ' delivery_config_id, version, scope_kind, obligation_namespace, offering_id)', + events); + END IF; IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = events::regclass AND conname = events || '_offering') THEN EXECUTE format( @@ -267,3 +344,86 @@ BEGIN $projection_event_guard$; END $checkpoint_offerings$; + +-- While the ledger is expanded and not yet contracted, processes of the +-- earlier release (adcp-reporting-storage 0.1.x/0.2.0b0) may still be running +-- and restarting beside this one. Each of their boots with status +-- notifications re-creates the two guards above with their own bodies, and +-- both releases' schema validators compare function bodies. So while +-- uncontracted this release installs exactly the earlier release's bodies too +-- (the text below is theirs, byte for byte), and neither release's boot +-- changes what the other validates. They are correct for this release's writes +-- while uncontracted: every generation is account-level, so offering_id is '' +-- wherever they omit it. The contract step re-runs this file, which then keeps +-- the bodies above. +DO $uncontracted_status_guards$ +BEGIN + IF to_regclass(quote_ident(current_schema()) || '.reporting_status_scope_checkpoints') IS NULL + OR NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'reporting_configurations'::regclass + AND conname = 'reporting_configurations_generation_key') THEN + RETURN; + END IF; + CREATE OR REPLACE FUNCTION reporting_status_scope_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $scope_guard$ + DECLARE owner reporting_obligations; + BEGIN + IF TG_OP='UPDATE' AND ( + ROW(NEW.account_id, NEW.consumer_namespace, NEW.delivery_config_id, NEW.version, + NEW.scope_kind, NEW.obligation_namespace) IS DISTINCT FROM + ROW(OLD.account_id, OLD.consumer_namespace, OLD.delivery_config_id, OLD.version, + OLD.scope_kind, OLD.obligation_namespace) + OR NEW.generation < OLD.generation OR NEW.generation > OLD.generation + 1 + OR NEW.source_sequence < OLD.source_sequence + OR (OLD.initialized AND NOT NEW.initialized) + OR (OLD.baseline AND NOT NEW.baseline) + OR (OLD.initialized AND + ((NEW.generation <> OLD.generation) <> (NEW.fingerprint <> OLD.fingerprint)))) THEN + RAISE EXCEPTION 'invalid status checkpoint transition' USING ERRCODE='23514'; + END IF; + IF NEW.scope_kind='obligation' THEN + SELECT * INTO owner FROM reporting_obligations + WHERE account_id=NEW.account_id AND consumer_id=NEW.consumer_namespace + AND reporting_obligation_id=NEW.obligation_namespace; + IF NOT FOUND OR owner.delivery_config_id <> NEW.delivery_config_id + OR owner.delivery_config_version <> NEW.version THEN + RAISE EXCEPTION 'invalid status scope owner' USING ERRCODE='23514'; + END IF; + END IF; + IF NEW.initialized AND ( + NOT NEW.publishable OR NEW.snapshot->>'health' IS NULL + OR jsonb_typeof(NEW.snapshot->'issues') IS DISTINCT FROM 'array' + OR CASE WHEN NEW.snapshot->>'health' IN ('delayed','action_required') THEN + NOT EXISTS (SELECT 1 FROM jsonb_array_elements(NEW.snapshot->'issues') i + WHERE i->>'severity'=NEW.snapshot->>'health') + ELSE jsonb_array_length(NEW.snapshot->'issues')<>0 END) THEN + RAISE EXCEPTION 'invalid status issue invariant' USING ERRCODE='23514'; + END IF; + RETURN NEW; + END + $scope_guard$; + CREATE OR REPLACE FUNCTION reporting_status_event_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $event_guard$ + DECLARE checkpoint reporting_status_scope_checkpoints; ids jsonb; + BEGIN + SELECT * INTO checkpoint FROM reporting_status_scope_checkpoints + WHERE account_id=NEW.account_id AND consumer_namespace=NEW.consumer_namespace + AND delivery_config_id=NEW.delivery_config_id AND version=NEW.version + AND scope_kind=NEW.scope_kind AND obligation_namespace=NEW.obligation_namespace; + IF NOT FOUND OR NOT checkpoint.initialized OR NOT checkpoint.publishable + OR checkpoint.generation<>NEW.cause_generation OR checkpoint.fingerprint<>NEW.fingerprint + OR checkpoint.snapshot->>'health' IS DISTINCT FROM NEW.snapshot #>> '{cause,health}' THEN + RAISE EXCEPTION 'invalid status event checkpoint' USING ERRCODE='23514'; + END IF; + SELECT coalesce(jsonb_agg(issue_id ORDER BY issue_id COLLATE "C"), '[]') INTO ids + FROM (SELECT DISTINCT i->>'issue_id' COLLATE "C" AS issue_id + FROM jsonb_array_elements(checkpoint.snapshot->'issues') i + ORDER BY issue_id LIMIT 16) selected; + IF ids IS DISTINCT FROM NEW.snapshot #> '{cause,issue_ids}' THEN + RAISE EXCEPTION 'invalid status event issues' USING ERRCODE='23514'; + END IF; + RETURN NEW; + END + $event_guard$; +END +$uncontracted_status_guards$; diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/materializer/schema.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/materializer/schema.py index bfc4fcb49b..2d778728e5 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/materializer/schema.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/materializer/schema.py @@ -11,6 +11,7 @@ REQUIRED_OBJECTS, is_legacy_row_object, legacy_rows_retired, + satisfies, schema_objects, validate_schema, ) @@ -31,7 +32,7 @@ async def validate_materializer_schema(connection: Any, *, notifications: bool = raise LedgerConflictError( "MATERIALIZER_SCHEMA_UNREADY", "materializer schema catalog is unavailable" ) - if any(installed.get(key) != expected for key, expected in required.items()): + if any(not satisfies(installed, key, expected) for key, expected in required.items()): raise LedgerConflictError("MATERIALIZER_SCHEMA_UNREADY", "install the materializer schema") # Inherit the reviewed foundation prerequisites without appending B2 objects # to A's manifest or requiring optional notification delivery infrastructure. @@ -49,7 +50,7 @@ async def validate_materializer_schema(connection: Any, *, notifications: bool = continue if retired and is_legacy_row_object(key): continue - if installed.get(key) != expected: + if not satisfies(installed, key, expected): raise LedgerConflictError( "MATERIALIZER_SCHEMA_UNREADY", "install the retained evidence schema" ) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0002_reporting_subscription_generations.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0002_reporting_subscription_generations.sql index 3abd330c69..93df54040e 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0002_reporting_subscription_generations.sql +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0002_reporting_subscription_generations.sql @@ -1,12 +1,15 @@ -- adcp-migration: 0002 reporting_subscription_generations --- adcp-upgrade: stopped-worker -- Widens the owned configuration-generation key to -- (account_id, consumer_id, delivery_config_id, delivery_config_version, -- offering_id), so a reporting subscription keeps one generation per resolved -- offering of a version (adcp-client-python#1729). Existing rows keep -- offering_id '' and their uniqueness. -- --- This is a stopped-worker upgrade: stop every reporting worker before running --- it, because workers of earlier releases refuse the widened schema. Start the --- new release only after it commits. On a new schema it changes nothing. +-- Rolling-deploy safe. A ledger created by adcp-reporting-storage 0.1.x or +-- 0.2.0b0 is EXPANDED: the keys and references that release uses keep their +-- names, the widened ones are added beside them, and CHECK (offering_id = '') +-- holds until the contract step (0007). Processes of that release keep +-- booting, reading and writing account-level generations, and a rollback +-- works until contract. On a new ledger, or one already widened, it changes +-- nothing. -- include: ledger/reporting_ledger_account_generations.sql diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0006_reporting_status_checkpoint_offerings.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0006_reporting_status_checkpoint_offerings.sql index 703da2499a..b645eef37d 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0006_reporting_status_checkpoint_offerings.sql +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0006_reporting_status_checkpoint_offerings.sql @@ -1,5 +1,4 @@ -- adcp-migration: 0006 reporting_status_checkpoint_offerings --- adcp-upgrade: stopped-worker -- Keys status checkpoints, and the status events that reference them, by the -- generation's resolved offering: (account_id, consumer_namespace, -- delivery_config_id, version, scope_kind, obligation_namespace, offering_id). @@ -7,9 +6,8 @@ -- their own checkpoints. Existing rows keep offering_id '' and their key, and -- the ids and digests derived from an account-level checkpoint are unchanged. -- --- This is a stopped-worker upgrade: stop every reporting worker before running --- it. Workers of earlier releases insert checkpoints on the six-column key, --- which no longer exists, and swapping a primary key takes an exclusive lock --- on the checkpoint and status event tables. Start the new release only after --- it commits. It changes nothing on a schema without status notifications. +-- Rolling-deploy safe. While the ledger is expanded (0002), the six-column keys +-- that workers of adcp-reporting-storage 0.1.x/0.2.0b0 write keep their names +-- and the widened ones are added beside them until the contract step (0007). +-- It changes nothing on a schema without status notifications. -- include: ledger/reporting_status_checkpoint_offerings.sql diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0007_reporting_contract_generation_keys.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0007_reporting_contract_generation_keys.sql new file mode 100644 index 0000000000..ed84b8b06c --- /dev/null +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0007_reporting_contract_generation_keys.sql @@ -0,0 +1,19 @@ +-- adcp-migration: 0007 reporting_contract_generation_keys +-- adcp-upgrade: contract +-- Contracts a ledger that 0002 and 0006 expanded: drops what processes of +-- adcp-reporting-storage 0.1.x/0.2.0b0 (storage contract 1) needed to keep +-- running beside this release (the four-column configuration key, the +-- six-column obligation-period and status checkpoint keys, the references onto +-- them and CHECK (offering_id = '')), gives the widened keys their final names +-- and restores this release's function bodies. The ledger is then exactly a +-- new ledger's catalog, and reporting subscriptions can record a second +-- offering of a configuration version. +-- +-- A contract step: run it with "python -m adcp migrate --contract" once every +-- earlier-release reporting process has stopped. Plain "python -m adcp migrate" +-- and create_schema() never apply it to an expanded ledger. On a ledger that +-- was never expanded (a new one, or one that 0.2.0b1/b2 widened in place) it +-- changes nothing, and "python -m adcp migrate" records it without --contract. +-- include: ledger/reporting_ledger_contract.sql +-- include: ledger/reporting_ledger_reconciliation.sql +-- include: ledger/reporting_status_checkpoint_offerings.sql diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/__init__.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/__init__.py index 013239c87c..97fcdef5e6 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/__init__.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/__init__.py @@ -7,6 +7,8 @@ * ``-- adcp-migration: NNNN name`` names it; * ``-- adcp-upgrade: stopped-worker`` marks an upgrade that workers of earlier releases cannot run beside: stop every reporting worker first; +* ``-- adcp-upgrade: contract`` marks the contract half of an expand/contract + upgrade (below); * ``-- include: `` applies one of the package's idempotent DDL files, in order, so each statement lives in exactly one place. Any other SQL in the file runs after its includes. @@ -24,6 +26,20 @@ by either entry point carries its schema version; ``python -m adcp migrate`` then adds the later ones (``0004`` onward), which only ``postgres_storage`` and the notification stores need. + +**Expand/contract.** A ledger created by adcp-reporting-storage 0.1.x or +0.2.0b0 (storage contract 1) is upgraded in two halves, so a rolling deploy +works. ``0002`` and ``0006`` *expand* it: processes of the earlier release keep +booting, reading and writing account-level generations beside this one, and a +rollback keeps working. ``0007`` *contracts* it to exactly a new ledger's +catalog, which reporting subscriptions need. The contract step runs only on +request, :data:`CONTRACT_COMMAND` (``migrate(contract=True)``), once every +earlier-release reporting process has stopped: plain ``python -m adcp migrate`` +(typically a pre-deploy job, while earlier pods still run), ``create_schema()`` +and :func:`apply_pending` stop before it. On a ledger that was never expanded +(a new one, or one that 0.2.0b1/b2 widened in place) the contract step changes +nothing and is applied and recorded without the request. ``postgres_storage`` +starts on a ledger whose only pending migration is a contract step. """ from __future__ import annotations @@ -35,20 +51,31 @@ from importlib.resources import files from typing import Any +from adcp.reporting.ledger.store import LedgerConflictError + __all__ = [ + "CONTRACT_COMMAND", + "CONTRACT_WHEN", "MIGRATE_COMMAND", "Migration", "MigrationStatus", + "ReportingUpgradeNotContractedError", "applied_migrations", "apply_pending", + "describe", "migrate", "migration_status", "migrations", + "needs_contract", "pending_migrations", ] #: The command that applies pending migrations. MIGRATE_COMMAND = "python -m adcp migrate" +#: The command that also applies the contract step of an expanded ledger. +CONTRACT_COMMAND = "python -m adcp migrate --contract" +#: When to run :data:`CONTRACT_COMMAND`. +CONTRACT_WHEN = "once every earlier-release reporting process has stopped" _TABLE = """ CREATE TABLE IF NOT EXISTS adcp_schema_migrations ( @@ -61,6 +88,57 @@ "SELECT pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema()))" ) _FILE = re.compile(r"^(\d{4})_([a-z0-9_]+)\.sql$") +# An expanded ledger keeps the four-column configuration key, with the widened +# one beside it under this name; an earlier release's ledger (to be expanded) +# has the four-column key and no offering_id. Either needs the contract step. +_NEEDS_CONTRACT = """ +SELECT EXISTS ( + SELECT 1 FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = current_schema() AND t.relname = 'reporting_configurations' + AND (c.conname = 'reporting_configurations_generation_key' + OR (c.contype = 'p' AND NOT EXISTS ( + SELECT 1 FROM pg_attribute a + WHERE a.attrelid = t.oid AND a.attnum = ANY(c.conkey) + AND a.attname = 'offering_id')))) +""" +# PostgreSQL: a CHECK constraint was violated. +_CHECK_VIOLATION = "23514" +_UNCONTRACTED_SUFFIX = "_uncontracted" + + +class ReportingUpgradeNotContractedError(LedgerConflictError): + """A subscription generation was written before the contract step. + + An expanded ledger (upgraded from adcp-reporting-storage 0.1.x/0.2.0b0 + while processes of that release may still run) holds account-level + generations only, under ``CHECK (offering_id = '')``. Code + ``REPORTING_UPGRADE_NOT_CONTRACTED``; the message names the contract step. + """ + + CODE = "REPORTING_UPGRADE_NOT_CONTRACTED" + + def __init__(self) -> None: + super().__init__( + self.CODE, + "this reporting ledger was upgraded from adcp-reporting-storage 0.1.x/0.2.0b0 and is" + " not contracted yet, so it records account-level generations only: a reporting" + " subscription's generation (offering_id) needs the contract step. Run" + f' "{CONTRACT_COMMAND}" {CONTRACT_WHEN}.', + field="offering_id", + ) + + +def upgrade_not_contracted_error(error: BaseException) -> ReportingUpgradeNotContractedError | None: + """The typed error for a write an expanded ledger's ``CHECK (offering_id = '')`` refused.""" + if getattr(error, "sqlstate", None) != _CHECK_VIOLATION: + return None + diagnostic = getattr(error, "diag", None) + constraint = getattr(diagnostic, "constraint_name", None) or "" + if not constraint.endswith(_UNCONTRACTED_SUFFIX): + return None + return ReportingUpgradeNotContractedError() @dataclass(frozen=True) @@ -73,6 +151,9 @@ class Migration: includes: tuple[str, ...] body: str description: str + #: The contract half of an expand/contract upgrade: applied to an expanded + #: ledger only on request (:data:`CONTRACT_COMMAND`). + contract: bool = False def statements(self) -> list[str]: """The SQL this migration runs, in order.""" @@ -103,14 +184,14 @@ def _parse(filename: str, text: str) -> Migration: includes: list[str] = [] body: list[str] = [] description: list[str] = [] - stopped_worker = False + upgrade = "" declared: str | None = None for line in text.splitlines(): stripped = line.strip() if stripped.startswith("-- adcp-migration:"): declared = stripped.removeprefix("-- adcp-migration:").strip() elif stripped.startswith("-- adcp-upgrade:"): - stopped_worker = stripped.removeprefix("-- adcp-upgrade:").strip() == "stopped-worker" + upgrade = stripped.removeprefix("-- adcp-upgrade:").strip() elif stripped.startswith("-- include:"): includes.append(stripped.removeprefix("-- include:").strip()) elif stripped.startswith("--"): @@ -119,13 +200,16 @@ def _parse(filename: str, text: str) -> Migration: body.append(line) if declared != f"{version} {name}": raise ValueError(f"migration file {filename!r} must declare '{version} {name}'") + if upgrade not in ("", "stopped-worker", "contract"): + raise ValueError(f"migration file {filename!r} declares an unknown upgrade {upgrade!r}") return Migration( version=version, name=name, - stopped_worker=stopped_worker, + stopped_worker=upgrade == "stopped-worker", includes=tuple(includes), body="\n".join(body), description=" ".join(item for item in description if item), + contract=upgrade == "contract", ) @@ -168,6 +252,20 @@ async def applied_migrations(connection: Any) -> dict[str, datetime]: return {str(version): applied_at for version, applied_at in await cursor.fetchall()} +async def needs_contract(connection: Any) -> bool: + """Whether this schema's ledger is expanded, or is an earlier release's to be expanded. + + Such a ledger needs the contract step (:data:`CONTRACT_COMMAND`); a new + ledger, or one already contracted or widened in place, does not. + ``connection`` is a psycopg ``AsyncConnection`` or ``AsyncConnectionPool``. + """ + if hasattr(connection, "getconn"): # a pool + async with connection.connection() as borrowed: + return await needs_contract(borrowed) + row = await (await connection.execute(_NEEDS_CONTRACT)).fetchone() + return bool(row and row[0]) + + async def migration_status(connection: Any) -> list[MigrationStatus]: """Every shipped migration with its ``applied_at``, or ``None`` while pending. @@ -186,26 +284,39 @@ async def pending_migrations(connection: Any) -> list[Migration]: return [item.migration for item in await migration_status(connection) if item.pending] -async def migrate(pool: Any, *, dry_run: bool = False) -> list[Migration]: +async def migrate(pool: Any, *, dry_run: bool = False, contract: bool = False) -> list[Migration]: """Apply every pending migration in order; return what was (or would be) applied. ``pool`` is a psycopg ``AsyncConnectionPool`` or ``AsyncConnection`` whose ``search_path`` selects the schema. Each migration commits on its own, so a failure leaves the earlier ones applied and the failed one pending. + + The walk stops before a contract step whose ledger is expanded (or is an + earlier release's, which this run expands) unless ``contract`` is set: pass + it only once every earlier-release reporting process has stopped. """ if hasattr(pool, "getconn"): # a pool async with pool.connection() as connection: - return await _migrate(connection, dry_run=dry_run) - return await _migrate(pool, dry_run=dry_run) + return await _migrate(connection, dry_run=dry_run, contract=contract) + return await _migrate(pool, dry_run=dry_run, contract=contract) -async def _migrate(connection: Any, *, dry_run: bool) -> list[Migration]: +async def _migrate(connection: Any, *, dry_run: bool, contract: bool) -> list[Migration]: pending = await pending_migrations(connection) - if dry_run or not pending: + if dry_run: + if not contract and any(item.contract for item in pending): + async with connection.transaction(): + expanded = await needs_contract(connection) + if expanded: + first = next(index for index, item in enumerate(pending) if item.contract) + return pending[:first] return pending applied: list[Migration] = [] for migration in pending: async with connection.transaction(): + await connection.execute(_LOCK) + if migration.contract and not contract and await needs_contract(connection): + break # stopped-worker: the operator asks for it explicitly if await _apply(connection, migration): applied.append(migration) return applied @@ -224,7 +335,9 @@ async def apply_pending( ``within`` bounds what the caller owns: migrations are applied in order and the walk stops at the first pending one that includes a file outside it (a later migration may depend on an earlier one, so none is skipped). - The rest stay pending for ``python -m adcp migrate``. + The rest stay pending for ``python -m adcp migrate``. The walk also stops + at a contract step that the ledger needs: a booting process never + contracts, since processes of the earlier release may still be running. """ await connection.execute(_LOCK) await connection.execute(_TABLE) @@ -235,6 +348,8 @@ async def apply_pending( continue if within is not None and not set(migration.includes) <= within: break + if migration.contract and await needs_contract(connection): + break if await _apply(connection, migration): result.append(migration) return result @@ -259,15 +374,24 @@ async def _apply(connection: Any, migration: Migration) -> bool: def describe(statuses: Sequence[MigrationStatus]) -> list[str]: - """``--status`` lines: one per migration, flagging stopped-worker upgrades.""" + """``--status`` lines: one per migration, flagging stopped-worker and contract steps.""" lines: list[str] = [] for status in statuses: item = status.migration state = "pending" if status.pending else f"applied {status.applied_at:%Y-%m-%dT%H:%M:%SZ}" - note = ( - " [stopped-worker upgrade: stop every reporting worker before migrating]" - if item.stopped_worker and status.pending - else " [stopped-worker upgrade]" if item.stopped_worker else "" - ) + if item.contract: + note = ( + f" [contract: run {CONTRACT_WHEN}: {CONTRACT_COMMAND}]" + if status.pending + else " [contract]" + ) + elif item.stopped_worker: + note = ( + " [stopped-worker upgrade: stop every reporting worker before migrating]" + if status.pending + else " [stopped-worker upgrade]" + ) + else: + note = "" lines.append(f"{item.version} {item.name}: {state}{note}") return lines diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/cli.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/cli.py index 0f02de60a7..c4017b3391 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/cli.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/cli.py @@ -13,7 +13,7 @@ import sys from collections.abc import Sequence -USAGE = "python -m adcp migrate [--dsn URL] [--status | --dry-run]" +USAGE = "python -m adcp migrate [--dsn URL] [--status | --dry-run] [--contract]" def main(argv: Sequence[str]) -> int: @@ -23,7 +23,9 @@ def main(argv: Sequence[str]) -> int: description=( "Apply the reporting storage migrations (adcp-reporting-storage) to a PostgreSQL " "database, in order. Each migration commits on its own under the reporting schema " - "lock. A stopped-worker upgrade needs every reporting worker stopped first." + "lock. A ledger created by adcp-reporting-storage 0.1.x/0.2.0b0 is expanded, so " + "processes of that release keep running through a rolling deploy; the contract " + "step that completes the upgrade runs only with --contract." ), ) parser.add_argument( @@ -35,11 +37,21 @@ def main(argv: Sequence[str]) -> int: mode = parser.add_mutually_exclusive_group() mode.add_argument("--status", action="store_true", help="List applied and pending migrations") mode.add_argument("--dry-run", action="store_true", help="Print what would be applied") + parser.add_argument( + "--contract", + action="store_true", + help="Also apply the contract step of an expanded ledger. Run it only once every " + "earlier-release reporting process has stopped: they cannot write a contracted ledger.", + ) args = parser.parse_args(list(argv)) if not args.dsn: parser.error("pass --dsn or set DATABASE_URL") + if args.status and args.contract: + parser.error("--contract applies migrations; it does not combine with --status") try: - return asyncio.run(_run(args.dsn, status=args.status, dry_run=args.dry_run)) + return asyncio.run( + _run(args.dsn, status=args.status, dry_run=args.dry_run, contract=args.contract) + ) except ImportError as error: print( f"{error}. Install the PostgreSQL storage: pip install 'adcp-reporting-storage[pg]'", @@ -48,10 +60,17 @@ def main(argv: Sequence[str]) -> int: return 2 -async def _run(dsn: str, *, status: bool, dry_run: bool) -> int: +async def _run(dsn: str, *, status: bool, dry_run: bool, contract: bool = False) -> int: from psycopg import AsyncConnection - from adcp_reporting_storage.migrations import describe, migrate, migration_status + from adcp_reporting_storage.migrations import ( + CONTRACT_COMMAND, + CONTRACT_WHEN, + describe, + migrate, + migration_status, + needs_contract, + ) async with await AsyncConnection.connect(dsn, autocommit=True) as connection: if status: @@ -59,14 +78,43 @@ async def _run(dsn: str, *, status: bool, dry_run: bool) -> int: for line in describe(statuses): print(line) pending = [item for item in statuses if item.pending] - print(f"{len(pending)} pending" if pending else "up to date") + if not pending: + print("up to date") + elif all(item.migration.contract for item in pending): + if await needs_contract(connection): + print( + "up to date except the contract step: run " + f'"{CONTRACT_COMMAND}" {CONTRACT_WHEN}' + ) + else: + print( + "up to date except recording the contract step, which changes nothing " + 'on this ledger: run "python -m adcp migrate"' + ) + else: + print(f"{len(pending)} pending") return 0 - planned = await migrate(connection, dry_run=dry_run) + planned = await migrate(connection, dry_run=dry_run, contract=contract) verb = "would apply" if dry_run else "applied" for item in planned: - note = " (stopped-worker upgrade)" if item.stopped_worker else "" + note = ( + " (contract)" + if item.contract + else " (stopped-worker upgrade)" if item.stopped_worker else "" + ) print(f"{verb} {item.version} {item.name}{note}") - if not planned: + held = [ + item + for item in (await migration_status(connection)) + if item.pending and item.migration not in planned + ] + if held and held[0].migration.contract: + item = held[0].migration + print( + f"not applied {item.version} {item.name} (contract): run " + f'"{CONTRACT_COMMAND}" {CONTRACT_WHEN}' + ) + elif not planned: print("up to date") return 0 diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/_schema.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/_schema.py index 86276fd361..2f04cc6146 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/_schema.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/_schema.py @@ -25,6 +25,48 @@ SCHEMA_CONTRACT: dict[str, str] = { key: value["fingerprint"] for key, value in REQUIRED_OBJECTS.items() } +#: An EXPANDED ledger (one created by adcp-reporting-storage 0.1.x/0.2.0b0 and +#: upgraded while processes of that release may still run, until the contract +#: step) keeps what that release needs under its original names. Each +#: ``alternates`` entry is the expanded fingerprint a required object may have +#: instead, accepted only with every object it ``requires`` (the widened key +#: that replaces it, from ``objects``) in place. See +#: ``ledger/reporting_ledger_account_generations.sql``. +_EXPAND_SCHEMA: dict[str, dict[str, dict[str, Any]]] = json.loads( + files("adcp_reporting_storage.outbox").joinpath("required_expand_schema.json").read_text() +) +EXPAND_ALTERNATES: dict[str, dict[str, Any]] = _EXPAND_SCHEMA["alternates"] +EXPAND_OBJECTS: dict[str, dict[str, Any]] = _EXPAND_SCHEMA["objects"] + + +def object_state( + installed: dict[str, dict[str, Any]], key: str, expected: dict[str, Any] +) -> str | None: + """``None`` when ``key`` satisfies ``expected``, else ``missing``/``disabled``/``changed``. + + An expanded ledger's fingerprint for the object satisfies it too, while + the widened objects that replace it are installed exactly. + """ + actual = installed.get(key) + if actual is None: + return "missing" + if not actual["enabled"]: + return "disabled" + if actual["fingerprint"] == expected["fingerprint"]: + return None + alternate = EXPAND_ALTERNATES.get(key) + if ( + alternate is not None + and actual["fingerprint"] == alternate["fingerprint"] + and all(installed.get(item) == EXPAND_OBJECTS[item] for item in alternate["requires"]) + ): + return None + return "changed" + + +def satisfies(installed: dict[str, dict[str, Any]], key: str, expected: dict[str, Any]) -> bool: + """Whether the installed ``key`` meets ``expected`` (contracted or expanded).""" + return installed.get(key) == expected or object_state(installed, key, expected) is None def _digest(value: object) -> str: @@ -227,14 +269,8 @@ def _validate_schema_objects( continue if legacy_rows_retired and is_legacy_row_object(key): continue - actual = installed.get(key) - if actual is None: - classification = "missing" - elif not actual["enabled"]: - classification = "disabled" - elif actual["fingerprint"] != expected["fingerprint"]: - classification = "changed" - else: + classification = object_state(installed, key, expected) + if classification is None: continue raise ReportingNotificationError(f"notification_schema_unready:{classification}:{key}") diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/required_expand_schema.json b/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/required_expand_schema.json new file mode 100644 index 0000000000..a4777b088d --- /dev/null +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/required_expand_schema.json @@ -0,0 +1,154 @@ +{ + "alternates": { + "constraint:reporting_configurations.reporting_configurations_pkey": { + "enabled": true, + "fingerprint": "cb0658b07aa5249a6934cca6248293967db4ab1a848d706c3d5310fa98f08785", + "requires": [ + "constraint:reporting_configurations.reporting_configurations_generation_key", + "constraint:reporting_configurations.reporting_configurations_uncontracted", + "index:reporting_configurations.reporting_configurations_generation_key" + ] + }, + "constraint:reporting_status_notification_events.reporting_status_notification_account_id_consumer_namespac_fkey": { + "enabled": true, + "fingerprint": "ee26a73f446c794ba69fc056f5fb1b8e094b25ae48743fe5e23045749e84f1c2", + "requires": [ + "constraint:reporting_status_notification_events.reporting_status_notification_events_offering_fkey", + "constraint:reporting_status_notification_events.reporting_status_notification_events_offering_key", + "index:reporting_status_notification_events.reporting_status_notification_events_offering_key" + ] + }, + "constraint:reporting_status_notification_events.reporting_status_notification_account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "a21acc6c4e405fcbf7711fe5a82c55206c994c6c969b498f9afe35af859ed206", + "requires": [ + "constraint:reporting_status_notification_events.reporting_status_notification_events_offering_fkey", + "constraint:reporting_status_notification_events.reporting_status_notification_events_offering_key", + "index:reporting_status_notification_events.reporting_status_notification_events_offering_key" + ] + }, + "constraint:reporting_status_scope_checkpoints.reporting_status_scope_checkpoints_pkey": { + "enabled": true, + "fingerprint": "9248b24cf88bd9d81f5c44079431cbd72714f7922600c4dd6a99f9a7a190436b", + "requires": [ + "constraint:reporting_status_scope_checkpoints.reporting_status_scope_checkpoints_offering_key", + "index:reporting_status_scope_checkpoints.reporting_status_scope_checkpoints_offering_key" + ] + }, + "function:reporting_reconciliation_evidence(r reporting_reconciliation_records)": { + "enabled": true, + "fingerprint": "cfa4e3f5b4c6505d2fe72d9828d95d3760555b1957b6d0754e3bac86c925bfdf", + "requires": [ + "constraint:reporting_configurations.reporting_configurations_generation_key" + ] + }, + "function:reporting_reconciliation_reference_immutable()": { + "enabled": true, + "fingerprint": "56fa1c28e2461a3ca5d7a68f8356f6ffb7652c0418dfaa6fad87963c849a92c1", + "requires": [ + "constraint:reporting_configurations.reporting_configurations_generation_key" + ] + }, + "function:reporting_reconciliation_validate(r reporting_reconciliation_records)": { + "enabled": true, + "fingerprint": "c4790a050f91451c7b7d5974c4513a8f9d2c5703c520604c4b87ba7aea443e8a", + "requires": [ + "constraint:reporting_configurations.reporting_configurations_generation_key" + ] + }, + "function:reporting_status_event_guard()": { + "enabled": true, + "fingerprint": "e56d1d9017667282bc1a046656fc2833a97252af37dd2c632a8e0e686b5fb175", + "requires": [ + "constraint:reporting_configurations.reporting_configurations_generation_key" + ] + }, + "function:reporting_status_scope_guard()": { + "enabled": true, + "fingerprint": "e52efa691de16f3f72df213ca103d8290807fb127ff790b15d7e68c9133757b7", + "requires": [ + "constraint:reporting_configurations.reporting_configurations_generation_key" + ] + }, + "index:reporting_configurations.reporting_configurations_pkey": { + "enabled": true, + "fingerprint": "477da25b66a532603d65020ecb7cf334aacf0c68906c04d1c842036ea3c8db74", + "requires": [ + "constraint:reporting_configurations.reporting_configurations_generation_key", + "constraint:reporting_configurations.reporting_configurations_uncontracted", + "index:reporting_configurations.reporting_configurations_generation_key" + ] + }, + "index:reporting_obligations.reporting_obligations_period_key": { + "enabled": true, + "fingerprint": "d21697e53fd171d1c462124684fe855a85d978e99ee015389600fddcb724f1df", + "requires": [ + "constraint:reporting_obligations.reporting_obligations_uncontracted", + "index:reporting_obligations.reporting_obligations_offering_period_key" + ] + }, + "index:reporting_status_notification_events.reporting_status_notification_account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "1c8d9302d57b13d6ffed87ad1ed7ed3199119151ef68a5c28b6a075b01157e3a", + "requires": [ + "constraint:reporting_status_notification_events.reporting_status_notification_events_offering_fkey", + "constraint:reporting_status_notification_events.reporting_status_notification_events_offering_key", + "index:reporting_status_notification_events.reporting_status_notification_events_offering_key" + ] + }, + "index:reporting_status_scope_checkpoints.reporting_status_scope_checkpoints_pkey": { + "enabled": true, + "fingerprint": "83b2cc13fef08e3cb5751d0182ff7b6b3b0aab04f7396a233d8a8f85cccd4a99", + "requires": [ + "constraint:reporting_status_scope_checkpoints.reporting_status_scope_checkpoints_offering_key", + "index:reporting_status_scope_checkpoints.reporting_status_scope_checkpoints_offering_key" + ] + } + }, + "objects": { + "constraint:reporting_configurations.reporting_configurations_generation_key": { + "enabled": true, + "fingerprint": "a2e5fcb8a81ecb287a401c51166eb767c1a3e44e938f339637477bbef69e78d1" + }, + "constraint:reporting_configurations.reporting_configurations_uncontracted": { + "enabled": true, + "fingerprint": "8acc83717805dad2b34cd3943ff8ceea21b1a876685f4e3bca5a44e99f38a768" + }, + "constraint:reporting_consumer_statuses.reporting_consumer_statuses_uncontracted": { + "enabled": true, + "fingerprint": "8acc83717805dad2b34cd3943ff8ceea21b1a876685f4e3bca5a44e99f38a768" + }, + "constraint:reporting_obligations.reporting_obligations_uncontracted": { + "enabled": true, + "fingerprint": "8acc83717805dad2b34cd3943ff8ceea21b1a876685f4e3bca5a44e99f38a768" + }, + "constraint:reporting_status_notification_events.reporting_status_notification_events_offering_fkey": { + "enabled": true, + "fingerprint": "a88d869a73abf6c88d6af29502b32e61e5f9657a03f80959629cb89086522dad" + }, + "constraint:reporting_status_notification_events.reporting_status_notification_events_offering_key": { + "enabled": true, + "fingerprint": "eea1d95be27f57f078192704cd006232ad6cdb37030fb397e2eccb3a3d430616" + }, + "constraint:reporting_status_scope_checkpoints.reporting_status_scope_checkpoints_offering_key": { + "enabled": true, + "fingerprint": "672918b7e185f9a8a83e94c5a9cee1b41209ee6f3bf24dcf54f8b73417e38530" + }, + "index:reporting_configurations.reporting_configurations_generation_key": { + "enabled": true, + "fingerprint": "35fc04ffd953b81c8e5cd056197a6fc13a59f342380e02eaba6718352e3ad61b" + }, + "index:reporting_obligations.reporting_obligations_offering_period_key": { + "enabled": true, + "fingerprint": "f79c76eaf5e8d69f915b9db57e666f144abdf71435642c15257390da34d26e2f" + }, + "index:reporting_status_notification_events.reporting_status_notification_events_offering_key": { + "enabled": true, + "fingerprint": "b567363d11ed0acbd0e10aa0d8d13bb90d0cd6da8e90814715e1841e517e2032" + }, + "index:reporting_status_scope_checkpoints.reporting_status_scope_checkpoints_offering_key": { + "enabled": true, + "fingerprint": "33133b321ba67e1df8da93dfb98182c06d26cd8d30c515aa37451b460d5dbfe2" + } + } +} diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/status_schema.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/status_schema.py index 876abad8f9..4b6eacc440 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/status_schema.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/status_schema.py @@ -7,7 +7,7 @@ from typing import Any from adcp.reporting.ledger.notification_models import ReportingNotificationError -from adcp_reporting_storage.outbox._schema import schema_objects, validate_schema +from adcp_reporting_storage.outbox._schema import object_state, schema_objects, validate_schema REQUIRED_STATUS_OBJECTS: dict[str, dict[str, Any]] = json.loads( files("adcp_reporting_storage.outbox").joinpath("required_status_schema.json").read_text() @@ -48,14 +48,8 @@ def _validate_status_objects( ) ): continue - actual = installed.get(key) - if actual is None: - reason = "missing" - elif not actual["enabled"]: - reason = "disabled" - elif actual["fingerprint"] != expected["fingerprint"]: - reason = "changed" - else: + reason = object_state(installed, key, expected) + if reason is None: continue raise ReportingNotificationError(f"status_schema_unready:{reason}:{key}") if status: diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/production/schema.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/production/schema.py index 228d541d3b..d95ec79061 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/production/schema.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/production/schema.py @@ -7,7 +7,7 @@ from typing import Any from adcp.reporting.ledger.notification_models import ReportingNotificationError -from adcp_reporting_storage.outbox._schema import schema_objects +from adcp_reporting_storage.outbox._schema import satisfies, schema_objects from adcp_reporting_storage.projection.schema import validate_projection_schema @@ -27,7 +27,7 @@ async def validate_production_schema( ) ) actual = await schema_objects(connection) - if not required or any(actual.get(k) != v for k, v in required.items()): + if not required or any(not satisfies(actual, k, v) for k, v in required.items()): raise ValueError await validate_projection_schema(connection, notifications=notifications) except Exception: diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/projection/schema.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/projection/schema.py index e4bb3c4a0b..a4f943ad18 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/projection/schema.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/projection/schema.py @@ -8,7 +8,7 @@ from adcp.reporting.ledger.notification_models import ReportingNotificationError from adcp_reporting_storage.feed.schema import validate_feed_schema -from adcp_reporting_storage.outbox._schema import schema_objects +from adcp_reporting_storage.outbox._schema import satisfies, schema_objects from adcp_reporting_storage.outbox.status_schema import validate_status_schema @@ -18,7 +18,7 @@ async def validate_projection_schema(connection: Any, *, notifications: bool = F files("adcp_reporting_storage.projection").joinpath("required_schema.json").read_text() ) actual = await schema_objects(connection) - if not required or any(actual.get(k) != v for k, v in required.items()): + if not required or any(not satisfies(actual, k, v) for k, v in required.items()): raise ValueError await validate_feed_schema(connection, notifications=notifications) await validate_status_schema(connection) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/storage.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/storage.py index 0433f8a549..d94516511f 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/storage.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/storage.py @@ -18,8 +18,9 @@ The schema is managed by versioned migrations (:mod:`adcp_reporting_storage.migrations`): run ``python -m adcp migrate`` before starting. Startup never changes the schema; it refuses one that is behind with ``REPORTING_SCHEMA_BEHIND``, naming -the command and the pending versions. Durability is a property of the -storage, never a flag. +the command and the pending versions. A ledger whose only pending migration is +the contract step (``python -m adcp migrate --contract``) is not behind: it +starts. Durability is a property of the storage, never a flag. """ from __future__ import annotations @@ -74,7 +75,10 @@ async def open_pool() -> None: async def check_schema() -> None: await open_pool() - pending = await pending_migrations(pool) + # A pending contract step is not "behind": an expanded ledger serves + # this release fully, except reporting subscriptions' second offerings, + # until the operator contracts it once earlier-release processes stop. + pending = [item for item in await pending_migrations(pool) if not item.contract] if pending: versions = ", ".join(item.version for item in pending) stopped = [item.version for item in pending if item.stopped_worker] diff --git a/tests/conformance/reporting/_expand_manifest.py b/tests/conformance/reporting/_expand_manifest.py new file mode 100644 index 0000000000..1cf7f9255f --- /dev/null +++ b/tests/conformance/reporting/_expand_manifest.py @@ -0,0 +1,63 @@ +"""Derive ``outbox/required_expand_schema.json`` from an expanded ledger's catalog. + +An earlier release's ledger (adcp-reporting-storage 0.1.x/0.2.0b0), expanded +by this checkout, keeps what that release needs under its original names, so a +few required objects of this release's manifests differ there (the ``alternates``) +and the widened ones exist under new names (the ``objects``). The published +upgrade test seeds such a ledger with the actual wheels, expands it, and +asserts the packaged file equals what this derives from its catalog. + +Regenerate after changing an expanded object: run that test with +``ADCP_WRITE_EXPAND_MANIFEST=1``. +""" + +from __future__ import annotations + +import json +from typing import Any + +#: Name suffixes of the objects that exist only while a ledger is expanded. +EXPAND_SUFFIXES = ( + "_generation_key", + "_offering_period_key", + "_uncontracted", + "_offering_key", + "_offering_fkey", +) +#: Every function alternate needs the expanded configuration key. +FUNCTION_REQUIRES = ("constraint:reporting_configurations.reporting_configurations_generation_key",) + + +def _table(key: str) -> str | None: + kind, _, name = key.partition(":") + if kind == "function": + return None + return name.split(".", 1)[0] + + +def _expand_only(key: str) -> bool: + kind, _, name = key.partition(":") + return kind in ("constraint", "index") and name.rsplit(".", 1)[-1].endswith(EXPAND_SUFFIXES) + + +def derive(installed: dict[str, dict[str, Any]], manifests: dict[str, dict[str, Any]]) -> dict: + """``{"alternates": ..., "objects": ...}`` for an expanded catalog and the manifests.""" + objects = {key: installed[key] for key in sorted(installed) if _expand_only(key)} + alternates: dict[str, dict[str, Any]] = {} + for key in sorted(manifests): + actual = installed.get(key) + if actual is None or actual == manifests[key]: + continue + table = _table(key) + requires = ( + list(FUNCTION_REQUIRES) + if table is None + else [item for item in objects if _table(item) == table] + ) + assert requires, f"{key} differs while expanded but nothing expanded replaces it" + alternates[key] = {**actual, "requires": requires} + return {"alternates": alternates, "objects": objects} + + +def render(manifest: dict) -> str: + return json.dumps(manifest, sort_keys=True, indent=2) + "\n" diff --git a/tests/conformance/reporting/_published_ledger_seed.py b/tests/conformance/reporting/_published_ledger_seed.py index 964d61a8be..29bf35ac6c 100644 --- a/tests/conformance/reporting/_published_ledger_seed.py +++ b/tests/conformance/reporting/_published_ledger_seed.py @@ -4,7 +4,8 @@ creates the ledger with the installed pair's own ``create_schema()`` and writes two account-level generations with obligations, revisions and rows, the way a seller on that release did; ``read`` reads them back; ``upgrade`` runs the -installed storage's ``create_schema()`` first. With ``notifications`` the +installed storage's ``create_schema()`` first; ``old_pod`` does what a still +running, or restarted, pod of that pair does (``old_pod`` below). With ``notifications`` the outbox and status-notification stores are created too, as a seller with status notifications boots them. Every action returns an ``image`` of every ``reporting_*``/``adcp_reporting_*`` table so the test can prove no @@ -187,6 +188,145 @@ async def create_schema(pool, store, *, notifications): await PgStatusNotificationStore(store).create_schema() +def _capability_offering(route): + return { + "offering_id": f"{route.upper()}_SNAPSHOT_V1", + "feed_purpose": "analytics", + "report_definition_id": "PAID_MEDIA_DAILY_V1", + "report_definition_uri": "https://contracts.example.test/reporting/paid-media-daily-v1", + "report_definition_sha256": "a" * 64, + "reporting_profile": { + "id": "paid_media_delivery", + "version": "1.0.0", + "schema_uri": "https://contracts.example.test/reporting/paid-media-daily-v1/schema", + "schema_sha256": "b" * 64, + "schema_dialect": "https://json-schema.org/draft/2020-12/schema", + "schema_ref_policy": "local_fragment_only", + "grain": "one row per media buy per period", + "primary_keys": ["media_buy_id"], + }, + "schedule": {"period_duration": "PT1H", "delivery_sla": "PT10M", "alignment": "utc"}, + "supported_finality": ["snapshot"], + "reconciliation_mode": "delivery_only", + } + + +def _account_context(configuration): + from adcp.reporting.fixtures import SNAPSHOT_OFFERING_ID, redacted_capabilities + from adcp.reporting.service import ReportingAccountContext + + route = configuration.delivery_config_id.removesuffix("-delivery") + return ReportingAccountContext( + account_id=configuration.account_id, + adapter=route, + currency="USD", + account_timezone=configuration.account_timezone, + source_scope=redacted_capabilities().source_scope, + snapshot_offering_id=SNAPSHOT_OFFERING_ID, + requested_dimensions=("campaign_id",), + capability_offering=_capability_offering(route), + publication_namespace="reporting-source:fixture", + ) + + +async def worker_turn(store, now): + """One full worker turn of this pair's service over the stored ``gam`` generations. + + Generations other test steps record for reconciliation scaffolding (the + ``daily`` route) are left out of the startup scan; every other stored + generation, whichever release recorded it, is recovered and worked. + """ + from adcp.reporting.fixtures import redacted_capabilities + from adcp.reporting.service import ReliableReportingService + from adcp.reporting.testing import ScriptedReportingAdapter + + rows = [ + { + "media_buy_id": "media-buy-redacted", + "campaign_id": "campaign-redacted-1", + "impressions": 10, + "spend": "1.25", + } + ] + every = store.list_all_configurations + + async def gam_generations(): + return tuple(item for item in await every() if item.delivery_config_id == "gam-delivery") + + store.list_all_configurations = gam_generations + service = ReliableReportingService( + store=store, clock=lambda: now, account_context=_account_context + ) + service.sources.register("gam", ScriptedReportingAdapter(redacted_capabilities(), [rows] * 8)) + try: + await service.start() # create_schema(): this pair's boot + if service.initialization_errors: + raise RuntimeError(f"initialization errors {service.initialization_errors}") + turn = await service.run_worker(now=now) + finally: + await service.close() + del store.list_all_configurations + if turn.configuration_errors: + raise RuntimeError(f"configuration errors {turn.configuration_errors}") + return sorted( + [key.account_id, key.consumer_id, key.delivery_config_id, key.delivery_config_version] + for key in turn.configurations + ) + + +async def old_pod(pool, store, settings, result): + """What a still-running (or restarted) pod of this pair does, one step at a time. + + Each step's outcome is recorded, so the test can assert every one: a + restart (``create_schema``), reads, recording a new generation, closing a + period of an existing one, a full worker turn and, with notifications, a + status baseline and projection. + """ + notifications = bool(settings.get("notifications")) + ordinal = int(settings.get("ordinal", 2)) + steps = result["steps"] = {} + + async def step(name, operation): + try: + outcome = await operation() + except Exception as error: + steps[name] = f"{type(error).__name__}: {error}" + return None + steps[name] = "ok" + return outcome + + status = None + if notifications: + from adcp_reporting_storage.outbox.status_pg import PgStatusNotificationStore + + status = PgStatusNotificationStore(store) + await step("create_schema", lambda: create_schema(pool, store, notifications=notifications)) + result["readback"] = await step("readback", lambda: readback(store)) + if status is not None: + await step("status_baseline", lambda: status.baseline(account_id=ACCOUNTS[0])) + + async def close_period(): + configuration = _configuration(ACCOUNTS[0]) + obligation = await store.commit_obligation(_obligation(configuration, ordinal)) + revision, rows = _revision(obligation, f"rpr_{ACCOUNTS[0]}_{ordinal}") + await store.commit_revision(revision, rows) + + await step("commit_obligation_and_revision", close_period) + await step( + "put_configuration", + lambda: store.put_configuration(_configuration(settings.get("account", "account-old-pod"))), + ) + if settings.get("now"): + now = datetime.fromisoformat(settings["now"]) + result["turn"] = await step("worker_turn", lambda: worker_turn(store, now)) + if status is not None: + await step("status_project", lambda: status.project_one(account_id=ACCOUNTS[0])) + checkpoints = await step( + "status_checkpoints", lambda: status.checkpoints(account_id=ACCOUNTS[0]) + ) + result["checkpoints"] = len(checkpoints or ()) + + async def main(settings): from adcp_reporting_storage.ledger.pg import PgReportingLedgerStore from psycopg_pool import AsyncConnectionPool @@ -211,7 +351,10 @@ async def main(settings): ) revision, rows = await _async_revision(obligation, account_id, ordinal) await store.commit_revision(revision, rows) - result["readback"] = await readback(store) + if settings["action"] == "old_pod": + await old_pod(pool, store, settings, result) + else: + result["readback"] = await readback(store) except Exception as error: # reported, so the test can assert on the repro result["error"] = f"{type(error).__name__}: {error}" async with pool.connection() as connection: diff --git a/tests/conformance/reporting/test_reporting_storage_migrations_pg.py b/tests/conformance/reporting/test_reporting_storage_migrations_pg.py index 4a2d266483..c3cfdf542d 100644 --- a/tests/conformance/reporting/test_reporting_storage_migrations_pg.py +++ b/tests/conformance/reporting/test_reporting_storage_migrations_pg.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from contextlib import asynccontextmanager import pytest @@ -32,7 +33,10 @@ async def test_migrate_applies_every_version_once_and_status_reads_it() -> None: assert await migrate(pool) == [] # idempotent lines = describe(await migration_status(pool)) assert all(": applied " in line for line in lines) - assert lines[1].endswith("[stopped-worker upgrade]") + # A new ledger was never expanded: its contract step applied without --contract. + assert lines[-1].startswith("0007 reporting_contract_generation_keys: applied ") + assert lines[-1].endswith("[contract]") + assert not any("stopped-worker" in line for line in lines) @pytest.mark.asyncio @@ -49,9 +53,10 @@ async def test_storage_refuses_a_schema_that_is_behind_and_starts_once_migrated( await storage.check_schema() error = refused.value assert error.code == "REPORTING_SCHEMA_BEHIND" and error.recovery == "correctable" - assert "pending: 0001, 0002, 0003, 0004, 0005, 0006" in str(error) + # The contract step alone never makes a schema "behind" (0007 is not listed). + assert "pending: 0001, 0002, 0003, 0004, 0005, 0006)" in str(error) assert "python -m adcp migrate" in str(error) - assert "0002, 0006 are stopped-worker upgrades" in str(error) + assert "stopped-worker" not in str(error) # every upgrade is rolling-deploy safe await migrate(pool) await storage.check_schema() @@ -87,12 +92,17 @@ async def test_the_cli_migrates_and_reports_status(capsys) -> None: dsn = f"{base}{'&' if '?' in base else '?'}options=-csearch_path%3D{schema}" assert await _run(dsn, status=True, dry_run=False) == 0 status = capsys.readouterr().out - assert "0002 reporting_subscription_generations: pending [stopped-worker upgrade" in status - assert "6 pending" in status + assert "0002 reporting_subscription_generations: pending\n" in status + assert ( + "0007 reporting_contract_generation_keys: pending [contract: run once every" + " earlier-release reporting process has stopped: python -m adcp migrate --contract]" + ) in status + assert "7 pending" in status assert await _run(dsn, status=False, dry_run=False) == 0 - assert "applied 0002 reporting_subscription_generations (stopped-worker upgrade)" in ( - capsys.readouterr().out - ) + applied = capsys.readouterr().out + assert "applied 0002 reporting_subscription_generations\n" in applied + # A new ledger has nothing to contract: the step is recorded without --contract. + assert "applied 0007 reporting_contract_generation_keys (contract)" in applied assert await _run(dsn, status=True, dry_run=False) == 0 assert capsys.readouterr().out.strip().endswith("up to date") @@ -256,5 +266,66 @@ async def test_create_schema_builds_its_own_schema_beside_a_migrated_one() -> No async with _search_path(own_schema, other_schema) as shared: await PgReportingLedgerStore(pool=shared).create_schema() # create_schema() records its own 0001-0003; the rest stay for ``migrate``. - assert [item.version for item in await pending_migrations(own)] == ["0004", "0005", "0006"] + assert [item.version for item in await pending_migrations(own)] == [ + "0004", + "0005", + "0006", + "0007", + ] assert await _catalog(other, other_schema) == before + + +#: Objects that exist only while a ledger is expanded (#1856). +_EXPAND_ONLY = re.compile( + r"_(generation_key|offering_period_key|uncontracted|offering_key|offering_fkey)$" +) + + +async def _expand_objects(pool) -> list[str]: + async with pool.connection() as connection: + rows = await ( + await connection.execute( + "SELECT conname FROM pg_constraint" + " WHERE connamespace = current_schema()::regnamespace" + " UNION ALL SELECT indexname FROM pg_indexes WHERE schemaname = current_schema()" + ) + ).fetchall() + return sorted(name for (name,) in rows if _EXPAND_ONLY.search(name)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("entry", ["migrate", "create_schema"]) +async def test_a_new_ledger_is_created_contracted(entry: str, capsys) -> None: + """A new ledger has no earlier-release processes: it is never expanded (#1856).""" + import os + + from adcp_reporting_storage.ledger.pg import PgReportingLedgerStore + from adcp_reporting_storage.migrations import migrate, needs_contract, pending_migrations + from adcp_reporting_storage.migrations.cli import _run + from adcp_reporting_storage.outbox.status_pg import PgStatusNotificationStore + + from adcp.storage.postgres import postgres_storage + + async with isolated_reporting_pool(autocommit=True) as pool: + store = PgReportingLedgerStore(pool=pool, notifications=True) + if entry == "migrate": + await migrate(pool) + await PgStatusNotificationStore(store).create_schema() + else: + await PgStatusNotificationStore(store).create_schema() + # The migrations the boot path leaves pending change nothing here, and + # the contract step is recorded without --contract. + assert [item.version for item in await migrate(pool)] == [ + "0004", + "0005", + "0006", + "0007", + ] + assert await _expand_objects(pool) == [] + assert not await needs_contract(pool) + assert await pending_migrations(pool) == [] + await postgres_storage(pool=pool).check_schema() + base = os.environ["ADCP_PG_TEST_URL"] + dsn = f"{base}{'&' if '?' in base else '?'}options=-csearch_path%3D{await _schema(pool)}" + assert await _run(dsn, status=True, dry_run=False) == 0 + assert capsys.readouterr().out.strip().endswith("up to date") diff --git a/tests/conformance/reporting/test_reporting_storage_published_upgrade_pg.py b/tests/conformance/reporting/test_reporting_storage_published_upgrade_pg.py index aab61c5ace..015b559296 100644 --- a/tests/conformance/reporting/test_reporting_storage_published_upgrade_pg.py +++ b/tests/conformance/reporting/test_reporting_storage_published_upgrade_pg.py @@ -11,10 +11,16 @@ * no retained evidence row changes beyond the added ``offering_id = ''``; * every record reads back and a restart rebinds the stored generations; -* the upgraded catalog is identical to a freshly migrated one; +* the upgrade is rolling-deploy safe (#1856): this checkout EXPANDS the + ledger, so processes of the published pair keep restarting, reading, + writing and running worker turns beside it, a rollback keeps working, and a + subscription generation is refused with a typed error until the contract + step, which leaves a catalog identical to a freshly migrated one; * the applied schema versions are recorded (``create_schema()`` records the - ledger's own ``0001``–``0003``; ``migrate`` the rest), re-running is a - no-op, and concurrent boots serialize; + ledger's own ``0001``–``0003``; ``migrate`` the rest, except the contract + step, which needs ``contract=True``), re-running is a no-op, and concurrent + boots serialize, old and new alike; +* a ledger the published 0.2.0b1 already widened in place counts as contracted; * ``adcp==10.0.0b4`` with *this* storage, the pair a seller upgrades to, upgrades a 0.2.0b0 ledger, where the published 0.2.0b0 fails. @@ -225,14 +231,49 @@ def assert_evidence_unchanged(before: dict[str, Any], after: dict[str, Any]) -> LEDGER_VERSIONS = ["0001", "0002", "0003"] -def _shipped() -> list[str]: +def _shipped(*, contract: bool = True) -> list[str]: + """Every shipped version; ``contract=False`` leaves out the contract steps.""" from adcp_reporting_storage.migrations import migrations - return [item.version for item in migrations()] + return [item.version for item in migrations() if contract or not item.contract] -def _rest() -> list[str]: - return [version for version in _shipped() if version not in LEDGER_VERSIONS] +def _rest(*, contract: bool = True) -> list[str]: + return [version for version in _shipped(contract=contract) if version not in LEDGER_VERSIONS] + + +#: The contract steps: applied to an expanded ledger only with ``contract=True``. +CONTRACT_VERSIONS = ["0007"] +#: The objects that exist only while a ledger is expanded (catalog lines). +EXPANDED_LEDGER = { + "constraint reporting_configurations reporting_configurations_generation_key", + "constraint reporting_configurations reporting_configurations_uncontracted", + "constraint reporting_consumer_statuses reporting_consumer_statuses_uncontracted", + "constraint reporting_obligations reporting_obligations_uncontracted", + "index reporting_configurations reporting_configurations_generation_key", + "index reporting_obligations reporting_obligations_offering_period_key", +} +EXPANDED_STATUS = { + "constraint reporting_status_scope_checkpoints reporting_status_scope_checkpoints_offering_key", + "index reporting_status_scope_checkpoints reporting_status_scope_checkpoints_offering_key", + "constraint reporting_status_notification_events" + " reporting_status_notification_events_offering_fkey", + "constraint reporting_status_notification_events" + " reporting_status_notification_events_offering_key", + "index reporting_status_notification_events reporting_status_notification_events_offering_key", +} + + +def _expand_only(catalog: list[str]) -> set[str]: + """The expand-only objects in a catalog, as ``kind table name``.""" + from tests.conformance.reporting._expand_manifest import EXPAND_SUFFIXES + + found = set() + for line in catalog: + kind, table, name, *_ = line.split(" ") + if kind in ("constraint", "index") and name.endswith(EXPAND_SUFFIXES): + found.add(f"{kind} {table} {name}") + return found async def _versions(pool: Any) -> list[str]: @@ -265,6 +306,27 @@ async def _catalog(pool: Any) -> list[str]: return sorted(row[0] for row in rows) +async def _full_catalog(pool: Any) -> list[str]: + """``_catalog`` plus every function's full definition and trigger definitions.""" + async with pool.connection() as connection: + functions = await ( + await connection.execute( + "SELECT 'function ' || replace(pg_get_functiondef(p.oid)," + " quote_ident(current_schema()) || '.', '') FROM pg_proc p" + " WHERE p.pronamespace = current_schema()::regnamespace" + ) + ).fetchall() + triggers = await ( + await connection.execute( + "SELECT 'triggerdef ' || replace(pg_get_triggerdef(t.oid)," + " quote_ident(current_schema()) || '.', '') FROM pg_trigger t" + " JOIN pg_class c ON c.oid = t.tgrelid" + " WHERE NOT t.tgisinternal AND c.relnamespace = current_schema()::regnamespace" + ) + ).fetchall() + return sorted([*await _catalog(pool), *(row[0] for row in [*functions, *triggers])]) + + # -- tests ----------------------------------------------------------------- @@ -273,7 +335,7 @@ async def test_create_schema_upgrades_a_published_ledger_in_place( ) -> None: """The ``store=`` startup path (``ReliableReportingService`` calls ``create_schema``).""" from adcp_reporting_storage.ledger.pg import PgReportingLedgerStore - from adcp_reporting_storage.migrations import migrate + from adcp_reporting_storage.migrations import migrate, needs_contract from adcp.storage.postgres import postgres_storage @@ -298,13 +360,21 @@ async def test_create_schema_upgrades_a_published_ledger_in_place( assert await _image_here(pool) == after assert await _versions(pool) == LEDGER_VERSIONS - applied = await migrate(pool) - assert [item.version for item in applied] == _rest() + # The boot expanded the ledger: what the published release needs keeps + # its name beside the widened keys, until the contract step. + assert await needs_contract(pool) + assert _expand_only(upgraded) == EXPANDED_LEDGER + + applied = await migrate(pool) # a pre-deploy job: never contracts + assert [item.version for item in applied] == _rest(contract=False) await postgres_storage(pool=pool).check_schema() # the migrations-managed path agrees + assert [item.version for item in await migrate(pool, contract=True)] == CONTRACT_VERSIONS assert_evidence_unchanged(seeded["image"], await _image_here(pool)) + contracted = await _full_catalog(pool) async with isolated_reporting_pool(autocommit=True) as fresh: await PgReportingLedgerStore(pool=fresh).create_schema() - assert upgraded == await _catalog(fresh) + await migrate(fresh) + assert contracted == await _full_catalog(fresh) async def test_migrate_upgrades_a_published_ledger_to_the_fresh_catalog( @@ -317,14 +387,16 @@ async def test_migrate_upgrades_a_published_ledger_to_the_fresh_catalog( async with isolated_reporting_pool(autocommit=True) as pool: seeded = await _seed(python, pool, workspace) applied = await migrate(pool) - assert [item.version for item in applied] == _shipped() - assert await migrate(pool) == [] + assert [item.version for item in applied] == _shipped(contract=False) + assert await migrate(pool) == [] # the contract step waits for --contract + assert [item.version for item in await migrate(pool, contract=True)] == CONTRACT_VERSIONS + assert await migrate(pool, contract=True) == [] assert_evidence_unchanged(seeded["image"], await _image_here(pool)) assert await _readback_here(pool) == seeded["readback"] - upgraded = await _catalog(pool) + upgraded = await _full_catalog(pool) async with isolated_reporting_pool(autocommit=True) as fresh: await migrate(fresh) - assert upgraded == await _catalog(fresh) + assert upgraded == await _full_catalog(fresh) async def test_restart_rebinds_stored_generations_and_closes_the_next_period( @@ -415,12 +487,13 @@ async def test_concurrent_boots_and_migrate_serialize_on_a_published_ledger( finally: for item in pools: await item.close() - assert await _versions(pool) == _shipped() + assert await _versions(pool) == _shipped(contract=False) async with pool.connection() as connection: row = await ( await connection.execute("SELECT count(*) FROM adcp_schema_migrations") ).fetchone() - assert row[0] == len(_shipped()) # each version recorded once across the four runners + # Each version recorded once across the four runners; none contracted. + assert row[0] == len(_shipped(contract=False)) assert_evidence_unchanged(seeded["image"], await _image_here(pool)) @@ -441,9 +514,12 @@ async def test_a_notifications_enabled_published_ledger_upgrades_through_every_m await seed.create_schema(pool, store, notifications=True) # a seller's boot assert await _versions(pool) == LEDGER_VERSIONS applied = await migrate(pool) - assert [item.version for item in applied] == _rest() - assert await _versions(pool) == _shipped() + assert [item.version for item in applied] == _rest(contract=False) + assert await _versions(pool) == _shipped(contract=False) assert await migrate(pool) == [] + assert _expand_only(await _catalog(pool)) == EXPANDED_LEDGER | EXPANDED_STATUS + assert [item.version for item in await migrate(pool, contract=True)] == CONTRACT_VERSIONS + assert _expand_only(await _catalog(pool)) == set() assert_evidence_unchanged(seeded["image"], await _image_here(pool)) assert await _readback_here(pool) == seeded["readback"] @@ -471,3 +547,461 @@ async def test_adcp_b4_fails_on_the_published_storage_and_upgrades_with_this_one assert upgraded["readback"] == seeded["readback"] assert_evidence_unchanged(seeded["image"], upgraded["image"]) assert await _versions(pool) == LEDGER_VERSIONS + + +# -- rolling deploy: expand, old and new side by side, contract (#1856) ------ + +#: Worker-turn instants, one per pod round, so each round closes a new period. +TURNS = [datetime(2026, 11, 1, hour, 30, tzinfo=timezone.utc) for hour in range(4, 12)] +#: The published 0.2.0b1 pair (#1831), which widened ledgers in place. +WIDENED_IN_PLACE_PAIR = ("0.2.0b1", "10.0.0b8") + + +async def _old_pod( + python: Path, + pool: Any, + cwd: Path, + *, + notifications: bool, + ordinal: int, + now: datetime | None = None, + account: str = "account-old-pod", +) -> dict[str, Any]: + """A pod of the published pair restarts and works (``old_pod`` in the seed script).""" + settings = { + **_settings(pool, "old_pod", notifications=notifications), + "ordinal": ordinal, + "account": account, + **({"now": now.isoformat()} if now is not None else {}), + } + result = await asyncio.to_thread(_run, python, settings, cwd=cwd) + assert "error" not in result, result["error"] + return result + + +def _all_ok(result: dict[str, Any]) -> None: + failed = {step: outcome for step, outcome in result["steps"].items() if outcome != "ok"} + assert result["steps"] and not failed, failed + + +async def _new_pod( + pool: Any, + *, + notifications: bool, + ordinal: int, + now: datetime, + account: str, + reconciliation_account: str, +) -> None: + """A pod of this checkout boots and does the same work, plus reconciliation writes.""" + from adcp_reporting_storage import PgReportingReconciliationStore + from adcp_reporting_storage.ledger.pg import PgReportingLedgerStore + + from tests.conformance.reporting import _published_ledger_seed as seed + from tests.conformance.reporting._reconciliation_support import scenario + + store = PgReportingLedgerStore(pool=pool, notifications=notifications) + await seed.create_schema(pool, store, notifications=notifications) + configuration = seed._configuration(ACCOUNTS[0]) + obligation = await store.commit_obligation(seed._obligation(configuration, ordinal)) + await store.commit_revision(*seed._revision(obligation, f"rpr_{ACCOUNTS[0]}_{ordinal}")) + await store.put_configuration(seed._configuration(account)) + turned = await seed.worker_turn(store, now) + assert [ACCOUNTS[0], "buyer-1", "gam-delivery", 1] in turned + # Destination binding, delivery, attempt, materialization and receipt: every + # reconciliation function an earlier release's boot re-creates. + recorder = PgReportingReconciliationStore(pool=pool, notifications=notifications) + await scenario(recorder, account_id=reconciliation_account) + if notifications: + from adcp_reporting_storage.outbox.status_pg import PgStatusNotificationStore + + status = PgStatusNotificationStore(store) + for account_id in (ACCOUNTS[0], reconciliation_account): + await status.baseline(account_id=account_id) + await status.project_one(account_id=account_id) + assert await status.checkpoints(account_id=account_id) + + +async def _read_here(pool: Any, *accounts: str) -> dict[str, Any]: + """This checkout reads the seeded records, and every account's generations.""" + from adcp_reporting_storage.ledger.pg import PgReportingLedgerStore + + from adcp.reporting.ledger import ReportingStatusCaller + + store = PgReportingLedgerStore(pool=pool) + configurations = [] + for account in accounts: + caller = ReportingStatusCaller(account_id=account, consumer_id="buyer-1") + configurations += [ + item.account_id for item in await store.list_configurations(caller=caller) + ] + return {"configurations": configurations, "readback": await _readback_here(pool)} + + +def _subscription_configuration(account: str) -> Any: + from dataclasses import replace + + from adcp.reporting.ledger.models import ReportingSubscriptionGeneration + from tests.conformance.reporting import _published_ledger_seed as seed + + return replace( + seed._configuration(account), + delivery_config_id="analytics", + subscription=ReportingSubscriptionGeneration("analytics", "PAID_MEDIA_SNAPSHOT"), + ) + + +async def _fresh_catalog(*, notifications: bool) -> list[str]: + """A new ledger created by this checkout the way the rolled-out sellers' was.""" + from adcp_reporting_storage.ledger.pg import PgReportingLedgerStore + from adcp_reporting_storage.migrations import migrate + + from tests.conformance.reporting import _published_ledger_seed as seed + + async with isolated_reporting_pool(autocommit=True) as fresh: + store = PgReportingLedgerStore(pool=fresh, notifications=notifications) + await seed.create_schema(fresh, store, notifications=notifications) + await migrate(fresh) + return await _full_catalog(fresh) + + +@pytest.mark.parametrize("notifications", [False, True], ids=["ledger", "notifications"]) +async def test_a_rolling_deploy_runs_published_and_new_pods_side_by_side_until_contract( + published: tuple[str, Path], workspace: Path, notifications: bool +) -> None: + from adcp_reporting_storage.ledger.pg import PgReportingLedgerStore + from adcp_reporting_storage.migrations import ( + ReportingUpgradeNotContractedError, + migrate, + needs_contract, + ) + + from adcp.storage.postgres import postgres_storage + from tests.conformance.reporting import _published_ledger_seed as seed + + _, python = published + async with isolated_reporting_pool(autocommit=True) as pool: + # 1. The published pair's ledger. + seeded = await _seed(python, pool, workspace, notifications=notifications) + # 2. The first new pod boots (and the pre-deploy job migrates): expanded. + await seed.create_schema( + pool, + PgReportingLedgerStore(pool=pool, notifications=notifications), + notifications=notifications, + ) + assert [item.version for item in await migrate(pool)] == _rest(contract=False) + await postgres_storage(pool=pool).check_schema() # only the contract step pends + assert await needs_contract(pool) + expanded = EXPANDED_LEDGER | (EXPANDED_STATUS if notifications else set()) + assert _expand_only(await _catalog(pool)) == expanded + assert_evidence_unchanged(seeded["image"], await _image_here(pool)) + # 3. A published pod restarts, reads, records a generation, closes a + # period and runs a full worker turn. + old = await _old_pod( + python, pool, workspace, notifications=notifications, ordinal=2, now=TURNS[0] + ) + _all_ok(old) + assert old["readback"] == seeded["readback"] + assert [ACCOUNTS[0], "buyer-1", "gam-delivery", 1] in old["turn"] + # 4. A new pod reads everything, the published pod's rows too, and writes. + read = await _read_here(pool, *ACCOUNTS, "account-old-pod") + assert "account-old-pod" in read["configurations"] + assert set(seeded["readback"]["revisions"]) <= set(read["readback"]["revisions"]) + await _new_pod( + pool, + notifications=notifications, + ordinal=3, + now=TURNS[1], + account="account-new-pod", + reconciliation_account="acct_new_pod", + ) + # 5. A published pod restarts again (its boot re-creates its function + # bodies), then new pods still write reconciliation and status records. + again = await _old_pod( + python, + pool, + workspace, + notifications=notifications, + ordinal=4, + now=TURNS[2], + account="account-old-pod-2", + ) + _all_ok(again) + assert "account-new-pod" in [row[0] for row in again["turn"]] # the new pod's generation + await _new_pod( + pool, + notifications=notifications, + ordinal=5, + now=TURNS[3], + account="account-new-pod-2", + reconciliation_account="acct_after_old_restart", + ) + assert _expand_only(await _catalog(pool)) == expanded # no boot moved the ledger + async with pool.connection() as connection: + recorded = await ( + await connection.execute( + "SELECT count(DISTINCT account_id) FROM reporting_reconciliation_records" + " WHERE account_id IN ('acct_new_pod', 'acct_after_old_restart')" + ) + ).fetchone() + assert recorded[0] == 2 + if notifications: + events = await ( + await connection.execute( + "SELECT count(*) FROM reporting_status_notification_events" + ) + ).fetchone() + assert events[0] > 0 + # 6. A subscription generation before contract: a typed error naming the step. + store = PgReportingLedgerStore(pool=pool, notifications=notifications) + with pytest.raises(ReportingUpgradeNotContractedError) as refused: + await store.put_configuration(_subscription_configuration(ACCOUNTS[0])) + assert refused.value.code == "REPORTING_UPGRADE_NOT_CONTRACTED" + assert "python -m adcp migrate --contract" in str(refused.value) + # 7. Every published process has stopped: contract. + assert [item.version for item in await migrate(pool, contract=True)] == CONTRACT_VERSIONS + assert not await needs_contract(pool) + # 8. Exactly a new ledger's catalog; subscriptions now record. + assert await _full_catalog(pool) == await _fresh_catalog(notifications=notifications) + await seed.create_schema(pool, store, notifications=notifications) # a new pod boots + await store.put_configuration(_subscription_configuration(ACCOUNTS[0])) + await _new_pod( + pool, + notifications=notifications, + ordinal=6, + now=TURNS[4], + account="account-after-contract", + reconciliation_account="acct_after_contract", + ) + # 9. The published pair cannot run on the contracted ledger: its boot + # refuses (with the caller-ownership message of #1831) and its writes fail. + late = await _old_pod(python, pool, workspace, notifications=notifications, ordinal=7) + assert "caller ownership" in late["steps"]["create_schema"], late["steps"] + assert "no unique or exclusion constraint" in late["steps"]["put_configuration"] + assert late["steps"]["readback"] == "ok" # reads still work + + +async def test_a_rollback_before_contract_keeps_the_published_pair_working( + published: tuple[str, Path], workspace: Path +) -> None: + """Old pods alone, after the new release expanded the ledger and was rolled back.""" + from adcp_reporting_storage.ledger.pg import PgReportingLedgerStore + from adcp_reporting_storage.migrations import migrate + + from tests.conformance.reporting import _published_ledger_seed as seed + + _, python = published + async with isolated_reporting_pool(autocommit=True) as pool: + seeded = await _seed(python, pool, workspace, notifications=True) + store = PgReportingLedgerStore(pool=pool, notifications=True) + await seed.create_schema(pool, store, notifications=True) + await migrate(pool) + expanded = await _full_catalog(pool) + for round_, now in enumerate(TURNS[:2]): + old = await _old_pod( + python, + pool, + workspace, + notifications=True, + ordinal=2 + round_, + now=now, + account=f"account-rollback-{round_}", + ) + _all_ok(old) + # Every seeded record reads back; a worker turn may add revisions. + assert old["readback"]["obligations"] == seeded["readback"]["obligations"] + assert set(seeded["readback"]["revisions"]) <= set(old["readback"]["revisions"]) + assert old["checkpoints"] > 0 + assert await _full_catalog(pool) == expanded # the published boots changed nothing + # Rolling forward again later still works. + await _new_pod( + pool, + notifications=True, + ordinal=4, + now=TURNS[2], + account="account-forward", + reconciliation_account="acct_forward", + ) + + +async def test_concurrent_new_and_published_boots_serialize_on_a_published_ledger( + published: tuple[str, Path], workspace: Path +) -> None: + """Two new boots and a published boot at once: no deadlock, each step applied once.""" + from adcp_reporting_storage.ledger.pg import PgReportingLedgerStore + from psycopg_pool import AsyncConnectionPool + + from tests.conformance.reporting import _published_ledger_seed as seed + + _, python = published + async with isolated_reporting_pool(autocommit=True) as pool: + seeded = await _seed(python, pool, workspace, notifications=True) + pools = [ + AsyncConnectionPool( + pool.conninfo, kwargs=pool.kwargs, min_size=1, max_size=2, open=False + ) + for _ in range(2) + ] + for item in pools: + await item.open(wait=True) + settings = _settings(pool, "upgrade", notifications=True) + try: + old, *_ = await asyncio.gather( + asyncio.to_thread(_run, python, settings, cwd=workspace), + *( + seed.create_schema( + item, + PgReportingLedgerStore(pool=item, notifications=True), + notifications=True, + ) + for item in pools + ), + ) + finally: + for item in pools: + await item.close() + assert "error" not in old, old["error"] # the published boot, before or after + assert await _versions(pool) == LEDGER_VERSIONS + async with pool.connection() as connection: + row = await ( + await connection.execute("SELECT count(*) FROM adcp_schema_migrations") + ).fetchone() + assert row[0] == len(LEDGER_VERSIONS) + assert _expand_only(await _catalog(pool)) == EXPANDED_LEDGER | EXPANDED_STATUS + assert_evidence_unchanged(seeded["image"], await _image_here(pool)) + _all_ok(await _old_pod(python, pool, workspace, notifications=True, ordinal=2)) + + +async def test_the_expand_manifest_is_the_expanded_published_catalog( + published: tuple[str, Path], workspace: Path +) -> None: + """``outbox/required_expand_schema.json`` is derived from a real expanded ledger. + + Both releases' validators accept the expanded catalog: the published + pair's by object name (it boots, above), this checkout's through the + expand alternates. + """ + from importlib.resources import files + + from adcp_reporting_storage.ledger.pg import PgReportingLedgerStore + from adcp_reporting_storage.outbox import _schema, status_schema + + from tests.conformance.reporting import _published_ledger_seed as seed + from tests.conformance.reporting._expand_manifest import derive, render + + _, python = published + async with isolated_reporting_pool(autocommit=True) as pool: + await _seed(python, pool, workspace, notifications=True) + # Expand without validating, as the published boot leaves it. + _all_ok(await _old_pod(python, pool, workspace, notifications=True, ordinal=2)) + store = PgReportingLedgerStore(pool=pool, notifications=True) + await seed.create_schema(pool, store, notifications=True) + _all_ok(await _old_pod(python, pool, workspace, notifications=True, ordinal=3)) + async with pool.connection() as connection: + installed = await _schema.schema_objects(connection) + await status_schema.validate_status_schema(connection) # this checkout accepts it + manifests = {**_schema.REQUIRED_OBJECTS, **status_schema.REQUIRED_STATUS_OBJECTS} + derived = render(derive(installed, manifests)) + packaged = files("adcp_reporting_storage.outbox").joinpath("required_expand_schema.json") + if os.environ.get("ADCP_WRITE_EXPAND_MANIFEST"): + Path(str(packaged)).write_text(derived) + assert packaged.read_text() == derived + + +@pytest.fixture(scope="module") +def widened_in_place(workspace: Path) -> Path: + """The published 0.2.0b1 pair, which widened a 0.1.x ledger in place (#1831).""" + storage, adcp = WIDENED_IN_PLACE_PAIR + return _environment( + workspace, f"pair-{storage}", f"adcp=={adcp}", f"adcp-reporting-storage[pg]=={storage}" + ) + + +async def test_a_ledger_the_published_0_2_0b1_widened_counts_as_contracted( + published: tuple[str, Path], widened_in_place: Path, workspace: Path +) -> None: + from adcp_reporting_storage.ledger.pg import PgReportingLedgerStore + from adcp_reporting_storage.migrations import migrate, needs_contract + + from adcp.storage.postgres import postgres_storage + from tests.conformance.reporting import _published_ledger_seed as seed + + storage, python = published + if storage != "0.1.0b4": + pytest.skip("one seed is enough: 0.2.0b1 widens 0.1.0b4 and 0.2.0b0 ledgers alike") + async with isolated_reporting_pool(autocommit=True) as pool: + seeded = await _seed(python, pool, workspace, notifications=True) + upgraded = await asyncio.to_thread( + _run, widened_in_place, _settings(pool, "upgrade", notifications=True), cwd=workspace + ) + assert "error" not in upgraded, upgraded["error"] + assert not await needs_contract(pool) + before = await _catalog(pool) + assert _expand_only(before) == set() + store = PgReportingLedgerStore(pool=pool, notifications=True) + await seed.create_schema(pool, store, notifications=True) # boots, does not re-expand + assert await _catalog(pool) == before + # The contract step is recorded without --contract: it changes nothing here. + applied = [item.version for item in await migrate(pool)] + assert applied[-1:] == CONTRACT_VERSIONS + await postgres_storage(pool=pool).check_schema() + after = await _catalog(pool) # 0004 and 0005 add their own tables + assert set(before) <= set(after) and _expand_only(after) == set() + await store.put_configuration(_subscription_configuration(ACCOUNTS[0])) + assert_evidence_unchanged(seeded["image"], upgraded["image"]) + + +async def test_migrate_holds_the_contract_step_and_status_says_when_to_run_it( + published: tuple[str, Path], workspace: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """``python -m adcp migrate`` as a pre-deploy job, beside running published pods.""" + from adcp_reporting_storage.migrations import describe, migration_status + from adcp_reporting_storage.migrations.cli import _run + + from adcp.storage.postgres import postgres_storage + + storage, python = published + if storage != "0.1.0b4": + pytest.skip("the CLI's output does not depend on which published pair seeded") + async with isolated_reporting_pool(autocommit=True) as pool: + await _seed(python, pool, workspace) + async with pool.connection() as connection: + schema = (await (await connection.execute("SELECT current_schema()")).fetchone())[0] + base = os.environ["ADCP_PG_TEST_URL"] + dsn = f"{base}{'&' if '?' in base else '?'}options=-csearch_path%3D{schema}" + when = "once every earlier-release reporting process has stopped" + + assert await _run(dsn, status=False, dry_run=True) == 0 + planned = capsys.readouterr().out + assert "would apply 0006 reporting_status_checkpoint_offerings\n" in planned + assert "would apply 0007" not in planned # held + assert ( + "not applied 0007 reporting_contract_generation_keys (contract): run" + f' "python -m adcp migrate --contract" {when}' + ) in planned + + assert await _run(dsn, status=False, dry_run=False) == 0 + applied = capsys.readouterr().out + assert "applied 0002 reporting_subscription_generations\n" in applied + assert "\napplied 0007" not in applied + assert "not applied 0007 reporting_contract_generation_keys (contract)" in applied + await postgres_storage(pool=pool).check_schema() # starts: only the contract pends + + assert await _run(dsn, status=True, dry_run=False) == 0 + status = capsys.readouterr().out.strip().splitlines() + assert status[-2] == ( + "0007 reporting_contract_generation_keys: pending [contract: run" + f" {when}: python -m adcp migrate --contract]" + ) + assert status[-1] == ( + f'up to date except the contract step: run "python -m adcp migrate --contract" {when}' + ) + assert describe(await migration_status(pool))[-1] == status[-2] + _all_ok(await _old_pod(python, pool, workspace, notifications=False, ordinal=2)) + + assert await _run(dsn, status=False, dry_run=False, contract=True) == 0 + assert capsys.readouterr().out.strip() == ( + "applied 0007 reporting_contract_generation_keys (contract)" + ) + assert await _run(dsn, status=True, dry_run=False) == 0 + assert capsys.readouterr().out.strip().endswith("up to date") diff --git a/tests/conformance/reporting/test_reporting_subscription_ledger_pg.py b/tests/conformance/reporting/test_reporting_subscription_ledger_pg.py index fc0409e201..9ae341c28a 100644 --- a/tests/conformance/reporting/test_reporting_subscription_ledger_pg.py +++ b/tests/conformance/reporting/test_reporting_subscription_ledger_pg.py @@ -3,7 +3,8 @@ The generation key is (account, consumer, delivery_config_id, delivery_config_version, offering_id), with offering_id '' for account-level generations. These run the in-memory owed-set scenarios against PostgreSQL and -cover the in-place widening of a schema still on the four-column key. +cover the upgrade of a schema still on the four-column key: expanded beside +the earlier release's key, then contracted (#1856). """ from __future__ import annotations @@ -28,6 +29,22 @@ NOW = at("2026-10-08T00:00:00Z") +#: The published (storage contract 1) layout of the base ledger: a new ledger +#: is created on the widened key, so the four-column one is rebuilt from it. +FOUR_COLUMN_LAYOUT = """ +ALTER TABLE reporting_configurations DROP CONSTRAINT reporting_configurations_pkey; +ALTER TABLE reporting_configurations DROP COLUMN offering_id; +ALTER TABLE reporting_configurations ADD PRIMARY KEY + (account_id, consumer_id, delivery_config_id, delivery_config_version); +DROP INDEX reporting_obligations_period_key; +ALTER TABLE reporting_obligations DROP COLUMN offering_id; +CREATE UNIQUE INDEX reporting_obligations_period_key ON reporting_obligations + (account_id, consumer_id, delivery_config_id, delivery_config_version, + period_start, period_end); +ALTER TABLE reporting_consumer_statuses DROP COLUMN offering_id; +""" + + async def _store(pool) -> PgReportingLedgerStore: store = PgReportingLedgerStore(pool=pool, clock=lambda: NOW) await store.create_schema() @@ -135,12 +152,31 @@ async def test_two_principals_scope_and_status_ingest_over_postgres() -> None: assert status.offering_id == "pkg" -async def test_a_four_column_schema_is_widened_losslessly() -> None: +async def test_a_four_column_schema_is_expanded_then_contracted_losslessly() -> None: + from adcp_reporting_storage.migrations import migrate, needs_contract + + async def keys(pool) -> tuple[str, list[str]]: + async with pool.connection() as connection: + primary = await ( + await connection.execute( + "SELECT pg_get_constraintdef(oid) FROM pg_constraint" + " WHERE conrelid = 'reporting_configurations'::regclass AND contype = 'p'" + ) + ).fetchone() + unique = await ( + await connection.execute( + "SELECT conname FROM pg_constraint" + " WHERE conrelid = 'reporting_configurations'::regclass AND contype = 'u'" + ) + ).fetchall() + return primary[0], [name for (name,) in unique] + async with isolated_reporting_pool(autocommit=True) as pool: ledger = files("adcp_reporting_storage.ledger") async with pool.connection() as connection: # The published layout: the base ledger DDL on the four-column key. await connection.execute(ledger.joinpath("reporting_ledger.sql").read_text()) + await connection.execute(FOUR_COLUMN_LAYOUT) await connection.execute( "INSERT INTO reporting_configurations (delivery_config_id," " delivery_config_version, account_id, consumer_id, report_definition_id," @@ -151,19 +187,22 @@ async def test_a_four_column_schema_is_widened_losslessly() -> None: ) for _ in range(2): # idempotent await PgReportingLedgerStore(pool=pool).create_schema() + # Expanded: the earlier release's key stays, the widened one is beside it. + primary, unique = await keys(pool) + assert primary.endswith("delivery_config_id, delivery_config_version)") + assert unique == ["reporting_configurations_generation_key"] + assert await needs_contract(pool) + assert "0007" not in [item.version for item in await migrate(pool)] + assert [item.version for item in await migrate(pool, contract=True)] == ["0007"] + primary, unique = await keys(pool) + assert primary.endswith("delivery_config_version, offering_id)") + assert unique == [] and not await needs_contract(pool) async with pool.connection() as connection: - primary = await ( - await connection.execute( - "SELECT pg_get_constraintdef(oid) FROM pg_constraint" - " WHERE conrelid = 'reporting_configurations'::regclass AND contype = 'p'" - ) - ).fetchone() rows = await ( await connection.execute( "SELECT delivery_config_id, offering_id FROM reporting_configurations" ) ).fetchall() - assert primary[0].endswith("delivery_config_version, offering_id)") assert rows == [("daily", "")] @@ -174,6 +213,7 @@ async def test_a_foreign_reference_to_the_key_refuses_the_widening() -> None: ledger = files("adcp_reporting_storage.ledger") async with pool.connection() as connection: await connection.execute(ledger.joinpath("reporting_ledger.sql").read_text()) + await connection.execute(FOUR_COLUMN_LAYOUT) await connection.execute( 'CREATE TABLE adopter_reference (account_id TEXT COLLATE "C",' ' consumer_id TEXT COLLATE "C", delivery_config_id TEXT COLLATE "C",' @@ -273,7 +313,7 @@ def scope(config) -> ReportingStatusScope: return ReportingStatusScope("acct_a", config.generation_key, feed_purpose="analytics") (widening,) = [item for item in migrations() if item.version == "0006"] - assert widening.stopped_worker + assert not widening.stopped_worker # rolling-deploy safe while expanded (#1856) assert widening.includes == ("ledger/reporting_status_checkpoint_offerings.sql",) async with isolated_reporting_pool(autocommit=True) as pool: await migrate(pool) # 0006 changes nothing without status notifications diff --git a/tests/fixtures/public_api_snapshot.json b/tests/fixtures/public_api_snapshot.json index 820bea3a8c..02fddddcf2 100644 --- a/tests/fixtures/public_api_snapshot.json +++ b/tests/fixtures/public_api_snapshot.json @@ -9054,6 +9054,7 @@ "ReportingRowQuarantineReason": "union[value:'BINDING_MISMATCH',value:'ROWS_NOT_PORTABLE',value:'ROW_SEQUENCE_INVALID']", "ReportingRowStorageOptionsV1": "adcp_reporting_storage.ledger.row_storage", "ReportingRowStorageV1": "adcp_reporting_storage.ledger.row_storage", + "ReportingUpgradeNotContractedError": "adcp_reporting_storage.migrations", "STORAGE_CONTRACT": "value:2", "assert_gcs_reporting_row_bucket_policy_v1": "adcp_reporting_storage.ledger.row_storage_gcs", "backfill_legacy_reporting": "adcp_reporting_storage.migration", diff --git a/tests/test_reporting_storage_facade.py b/tests/test_reporting_storage_facade.py index c199a60659..a27aa651a9 100644 --- a/tests/test_reporting_storage_facade.py +++ b/tests/test_reporting_storage_facade.py @@ -116,24 +116,79 @@ def test_migrations_are_numbered_files_with_their_upgrade_notes() -> None: from adcp_reporting_storage.migrations import MigrationStatus, describe, migrations shipped = migrations() - assert [item.version for item in shipped] == ["0001", "0002", "0003", "0004", "0005", "0006"] - widened, offerings = shipped[1], shipped[5] - assert widened.name == "reporting_subscription_generations" and widened.stopped_worker + assert [item.version for item in shipped] == [ + "0001", + "0002", + "0003", + "0004", + "0005", + "0006", + "0007", + ] + widened, offerings, contract = shipped[1], shipped[5], shipped[6] + assert widened.name == "reporting_subscription_generations" assert widened.includes == ("ledger/reporting_ledger_account_generations.sql",) - assert offerings.name == "reporting_status_checkpoint_offerings" and offerings.stopped_worker + assert offerings.name == "reporting_status_checkpoint_offerings" assert offerings.includes == ("ledger/reporting_status_checkpoint_offerings.sql",) + # Expand (#1856): rolling-deploy safe, so no upgrade needs stopped workers ... for item in (widened, offerings): - assert "stop every reporting worker" in item.description - assert not any(item.stopped_worker for item in shipped if item not in (widened, offerings)) + assert "Rolling-deploy safe" in item.description + assert not any(item.stopped_worker for item in shipped) + # ... and the contract step runs only on request, once earlier releases stopped. + assert contract.name == "reporting_contract_generation_keys" and contract.contract + assert contract.includes == ( + "ledger/reporting_ledger_contract.sql", + "ledger/reporting_ledger_reconciliation.sql", + "ledger/reporting_status_checkpoint_offerings.sql", + ) + assert "python -m adcp migrate --contract" in contract.description + assert [item.version for item in shipped if item.contract] == ["0007"] for item in shipped: assert item.statements(), item.version # every include resolves lines = describe([MigrationStatus(item, None) for item in shipped]) - assert lines[1] == ( - "0002 reporting_subscription_generations: pending [stopped-worker upgrade: stop every " - "reporting worker before migrating]" + assert lines[1] == "0002 reporting_subscription_generations: pending" + assert lines[6] == ( + "0007 reporting_contract_generation_keys: pending [contract: run once every " + "earlier-release reporting process has stopped: python -m adcp migrate --contract]" ) +def test_an_unknown_upgrade_header_is_refused() -> None: + pytest.importorskip("adcp_reporting_storage") + from adcp_reporting_storage.migrations import _parse + + with pytest.raises(ValueError, match="unknown upgrade"): + _parse("0009_x.sql", "-- adcp-migration: 0009 x\n-- adcp-upgrade: contracted\n") + assert _parse("0009_x.sql", "-- adcp-migration: 0009 x\n-- adcp-upgrade: contract\n").contract + + +def test_only_the_uncontracted_check_is_the_contract_refusal() -> None: + pytest.importorskip("adcp_reporting_storage") + from types import SimpleNamespace + + from adcp_reporting_storage.migrations import ( + ReportingUpgradeNotContractedError, + upgrade_not_contracted_error, + ) + + from adcp.reporting.ledger.store import LedgerConflictError + + class _Error(Exception): + def __init__(self, sqlstate: str, constraint: str | None) -> None: + super().__init__(sqlstate) + self.sqlstate = sqlstate + self.diag = SimpleNamespace(constraint_name=constraint) + + refused = upgrade_not_contracted_error(_Error("23514", "reporting_configurations_uncontracted")) + assert isinstance(refused, ReportingUpgradeNotContractedError) + assert isinstance(refused, LedgerConflictError) # handled like every ledger conflict + assert refused.code == "REPORTING_UPGRADE_NOT_CONTRACTED" + assert "python -m adcp migrate --contract" in str(refused) + assert upgrade_not_contracted_error(_Error("23514", "reporting_obligations_check")) is None + assert upgrade_not_contracted_error(_Error("23505", "x_uncontracted")) is None + assert upgrade_not_contracted_error(ValueError("no sqlstate")) is None + + def test_migrate_cli_requires_a_database(capsys) -> None: from adcp_reporting_storage.migrations.cli import main From 9a36ae22821f61eb9e85ec700505a92870b0f4c3 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sun, 8 Nov 2026 03:17:03 +0000 Subject: [PATCH 2/3] docs(reporting-storage): a rolling update now works for the 0.1.x/0.2.0b0 ledger upgrade Rewrite "Deploying the upgrade", the storage README and MIGRATION_v9_to_v10: roll out normally, run python -m adcp migrate --contract once every earlier-release reporting process has stopped, subscriptions need the contract step, and a rollback works until it. Recreate/stopped-worker advice remains only for the materializer, production and projection extensions. Co-Authored-By: Claude Opus 5.5 --- MIGRATION_v9_to_v10.md | 22 +- docs/reporting-ledger-migration.md | 237 +++++++++++++--------- packages/adcp-reporting-storage/README.md | 50 +++-- 3 files changed, 190 insertions(+), 119 deletions(-) diff --git a/MIGRATION_v9_to_v10.md b/MIGRATION_v9_to_v10.md index b12b0b7ee7..ec7af9837f 100644 --- a/MIGRATION_v9_to_v10.md +++ b/MIGRATION_v9_to_v10.md @@ -73,12 +73,16 @@ adcp 10.0.0b0–b3 are 1): such check and still install beside 0.2.0b0, which then fails every reporting turn, so install `adcp>=10.0.0b7` ([#1837](https://github.com/adcontextprotocol/adcp-client-python/issues/1837)). -- An existing SDK 9 ledger is upgraded in place by `create_schema()` or - `python -m adcp migrate`: `offering_id` is added with `''` for every - existing account-level generation, obligation and status, and the applied - schema versions are recorded. Stop every SDK 9 reporting worker first. A - default rolling update does not: once the first new pod upgrades the ledger, - old pods can no longer record generations or close periods, and cannot start - reporting if they restart. There is no rollback short of restoring a backup. - See [Deploying the upgrade](docs/reporting-ledger-migration.md#deploying-the-upgrade) - for a Recreate or two-phase rollout. +- An existing SDK 9 ledger is upgraded by `create_schema()` or `python -m adcp + migrate`, and a default rolling update works. The upgrade first *expands* + the ledger: `offering_id` is added with `''` for every existing + account-level generation, obligation and status, the keys SDK 9 uses keep + their names beside the widened ones, and the applied schema versions are + recorded. SDK 9 pods keep starting, reading and writing through the rollout, + and a rollback works. Once every SDK 9 reporting process has stopped, run + `python -m adcp migrate --contract`: reporting subscriptions need it (before + it they raise `REPORTING_UPGRADE_NOT_CONTRACTED`), and after it SDK 9 can no + longer run against the ledger. Plain `python -m adcp migrate` never + contracts. The materializer, production and status projection extensions + still need their SDK 9 workers stopped first. See + [Deploying the upgrade](docs/reporting-ledger-migration.md#deploying-the-upgrade). diff --git a/docs/reporting-ledger-migration.md b/docs/reporting-ledger-migration.md index b1267ac461..05f226bd9e 100644 --- a/docs/reporting-ledger-migration.md +++ b/docs/reporting-ledger-migration.md @@ -55,36 +55,68 @@ subscription version, so the generation key gains `offering_id`: account-level generation, and their digests, work keys and persisted scopes are unchanged. -In PostgreSQL this is a **stopped-worker maintenance upgrade**: - -1. Stop every reporting worker. Workers of earlier releases refuse the widened - schema; mixed-version workers are unsupported. -2. Run `create_schema()` (or your usual migration entry point) once with the new - release. Under the schema lock it widens the key in place and losslessly: - - `reporting_configurations` gains `offering_id` (`''` for existing rows), - and its primary key includes it; - - `reporting_obligations` gains `offering_id`, and its period unique index - includes it; - - `reporting_consumer_statuses` gains `offering_id`; - - the SDK's own references (reconciliation records, materializer discovery, - status scope checkpoints, production generations) are re-pointed at the - widened key. Reconciliation records, materializer discovery and status - scope checkpoints also hold subscription generations, for managed - delivery. Production generations hold account-level generations only, - enforced by a `CHECK`. -3. Restart the workers on the new release. +A new ledger is created on the widened key. A ledger created by an earlier +release (adcp-reporting-storage 0.1.x or 0.2.0b0, storage contract 1) is +upgraded in two halves, **expand** and **contract**, so a rolling deploy works +(see [Deploying the upgrade](#deploying-the-upgrade)). + +**Expand** (migration `0002`, which `create_schema()` and `python -m adcp +migrate` both apply) is online-safe: processes of the earlier release keep +booting, reading and writing account-level generations beside the new one. +Under the schema lock, losslessly: + +- `reporting_configurations`, `reporting_obligations` and + `reporting_consumer_statuses` gain `offering_id` (`''` for existing rows), + with `CHECK (offering_id = '')` until contract; +- everything the earlier release needs keeps its name and definition: the + four-column `reporting_configurations_pkey`, the six-column + `reporting_obligations_period_key`, and the four-column references from + reconciliation records, materializer discovery, status scope checkpoints and + production generations; +- the widened keys are added beside them under new names: + `reporting_configurations_generation_key` (`UNIQUE` on the five columns, the + target of the new release's writes and of the offering-aware references, + `…_generation_fkey`) and `reporting_obligations_offering_period_key`. + +While the ledger is expanded, the three reconciliation functions (and, with +status notifications, the two status guards) keep the earlier release's bodies: +each earlier-release boot re-creates them, and both releases' schema validators +compare function bodies, so the new release installs those bodies too. They are +correct for its writes while every generation is account-level, which the +`CHECK` guarantees. The new release's validators accept the expanded catalog +(`outbox/required_expand_schema.json`). + +A **subscription generation needs the contract step.** Before it, recording +one (a second offering of a configuration version) raises +`ReportingUpgradeNotContractedError` (a `LedgerConflictError`, code +`REPORTING_UPGRADE_NOT_CONTRACTED`) naming `python -m adcp migrate --contract`. + +**Contract** (migration `0007`, header `-- adcp-upgrade: contract`) is a +stopped-worker step that runs only on request: `python -m adcp migrate +--contract` (or `migrate(pool, contract=True)`), once every earlier-release +reporting process has stopped. Plain `python -m adcp migrate`, +`create_schema()` and `apply_pending()` stop before it. It drops the +four-column key and references, the six-column period key and the `CHECK`s, +renames the widened keys to their final names (`reporting_configurations_pkey`, +`reporting_obligations_period_key`), re-creates the offering-aware references +onto the new primary key and restores the new release's function bodies. The +ledger is then exactly a new ledger's catalog. On a ledger that was never +expanded (a new one, or one that 0.2.0b1/b2 already widened in place) it +changes nothing, and `python -m adcp migrate` records it without `--contract`. +`postgres_storage(...)` starts on a ledger whose only pending migration is the +contract step. A schema created by a release that widened the key but did not yet serve managed subscription delivery still carries an account-only `CHECK` on reconciliation records, materializer discovery and status scope checkpoints, and a four-column materializer discovery key. `create_schema()` drops those checks and widens the discovery key in place, losslessly and idempotently. -Earlier workers refuse the upgraded schema, so stop them first here too. +Earlier workers refuse the upgraded schema, so stop them first here. A foreign key outside the SDK that references `reporting_configurations` -refuses the upgrade with a maintenance error, rather than being dropped. -Re-point it at the widened key, then run the upgrade. The upgrade is -idempotent; a fresh schema is created in the widened layout directly. +refuses the expand (and the contract) with a maintenance error, rather than +being dropped. Re-point it at the widened key, then run the upgrade. Both +halves are idempotent. ### Status checkpoints keyed by offering (migration `0006`) @@ -97,18 +129,28 @@ obligation_namespace, offering_id)`. Account-level checkpoints keep `offering_id` `''`, so their keys, status `cause_id` values and projection history keys are unchanged. -This is also a **stopped-worker upgrade**: stop every reporting worker, run `python -m adcp migrate` (migration `0006 reporting_status_checkpoint_offerings`) -or the status, projection or production store's `create_schema()` once with the -new release, then restart the workers. Under the schema lock it widens the -checkpoint primary key; the status event tables (`reporting_status_notification_events`, -and `reporting_projection_notification_events` where the projection is -installed) gain `offering_id`, filled from their checkpoint, and their -checkpoint reference and per-generation unique key include it. Constraints -keep their names. It changes nothing on a schema without status -notifications. A foreign key outside the SDK that references -`reporting_status_scope_checkpoints` refuses the upgrade instead of being -dropped. +or the status, projection or production store's `create_schema()` applies it. +It follows the configuration key: + +- **Expanded ledger** (online-safe): the six-column + `reporting_status_scope_checkpoints_pkey`, the status events' six-column + checkpoint reference and seven-column per-generation key keep their names; + `reporting_status_scope_checkpoints_offering_key`, + `reporting_status_notification_events_offering_fkey` and + `reporting_status_notification_events_offering_key` are added beside them, + and the contract step (`0007`) renames them back. Earlier-release status + workers keep writing checkpoints and events. +- **Contracted ledger**: the key is widened in place under the schema lock and + constraints keep their names. + +The status event tables gain `offering_id`, filled from their checkpoint. The +status projection's event table (`reporting_projection_notification_events`, +an extension) is widened in place in both cases, so projection workers of an +earlier release stop with the first new boot; stop them first. It changes +nothing on a schema without status notifications. A foreign key outside the +SDK that references `reporting_status_scope_checkpoints` refuses the upgrade +instead of being dropped. ## Historical account-generation upgrade @@ -293,81 +335,92 @@ refuse that pair at startup instead (storage contract 1 versus 2; see `packages/adcp-reporting-storage/README.md`). adcp 10.0.0b4 and 10.0.0b5 predate the check and still install beside 0.2.0b0, so that pair fails every turn: install `adcp>=10.0.0b7` with `adcp-reporting-storage>=0.2.0b1` -(10.0.0b6 was not published). From 0.2.0b1, both schema entry points -upgrade an existing ledger in place: +(10.0.0b6 was not published). Both schema entry points upgrade an existing +ledger: - `PgReportingLedgerStore.create_schema()`, which `ReliableReportingService` runs at startup when built with `store=`; - `python -m adcp migrate`, which `postgres_storage(...)` requires to have run. -The upgrade (migration `0002`) adds `offering_id TEXT NOT NULL DEFAULT ''` to -the widened tables, so every existing account-level generation, obligation and -status keeps its identity with `offering_id = ''`; rebuilds the generation -primary key and the obligation period index with the new column; and re-points -the managed-delivery foreign keys. No evidence row is rewritten or dropped. The -applied versions are recorded in `adcp_schema_migrations` (`create_schema()` -applies the ledger's own `0001`–`0003`; `python -m adcp migrate` also applies -the later ones, which only `postgres_storage` and the notification stores -need), re-running is a no-op, -and concurrent boots serialize on the reporting schema advisory lock inside -one transaction. It is a stopped-worker upgrade: stop every worker of the -earlier release first, because their configuration writes name the -four-column key. +From this release the upgrade is an expand/contract pair (see [the widened +key](#reporting-subscription-generations-the-widened-key-adcp-33)). Expanding +(migrations `0002` and, with status notifications, `0006`) adds `offering_id +TEXT NOT NULL DEFAULT ''` to the widened tables, so every existing +account-level generation, obligation and status keeps its identity with +`offering_id = ''`, and adds the widened keys beside the ones the earlier +release uses. No evidence row is rewritten or dropped. The applied versions are +recorded in `adcp_schema_migrations` (`create_schema()` applies the ledger's +own `0001`–`0003`; `python -m adcp migrate` also applies the later ones, which +only `postgres_storage` and the notification stores need), re-running is a +no-op, and concurrent boots serialize on the reporting schema advisory lock +inside one transaction. Contracting (`0007`) waits for `python -m adcp migrate +--contract`. + +adcp-reporting-storage 0.2.0b1 and 0.2.0b2 widened the key in place instead, +as a stopped-worker upgrade. A ledger they already upgraded has no expand +state: this release treats it as contracted and records `0007` without +`--contract`. If you are still on 0.1.x/0.2.0b0, skip 0.2.0b1/b2 and upgrade +straight to this release. ### Deploying the upgrade This applies whenever more than one process runs Reliable Reporting against -the ledger. That includes several replicas that each start -`ReliableReportingService`, and a separate worker deployment. A default -rolling update does **not** stop the earlier release first. The first new -process upgrades the ledger when it starts, while the old processes keep -running against it. From then on, an old process: - -- still reads configurations, obligations, revisions and rows; -- cannot record a new generation (`put_configuration` fails with - `InvalidColumnReference: there is no unique or exclusion constraint matching - the ON CONFLICT specification`); -- cannot close the next period of an existing generation (`commit_obligation` - fails with `LedgerConflictError: the ledger write violated an integrity rule`, - because the period key it names was widened); -- with status notifications, cannot record status checkpoints, whose key - migration `0006` widens; -- cannot start Reliable Reporting again if it restarts. `create_schema()` - raises `Reporting caller ownership requires a stopped-worker maintenance - migration`, with a hint to use `migrate_legacy_reporting`. **Ignore that - hint.** `migrate_legacy_reporting` is the SDK 8 → 9 caller-ownership - migration. It does not apply to this upgrade and would move reporting state - into quarantine. - -Nothing is corrupted: the new release reads every record the old one wrote. -But the old processes' reporting stops for the rest of the rollout. - -**There is no rollback.** Once the ledger is upgraded, no earlier release can -start Reliable Reporting against it. To go back, restore the backup taken -before the upgrade; evidence written after the backup is lost. +the ledger: several replicas that each start `ReliableReportingService`, or a +separate worker deployment. **A default rolling update works.** While old and +new processes share the expanded ledger: + +- old processes (adcp 9.3.1 with adcp-reporting-storage 0.1.0b4, adcp + 10.0.0b2 with 0.2.0b0) keep starting, reading and writing account-level + generations: configurations, obligations, revisions, worker turns and, with + status notifications, status checkpoints and events; +- new processes work fully, except that they cannot record a reporting + subscription's generation until contract (`REPORTING_UPGRADE_NOT_CONTRACTED`); +- a rollback to the earlier release works until you contract. To deploy: 1. Back up the database. -2. Stop every reporting worker of the earlier release. Two ways: - - **Recreate.** Roll this one deployment out with every old pod stopped - before any new pod starts (Kubernetes `strategy: {type: Recreate}`). - Reporting, and the pods, are down for the length of the swap. - - **Two phases.** First redeploy the *earlier* release with Reliable - Reporting turned off (don't start `ReliableReportingService`), or scale - a separate worker deployment to zero. Then roll out the new release - normally. Only reporting pauses, not the rest of the service. -3. Upgrade the schema: run `python -m adcp migrate` once, or let the first new - process do it at startup. Run `python -m adcp migrate` (or - `ReportingStatusService.migrate()`) if you use `postgres_storage(...)` or - status notifications. `create_schema()` alone applies only `0001`–`0003`. -4. Start the new release everywhere. +2. If you use `postgres_storage(...)` or status notifications, run `python -m + adcp migrate` (for example as a pre-deploy job). It is safe while earlier + pods still run: it expands and stops before the contract step, printing + `not applied 0007 … (contract)`. Otherwise the first new process expands + the ledger when it starts. +3. Roll out the new release normally. +4. **Once every earlier-release reporting process has stopped** (the rollout + is complete and you no longer intend to roll back), run `python -m adcp + migrate --contract`. It takes exclusive locks on the generation, obligation + and status checkpoint tables for the length of one transaction. Until then + `python -m adcp migrate --status` ends with `up to date except the contract + step: run "python -m adcp migrate --contract" once every earlier-release + reporting process has stopped`, and `postgres_storage(...)` starts normally. + +After the contract step, an earlier-release process can no longer run against +the ledger: its `create_schema()` raises `Reporting caller ownership requires +a stopped-worker maintenance migration` (**ignore its hint**: +`migrate_legacy_reporting` is the SDK 8 → 9 caller-ownership migration, does +not apply here and would move reporting state into quarantine), and its +writes fail (`put_configuration` with `InvalidColumnReference`, +`commit_obligation` with `LedgerConflictError`). Reads still work. To roll back +after contract, restore the backup. + +**Extensions stay stopped-worker.** The materializer, production and status +projection stores widen their own tables and re-create their own functions +when the new release starts, as before. If an earlier release runs any of them, +stop those workers before the first new process starts (Recreate, or redeploy +the earlier release with those workers turned off first), then contract as +above. `tests/conformance/reporting/test_reporting_storage_published_upgrade_pg.py` seeds ledgers with the actual published wheel pairs (`adcp==9.3.1` with -`adcp-reporting-storage==0.1.0b4`, `adcp==10.0.0b2` with `0.2.0b0`) and proves -the retained evidence reads back unchanged, a restart rebinds the stored -generations, and the upgraded catalog is identical to a fresh migration. +`adcp-reporting-storage==0.1.0b4`, `adcp==10.0.0b2` with `0.2.0b0`) and drives +the rolling sequence, with and without status notifications: the new release +expands; old pods restart, read, write and run worker turns; new pods read the +old pods' rows and write reconciliation and status records, also after an old +pod restarts again; a subscription generation is refused until contract; the +contracted catalog (columns, constraints, indexes, triggers and function +definitions) is identical to a fresh migration; and the old pair then refuses. +It also covers a rollback before contract, concurrent old and new boots, and a +ledger 0.2.0b1 widened in place. ## Upgrading a ledger that adcp-reporting-storage 0.1.0b5 wrote diff --git a/packages/adcp-reporting-storage/README.md b/packages/adcp-reporting-storage/README.md index 622ea8cfd2..5775ce7a74 100644 --- a/packages/adcp-reporting-storage/README.md +++ b/packages/adcp-reporting-storage/README.md @@ -82,24 +82,38 @@ Install `adcp>=10.0.0b7` with this release. Both `PgReportingLedgerStore.create_schema()` (what `ReliableReportingService` runs at startup when given `store=`) and `python -m adcp migrate` upgrade an -existing ledger in place: `offering_id` is added to `reporting_configurations`, -`reporting_obligations`, `reporting_consumer_statuses` and the managed-delivery -tables, backfilled with `''` for every existing account-level generation, the -generation keys and unique indexes are widened, and the applied schema -versions are recorded in `adcp_schema_migrations` (`create_schema()` applies -and records the ledger's own `0001`–`0003`; `python -m adcp migrate` also -applies the later ones, which only `postgres_storage` and the notification -stores need). No evidence row is -rewritten or dropped. The upgrade is transactional, serialized under the reporting schema -advisory lock, and a no-op when re-run. It is a **stopped-worker** upgrade: -stop every reporting worker of the earlier release first (its configuration -writes name the four-column key), run the upgrade, then start the new release. -A default rolling update does not stop them. Old pods' reporting writes fail -once the first new pod has upgraded the ledger, and an old pod cannot start -reporting again. Downgrading is unsupported: restore the backup taken before -the upgrade. See -[Deploying the upgrade](https://github.com/adcontextprotocol/adcp-client-python/blob/next/docs/reporting-ledger-migration.md#deploying-the-upgrade) -for a Recreate or two-phase rollout. +existing ledger, and **a default rolling update works**. The upgrade is an +expand/contract pair: + +- **Expand** (migrations `0002` and `0006`, online-safe): `offering_id` is + added to `reporting_configurations`, `reporting_obligations`, + `reporting_consumer_statuses` and the managed-delivery tables, with `''` for + every existing account-level generation and `CHECK (offering_id = '')`. The + keys and references the earlier release uses keep their names, and the + widened ones are added beside them. Processes of the earlier release keep + starting, reading and writing account-level generations (with status + notifications too) while the new release rolls out, and a rollback works. + No evidence row is rewritten or dropped. The applied schema versions are + recorded in `adcp_schema_migrations` (`create_schema()` applies and records + the ledger's own `0001`–`0003`; `python -m adcp migrate` also applies the + later ones, which only `postgres_storage` and the notification stores need). + Each step is transactional, serialized under the reporting schema advisory + lock, and a no-op when re-run. +- **Contract** (migration `0007`, stopped-worker): run `python -m adcp migrate + --contract` once every earlier-release reporting process has stopped. Plain + `python -m adcp migrate` and `create_schema()` never contract, so the former + is safe as a pre-deploy job. The contracted ledger is exactly a new ledger's + catalog. Reporting subscriptions need it: before it, recording a + subscription's generation raises `ReportingUpgradeNotContractedError` (code + `REPORTING_UPGRADE_NOT_CONTRACTED`). After it, the earlier release cannot + start against the ledger, so rolling back means restoring the backup. + +`postgres_storage(...)` starts on a ledger whose only pending migration is the +contract step. A new ledger is created contracted. A ledger that 0.2.0b1 or +0.2.0b2 already widened in place counts as contracted. The materializer, +production and status projection extensions still widen their own tables when +the new release starts: stop their earlier-release workers first. See +[Deploying the upgrade](https://github.com/adcontextprotocol/adcp-client-python/blob/next/docs/reporting-ledger-migration.md#deploying-the-upgrade). A ledger written by 0.1.0b5 (the `(xid, seq)` account change feed, 255-character row binding IDs, the orphan collector fence) upgrades the same way, with no From bb4bac43bfc5b01a9a967edba7133dd715cb3433 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sun, 8 Nov 2026 03:59:44 +0000 Subject: [PATCH 3/3] fix(reporting-storage): expand status tables an earlier-release pod creates while the ledger is expanded An earlier-release process that turns status notifications on after the expand creates its own six-column status tables without offering_id. A later new boot with notifications then failed on the seven-column offering key, and the contract step failed too. The checkpoint-offerings DDL now adds offering_id and the five-column generation reference to any checkpoint table that lacks them, before the expand or in-place widening, and 0007 re-runs it first, so contract never meets a half-expanded table. The published-wheel tests cover the sequence for both pairs: seed without notifications, expand, old pod with notifications, then a new pod with notifications or the contract step directly, and a contracted catalog equal to a fresh install. Docs: scope the rolling-update headline to the ledger, outbox and status notifications; say the contract step can time out or deadlock against busy new pods and is atomic, so retry; and say an old pod running past contract can still touch subscription generations' lease columns, so stop it first. Co-Authored-By: Claude Opus 5.5 --- MIGRATION_v9_to_v10.md | 9 ++-- docs/reporting-ledger-migration.md | 21 +++++++- packages/adcp-reporting-storage/README.md | 5 +- .../reporting_status_checkpoint_offerings.sql | 23 +++++++++ ...007_reporting_contract_generation_keys.sql | 5 ++ ..._reporting_storage_published_upgrade_pg.py | 50 +++++++++++++++++++ tests/test_reporting_storage_facade.py | 2 + 7 files changed, 109 insertions(+), 6 deletions(-) diff --git a/MIGRATION_v9_to_v10.md b/MIGRATION_v9_to_v10.md index ec7af9837f..cb0fb3a796 100644 --- a/MIGRATION_v9_to_v10.md +++ b/MIGRATION_v9_to_v10.md @@ -74,7 +74,10 @@ adcp 10.0.0b0–b3 are 1): turn, so install `adcp>=10.0.0b7` ([#1837](https://github.com/adcontextprotocol/adcp-client-python/issues/1837)). - An existing SDK 9 ledger is upgraded by `create_schema()` or `python -m adcp - migrate`, and a default rolling update works. The upgrade first *expands* + migrate`, and a default rolling update works for the ledger, the + notification outbox and status notifications (not yet for the materializer, + production and status projection extensions: stop their SDK 9 workers + first). The upgrade first *expands* the ledger: `offering_id` is added with `''` for every existing account-level generation, obligation and status, the keys SDK 9 uses keep their names beside the widened ones, and the applied schema versions are @@ -83,6 +86,6 @@ adcp 10.0.0b0–b3 are 1): `python -m adcp migrate --contract`: reporting subscriptions need it (before it they raise `REPORTING_UPGRADE_NOT_CONTRACTED`), and after it SDK 9 can no longer run against the ledger. Plain `python -m adcp migrate` never - contracts. The materializer, production and status projection extensions - still need their SDK 9 workers stopped first. See + contracts. Stop every SDK 9 process before `--contract`, and retry + `--contract` if it times out against busy new pods (it is atomic). See [Deploying the upgrade](docs/reporting-ledger-migration.md#deploying-the-upgrade). diff --git a/docs/reporting-ledger-migration.md b/docs/reporting-ledger-migration.md index 05f226bd9e..601ebc10ca 100644 --- a/docs/reporting-ledger-migration.md +++ b/docs/reporting-ledger-migration.md @@ -366,8 +366,11 @@ straight to this release. This applies whenever more than one process runs Reliable Reporting against the ledger: several replicas that each start `ReliableReportingService`, or a -separate worker deployment. **A default rolling update works.** While old and -new processes share the expanded ledger: +separate worker deployment. **A default rolling update works for the ledger, +the notification outbox and status notifications; the materializer, production +and status projection extensions still need their earlier-release workers +stopped first** (below). While old and new processes share the expanded +ledger: - old processes (adcp 9.3.1 with adcp-reporting-storage 0.1.0b4, adcp 10.0.0b2 with 0.2.0b0) keep starting, reading and writing account-level @@ -394,6 +397,20 @@ To deploy: step: run "python -m adcp migrate --contract" once every earlier-release reporting process has stopped`, and `postgres_storage(...)` starts normally. + The contract step locks `reporting_configurations` first, then the tables + that reference it. New-release pods that keep writing during it can make + it wait out a lock or statement timeout, or lose a deadlock. It is atomic: + nothing changes, so run it again (at a quieter moment if needed). It first + completes the expand of any status tables an earlier-release pod created + while the ledger was expanded. + + Stop every earlier-release process **before** contracting, not just + because it cannot restart afterwards: one still running past the contract + step can still update the lease columns (`lease_worker_id`, + `lease_expires_at`) of a subscription generation through its four-column + lease `UPDATE`. That never touches evidence, but it can delay that + generation's period close until the lease expires. + After the contract step, an earlier-release process can no longer run against the ledger: its `create_schema()` raises `Reporting caller ownership requires a stopped-worker maintenance migration` (**ignore its hint**: diff --git a/packages/adcp-reporting-storage/README.md b/packages/adcp-reporting-storage/README.md index 5775ce7a74..af4d3f6152 100644 --- a/packages/adcp-reporting-storage/README.md +++ b/packages/adcp-reporting-storage/README.md @@ -82,7 +82,10 @@ Install `adcp>=10.0.0b7` with this release. Both `PgReportingLedgerStore.create_schema()` (what `ReliableReportingService` runs at startup when given `store=`) and `python -m adcp migrate` upgrade an -existing ledger, and **a default rolling update works**. The upgrade is an +existing ledger, and **a default rolling update works for the ledger, the +notification outbox and status notifications** (the materializer, production +and status projection extensions still need their earlier-release workers +stopped first, below). The upgrade is an expand/contract pair: - **Expand** (migrations `0002` and `0006`, online-safe): `offering_id` is diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_status_checkpoint_offerings.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_status_checkpoint_offerings.sql index f8fa57dd0a..725626954b 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_status_checkpoint_offerings.sql +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_status_checkpoint_offerings.sql @@ -77,6 +77,29 @@ BEGIN WHERE conrelid = 'reporting_configurations'::regclass AND conname = 'reporting_configurations_generation_key'); + -- Every checkpoint carries its generation's offering and references the + -- five-column configuration key. reporting_ledger_account_generations.sql + -- adds both to a checkpoint table that exists when it expands a ledger, but + -- a process of the earlier release can still create its own (six-column, + -- without offering_id) while the ledger is expanded: add them here too. + IF NOT EXISTS (SELECT 1 FROM pg_attribute + WHERE attrelid = 'reporting_status_scope_checkpoints'::regclass + AND attname = 'offering_id' AND NOT attisdropped) THEN + LOCK TABLE reporting_status_scope_checkpoints IN ACCESS EXCLUSIVE MODE; + ALTER TABLE reporting_status_scope_checkpoints + ADD COLUMN offering_id TEXT COLLATE "C" NOT NULL DEFAULT ''; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'reporting_status_scope_checkpoints'::regclass + AND conname = 'reporting_status_scope_checkpoints_generation_fkey') THEN + ALTER TABLE reporting_status_scope_checkpoints + ADD CONSTRAINT reporting_status_scope_checkpoints_generation_fkey + FOREIGN KEY (account_id, consumer_namespace, delivery_config_id, version, + offering_id) + REFERENCES reporting_configurations(account_id, consumer_id, delivery_config_id, + delivery_config_version, offering_id); + END IF; + IF expand THEN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = 'reporting_status_scope_checkpoints'::regclass diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0007_reporting_contract_generation_keys.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0007_reporting_contract_generation_keys.sql index ed84b8b06c..cfe617ed73 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0007_reporting_contract_generation_keys.sql +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/migrations/0007_reporting_contract_generation_keys.sql @@ -14,6 +14,11 @@ -- and create_schema() never apply it to an expanded ledger. On a ledger that -- was never expanded (a new one, or one that 0.2.0b1/b2 widened in place) it -- changes nothing, and "python -m adcp migrate" records it without --contract. +-- +-- It first completes the expand of status tables an earlier-release process +-- created while the ledger was expanded (re-running the checkpoint-offerings +-- DDL), so the contract never meets a half-expanded table. +-- include: ledger/reporting_status_checkpoint_offerings.sql -- include: ledger/reporting_ledger_contract.sql -- include: ledger/reporting_ledger_reconciliation.sql -- include: ledger/reporting_status_checkpoint_offerings.sql diff --git a/tests/conformance/reporting/test_reporting_storage_published_upgrade_pg.py b/tests/conformance/reporting/test_reporting_storage_published_upgrade_pg.py index 015b559296..fa9bd3f7b0 100644 --- a/tests/conformance/reporting/test_reporting_storage_published_upgrade_pg.py +++ b/tests/conformance/reporting/test_reporting_storage_published_upgrade_pg.py @@ -781,6 +781,56 @@ async def test_a_rolling_deploy_runs_published_and_new_pods_side_by_side_until_c assert late["steps"]["readback"] == "ok" # reads still work +@pytest.mark.parametrize("new_pod_first", [True, False], ids=["new-pod", "contract-directly"]) +async def test_status_tables_a_published_pod_creates_while_expanded_are_expanded_and_contracted( + published: tuple[str, Path], workspace: Path, new_pod_first: bool +) -> None: + """A published pod turns status notifications on only after the new release expanded. + + Its boot creates the status tables on its own six-column key, without + ``offering_id``. A later new pod with notifications, and the contract step + even without one, expand those tables first, so neither fails. + """ + from adcp_reporting_storage.ledger.pg import PgReportingLedgerStore + from adcp_reporting_storage.migrations import migrate + + from tests.conformance.reporting import _published_ledger_seed as seed + + _, python = published + async with isolated_reporting_pool(autocommit=True) as pool: + await _seed(python, pool, workspace, notifications=False) + await seed.create_schema(pool, PgReportingLedgerStore(pool=pool), notifications=False) + assert [item.version for item in await migrate(pool)] == _rest(contract=False) + assert _expand_only(await _catalog(pool)) == EXPANDED_LEDGER + old = await _old_pod(python, pool, workspace, notifications=True, ordinal=2, now=TURNS[0]) + _all_ok(old) + assert old["checkpoints"] > 0 + if new_pod_first: + await _new_pod( + pool, + notifications=True, + ordinal=3, + now=TURNS[1], + account="account-new-pod", + reconciliation_account="acct_new_pod", + ) + assert _expand_only(await _catalog(pool)) == EXPANDED_LEDGER | EXPANDED_STATUS + again = await _old_pod( + python, pool, workspace, notifications=True, ordinal=4, now=TURNS[2] + ) + _all_ok(again) + assert [item.version for item in await migrate(pool, contract=True)] == CONTRACT_VERSIONS + assert await _full_catalog(pool) == await _fresh_catalog(notifications=True) + await _new_pod( + pool, + notifications=True, + ordinal=5, + now=TURNS[3], + account="account-after-contract", + reconciliation_account="acct_after_contract", + ) + + async def test_a_rollback_before_contract_keeps_the_published_pair_working( published: tuple[str, Path], workspace: Path ) -> None: diff --git a/tests/test_reporting_storage_facade.py b/tests/test_reporting_storage_facade.py index a27aa651a9..052d14cc91 100644 --- a/tests/test_reporting_storage_facade.py +++ b/tests/test_reporting_storage_facade.py @@ -137,6 +137,8 @@ def test_migrations_are_numbered_files_with_their_upgrade_notes() -> None: # ... and the contract step runs only on request, once earlier releases stopped. assert contract.name == "reporting_contract_generation_keys" and contract.contract assert contract.includes == ( + # Completes the expand of status tables an earlier release created late. + "ledger/reporting_status_checkpoint_offerings.sql", "ledger/reporting_ledger_contract.sql", "ledger/reporting_ledger_reconciliation.sql", "ledger/reporting_status_checkpoint_offerings.sql",