Honor removed derived relation history

Author: TonisOrmisson

src/openstatspec/sql/catalog_verification.py

@@ -235,11 +235,19 @@ def verify_catalog_relations(
         ):
             _reject({workflow.transformation_profile_identity.name})
         owned_tables.update(workflow_tables)
+        physically_removed = set(connection.execute(select(
+            workflow.derived_dataset_disposition_event.c.derived_dataset_id
+        ).where(
+            workflow.derived_dataset_disposition_event.c.event_kind
+            == "physical_removed"
+        )).scalars())
         for row in connection.execute(select(
             workflow.derived_dataset.c.derived_dataset_id,
             workflow.derived_dataset.c.physical_relation_name,
             workflow.derived_dataset.c.output_mode,
         )).mappings():
+            if row["derived_dataset_id"] in physically_removed:
+                continue
             name = str(row["physical_relation_name"])
             if row["output_mode"] == "view":
                 owned_views.add(name)

tests/test_sql_workflow.py

@@ -500,6 +500,9 @@ def test_append_only_retire_remove_and_reconcile_protocol(catalog):
     assert openstatspec.reconcile_derived_removals(
         database_url=url,
     )["reconciled"] == 0
+    assert openstatspec.initialize_catalog(
+        database_url=url,
+    )["catalog"] == "verified"
 
 
 def test_sqlite_is_the_only_advertised_and_executable_workflow_backend():