From 9a7b94ba8be3544542ecc8765b571cc4395bfc5b Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:49:46 +0100 Subject: [PATCH 1/5] test(databricks): the stand-in deletes no schema that still holds a function, view or volume Unity Catalog deletes a schema only when it holds nothing: the 2026-11-03 Free Edition run measured that `tofu destroy` fails at a schema that still holds functions. The stand-in refused only for tables, so a schema holding a mask function or a seeded metric view was deleted, and a test could pass where a workspace fails. --- tests/iac/_fake_databricks.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/iac/_fake_databricks.py b/tests/iac/_fake_databricks.py index e2b52c4a..714a4391 100644 --- a/tests/iac/_fake_databricks.py +++ b/tests/iac/_fake_databricks.py @@ -18,7 +18,8 @@ refuses it), ALTER TABLE SET / DROP ROW FILTER, ALTER COLUMN SET / DROP MASK, and REVOKE SELECT. The bindings are served on the table (``row_filter``, ``columns[].mask``) as Unity Catalog serves them, and a CREATE OR REPLACE TABLE keeps them (Databricks docs: REPLACE preserves row filters -and column masks). The key constraints forge-cli declares (``databricks_keys.py``): ALTER TABLE +and column masks). A schema that still holds a table, a view, a function or a volume is not +deleted (``Schema is not empty``), as Unity Catalog refuses it. The key constraints forge-cli declares (``databricks_keys.py``): ALTER TABLE ADD CONSTRAINT ... PRIMARY KEY / FOREIGN KEY ... REFERENCES and DROP CONSTRAINT [IF EXISTS], served as ``table_constraints``, with the rules and error texts a warehouse showed (a key needs NOT NULL columns, a foreign key its parent's primary key, a referenced primary key does not @@ -1760,7 +1761,10 @@ def _delete_schema(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: full = m.group(1) if full not in self.schemas: return _err(404, "SCHEMA_DOES_NOT_EXIST", f"Schema '{full}' does not exist.") - if any(t.startswith(full + ".") for t in self.tables): + # Unity Catalog deletes a schema only when it holds nothing: no table or view, and no + # function or volume either (a schema holding functions measured live, 2026-11-03). + held = (*self.tables, *self.metric_views, *self.functions, *self.volumes) + if any(name.rsplit(".", 1)[0] == full for name in held): return _err(400, "INVALID_STATE", "Schema is not empty") del self.schemas[full] self.grants.pop(("schema", full), None) From 8933d01bea77beb9c5d8de7ea4ab413bcbb765db Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:49:46 +0100 Subject: [PATCH 2/5] fix(databricks): a schema the contract stops using is destroyed, not left failing the apply When a table moves to another schema, or a schema's last expose is removed, the plan destroys the old schema, and Unity Catalog refuses to delete a schema that still holds anything. The product's metric views and its column mask and row filter functions there are no OpenTofu resource, so `fluid apply` failed at that destroy ("Schema is not empty") until someone dropped them by hand. The schemas now come from the saved plan: each databricks_schema it only deletes that the contract no longer names, so only schemas this environment's state holds are looked in. - Before `tofu apply` (new plugin hook prepare_destroys), drop this product's metric views (fluid.product_id, owned by the deploy identity) whose table the plan destroys or is gone, and fluid_-shaped functions the deploy identity owns that no table of the schema uses. A function a table still uses is never dropped while the table exists: Unity Catalog names a mask's function in the table, so a gap would let anyone with CREATE FUNCTION on the schema plant one under that name. - OpenTofu destroys a table before its schema inside one apply, so such a schema still fails there. The engine then calls unblock_destroys, which drops the functions the destroyed tables used (no table uses them now), plans again, and applies only if that plan is nothing but those schema deletes (fail closed otherwise). - --mode dry-run prints the drops and the functions that wait, and runs none. Engine plumbing: runner.planned_deletes and runner.errored_resources (one event iterator shared with planned_removals), _apply_saved_plan and _applied out of _apply_via_opentofu, and one _run_hook for the optional plugin hooks, whose typed refusals map through _binding_error (now taking the event). Prior art: none for clearing objects outside state between a child's and a parent's destroy. force_destroy would delete data not in state; OpenTofu's -exclude needs 1.9 (the floor is 1.6.0); a destroy-time provisioner does not run once its block is removed (hashicorp/terraform#13549). --- CHANGELOG.md | 18 +- HONESTLY_TESTED.md | 3 +- fluid_build/cli/_apply_opentofu_engine.py | 184 +++++++-- fluid_build/iac/providers/databricks.py | 45 ++ .../iac/providers/databricks_enforce.py | 5 +- .../iac/providers/databricks_metrics.py | 12 +- .../iac/providers/databricks_vacated.py | 391 ++++++++++++++++++ fluid_build/iac/runner.py | 55 ++- tests/iac/test_iac_apply_engine_destroys.py | 274 ++++++++++++ tests/iac/test_iac_databricks_enforce.py | 21 +- tests/iac/test_iac_databricks_metrics.py | 3 +- .../test_iac_databricks_metrics_fake_e2e.py | 99 ++++- tests/iac/test_iac_databricks_vacated.py | 342 +++++++++++++++ 13 files changed, 1396 insertions(+), 56 deletions(-) create mode 100644 fluid_build/iac/providers/databricks_vacated.py create mode 100644 tests/iac/test_iac_apply_engine_destroys.py create mode 100644 tests/iac/test_iac_databricks_vacated.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b14efe3..777c4d76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,7 +135,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `tofu apply`, like the column masks: `CREATE VIEW` when it is missing, `ALTER VIEW ... AS` (which keeps its grants) when its definition changed, nothing when it did not, and `DROP VIEW`, before the apply, once the contract no longer declares it (in the schemas its - tables are in, so another environment of the product keeps its own). `--mode dry-run` prints + tables are in, or one the plan destroys, so another environment of the product keeps its + own). `--mode dry-run` prints every statement. What metric view YAML cannot express is refused at `fluid validate`, not approximated: derived metrics, a ratio with a filter, a non-additive dimension that is no time dimension, `hour` or `minute` on a `DATE`, colliding names, a name that is no column, and @@ -285,6 +286,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Databricks: a schema the contract stops using is destroyed, not left failing the apply.** + When a table moves to another schema, or a schema's last expose is removed, the plan destroys + the old schema, and Unity Catalog refuses to delete a schema that still holds anything. The + product's metric views and the functions of its column masks and row filters are no OpenTofu + resource, so `fluid apply` failed there ("Schema is not empty") until someone dropped them by + hand. Now, in each schema the saved plan destroys and the contract no longer names, it drops + this product's metric views over the tables the plan destroys before `tofu apply`, and the + mask and row filter functions once `tofu apply` has destroyed the tables that used them, then + finishes the schema's destroy with a second, destroy-only plan. A function a table still uses + is never dropped (Unity Catalog binds a mask by the function's name). Only the deploy + identity's objects go, a view only with this product's `fluid.product_id`, and only in schemas + this environment's own OpenTofu state holds, so another environment of the product keeps its + objects. Anything else the schema holds is named, with what to do. `--mode dry-run` prints the + drops and runs none. + - **Quality checks measured a constant on BigQuery (and would have on Databricks).** The completeness, uniqueness, accuracy and valid_values checks quoted their column the ANSI way, `"col"`, which BigQuery and Spark SQL read as a string literal, and cast ratios to diff --git a/HONESTLY_TESTED.md b/HONESTLY_TESTED.md index 90ee80e3..966ca9aa 100644 --- a/HONESTLY_TESTED.md +++ b/HONESTLY_TESTED.md @@ -92,7 +92,8 @@ Markers declared in `pyproject.toml`. | Stage 1 unit + manual live: contract DQ rules as dbt tests in a Databricks dbt job | `tests/util/test_dq_checks.py` (the reader), `tests/iac/test_iac_databricks_dbt_tests.py` (the tests, each rendered in a dbt-shaped Jinja context, at parse and at execute), `tests/build_runners/test_databricks_job_run.py` (the run's report) | ✅ 51 + 65 + 4. Every rule type read once (`util/dq_checks.py`): selectors split on commas; a `valid_values` rule read as `fluid test`'s `value_domain` reads it (the rule's own list, else the column's first `enum`, split on `|` and `,`, `regex` and `range`, and `labels.references` through `consumes[]`); shares below 1.0, the severity map, unique names; each refusal with its fix. Each check rendered: dbt's own `unique`, `not_null`, `accepted_values`, `relationships` through `adapter.dispatch`, forge-cli's regex, bounds, freshness, schema and share SQL; hostile values (`'`, `\\`, `}}`, `{%`, `#}`, a character outside the BMP) stay inside their Jinja string and SQL literal; every test refs its model while dbt parses, and a real dbt (dbt-core with dbt-duckdb) parses the upload and `dbt ls --select +orders+` lists every rule's test. Refused at plan: a test name the project's own tests use, a selection that leaves the model out, a threshold no float holds, an empty range, an unpaired surrogate (in a rule, a value or the test path; one from the project's YAML is named escaped in a refusal); an `anomaly_detection` rule is left to data quality monitoring (#864) unless the expose turns it off, and one monitoring could not read either (a bad severity) is still refused. The `fluid_keys` task (#860) runs with `run_if: ALL_DONE`, so a failing rule does not leave the table without its keys. **Live, Databricks Free Edition, 2026-11-12** (scratch catalog `fluid_iactest_dq`, swept after: job, upload folder, catalog; 0 left): one dbt product, 9 rules as 11 tests, `fluid apply --mode amend-and-build` (`tofu` +27). Run 1, with a seeded bad row, ended `RUN_EXECUTION_ERROR`: completeness on `status` and the `amount` range `FAIL 1` (error), the country regex and the customer reference `WARN 1`, six `PASS`, each printed by `fluid apply`; it also showed the schema test was never selected (its `ref` sat behind `if execute`: dbt found 11 tests, ran 10), fixed. Run 2, clean: `SUCCESS`, the model and all 11 tests `PASS`; the next plan `+0 ~0 -0`. Offline: the demo's 14 dbt products compile to 230 tests with no refusal, and `dbt parse` / `dbt ls --select +model+` (dbt-core 1.12.5) select all 20 of DP005. | A share below 1.0 that fails, a reference share, and a regex or bounds share were not run live (only a passing completeness share). A Git-sourced build with rules is refused by design (no file can be added to a checkout). Not handled: a versioned model, and a `dbt_project.yml` that sets `+where` or `+limit` on tests, which would change the generated tests too. dbt's result lines are read from the task's output: lines Databricks cut from a long output are reported as not in it. The run page was read through the Jobs API (`runs/get-output`), not in a browser. | | Stage 1 unit: Databricks retention (`lifecycle.expire`) | `tests/iac/test_iac_databricks_retention.py`, `tests/cli/test_verify_databricks_retention.py` | ✅ 195 + 18. ISO 8601 durations (years and months at their shortest, P1Y = 365 and P1M = 28 days; a fraction, a sign, a lower-case letter, `PT`, `P1DT`, out-of-order and over-long numbers refused), Delta `interval` properties (rounded up, months refused) and the history as the larger of the two properties with Delta's defaults. A dbt model's `materialized` resolved from `config()` in the SQL, the model's YAML `config:` and `dbt_project.yml` (the `+` prefix or none, root, project, folder and model-name levels, the most specific wins, SQL over YAML over project), the default `view`, the model found by alias or file name, commented-out and `raw` configs ignored, a `config()` inside a Jinja block, `var()`, `**`, a custom schema or database, a Python model, a versioned or twin model, a linked file and a Git repository refused as undeterminable. The build's `--select` must include the resolved model (name, `fqn`, `path:`, `*`, unions and intersections; a graph operator on another node, a tag, a config or run state is refused as not evaluable). Accepted: a dbt `table` model and a DuckDB acquisition with P30D, P365D, P6Y; refused: P7D, P29D, P4W, P1M, a history set longer than the retention, `incremental`, `snapshot`, `view`, another acquisition engine, no writer, a mixed set of writers. `fluid validate` and the emitter agree on kind and message. `fluid verify`'s retention dimension against a stubbed client and, over HTTP, the stand-in workspace (pass, fail with the `ALTER TABLE`, the runtime property over the declared one, skip, predictive optimization off as a warning). | Live behaviour: no real Databricks endpoint was called. Whether Unity Catalog's GetTable serves a table's `delta.logRetentionDuration` and `delta.deletedFileRetentionDuration` in `properties` or `delta_runtime_properties_kvpairs` (read from databricks-sdk-go v0.190.0 `model.go`, not observed), whether `effective_predictive_optimization_flag` is filled for a managed table, that predictive optimization runs `VACUUM` within the retention (a Databricks guarantee, not tested), and that dbt-databricks `table` replaces the table each run. Delta history is only half the promise: the acquisition acceptance rests on the landing being an `INSERT OVERWRITE` of the whole table in the one mode it runs (`full_refresh`, read from the landing's own `LOAD_MODES`; another source mode is refused as `incremental`, none as `unmapped`), which the bronze landing row covers against the stand-in, not live. A GetTable answer with no properties is a verify warning (Delta's defaults assumed). Embedded-logic DuckDB builds, `dlt`, `airbyte` and the other acquisition engines are refused. Selection is matched on names and paths only, not on dbt's graph: `+other+` is refused even if `other` is upstream of the model. | | Stage 1: Databricks Unity Catalog metadata (comments, product tags, column tags; live: the 2026-11-04 row) | `tests/iac/test_iac_databricks_metadata.py` (the emitter), `tests/iac/test_iac_databricks_metadata_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against `tests/iac/_fake_databricks.py`), `tests/cli/test_verify_databricks.py` (the comment and column-tag cases) | ✅ 43 + 5 + 7. The table's `comment` is the expose's description, else the product's (its `purpose` when structured), each described column's `comment` its schema description, stripped; none on a table a dbt build writes (dbt's CREATE OR REPLACE owns them; `fluid validate` warns). The product's identity and owner team are tags on the table only (`fluid_product_id`, `fluid_product_name`, `fluid_domain`, `fluid_layer`, `fluid_product_type`, `fluid_product_version`, `fluid_version`, `fluid_owner_team`), an unstated field makes none, and they count with the governance labels toward 50; `lifecycle.state: deprecated` (the expose's, else the product's) sets `system.certification_status = deprecated`, else the label `certification: certified` (the expose's, else the product's) sets `certified` and is no plain tag (a product-level label certifies every expose's table, as intended; an expose-level one only its own); another label value is `databricks-certification-invalid`; a plan notice says the identity needs ASSIGN on that governed tag. Against the stand-in: certified, then deprecated in place, then removed. A classified column (`sensitivity`) carries `fluid_sensitivity` as a `columns` tag named `catalog.schema.table.column`, imported best-effort by the provider's id. Refused through `validate` and `emit` alike: a backslash in any comment (`databricks-comment-unsafe`; the provider's `parseComment` escapes only quotes) and a `${` (Spark variable substitution), a label meeting a product key (`databricks-tag-collision`), a value over 256 characters (`databricks-tag-value-too-long`), over 1,000 column tags (`databricks-tags-too-many`); a `${...}` in a tag value is rendered as a literal. Against the stand-in: apply writes the comments (an apostrophe included, through the provider's escaping), the product tags and the column tag, never an `OWNER TO`; a second plan is empty; a changed description is an in-place `COMMENT ON TABLE`, never a replace; removing a classification destroys only its column tag; a table made before this is adopted and gains its comments in place (`+5 ~1` for the product tags and the table update). `fluid verify` passes after the apply and fails a comment or column tag changed by hand; against a stubbed client it reads each column's tag list to its last page, ignores a tag or comment someone else added, and checks no comment on a dbt-written table. | **Not run live.** What real Unity Catalog stores and returns for a table or column comment (whitespace, newlines, length limits, which the docs do not state), whether it accepts a tag value with `, . : / = -` (the docs restrict only keys), whether `entity-tag-assignments/columns/{catalog.schema.table.column}` resolves as the SDK documents, and whether a dbt CREATE OR REPLACE keeps column tags (undocumented; `fluid verify` is the gate), and whether the Entity Tag Assignments API sets the system tag `system.certification_status` (its docs show the UI and `SET TAG`; the SDK documents governed tags through the API, needing ASSIGN; the stand-in checks no permission). A contract that adds a column while comments change for the first time hits the provider's mixed-change refusal (its fix-it says to apply in two steps). | -| Stage 1: Databricks metric views (`exposes[].semantics`) | `tests/iac/test_iac_databricks_metrics.py` (the YAML, the refusals and the hooks against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_databricks_metrics_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against the stand-in) | ✅ 91 + 4. A site-360-shaped semantics block compiles to the metric view YAML it should (the entity as its key column and categorical and time dimensions as dimensions, with comments, display names and synonyms from the columns; non-additive measures as `semiadditive: last` windows; filtered simple metrics as `FILTER (WHERE ...)`; a ratio as `MEASURE(a) / NULLIF(MEASURE(b), 0)`, after its inputs); the view is `_semantic_model` whatever the model is called; every column reference is `source.` (`
.` too), a unit or type keyword (`EXTRACT(YEAR ...)`, `INTERVAL 1 DAY`, `::date`) stays one; the time grains (`TRUNC` on a `DATE`, `DATE_TRUNC` otherwise); 42 refusal cases (names, collisions, expressions, quoting, calls that would lend the owner's rights, text limits). The hooks: a first apply creates the view with its `fluid.*` properties and grants it to the table's readers; a second runs nothing; a changed or hand-edited definition is an `ALTER VIEW` that keeps the grants; `SELECT` and `ALL PRIVILEGES` are revoked from anyone else, before the apply and first in it, so a failed grant or update leaves no removed reader; a view the contract no longer declares, in a schema the product's tables are in, is dropped before the apply, another product's is left and another environment's (same id, another schema of the catalog) is kept; a table, a view without this product's property or one another identity owns at the name is refused before the apply; a failed statement is `databricks-metric-view-failed` naming the view; without databricks-sdk a declared view is refused; when a restricted table's binding fails, `SELECT` on its views is revoked. Through the CLI: apply, a second apply `+0 ~0 -0` with no statement, dry-run prints and runs nothing, dropping the semantics drops the view, `fluid validate` refuses a derived metric. All 19 industry-model tower-management contracts and the 4 forge templates that bind a table compile with no refusal. | What Databricks accepts and answers (the 2026-11-12 live row); Genie and AI/BI dashboards themselves; a view left in a schema the contract stops using (its table moved, its last expose removed) is not found, and OpenTofu's destroy of that schema then fails until it is dropped by hand, as for the column masks' functions. | +| Stage 1: Databricks metric views (`exposes[].semantics`) | `tests/iac/test_iac_databricks_metrics.py` (the YAML, the refusals and the hooks against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_databricks_metrics_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against the stand-in) | ✅ 91 + 4. A site-360-shaped semantics block compiles to the metric view YAML it should (the entity as its key column and categorical and time dimensions as dimensions, with comments, display names and synonyms from the columns; non-additive measures as `semiadditive: last` windows; filtered simple metrics as `FILTER (WHERE ...)`; a ratio as `MEASURE(a) / NULLIF(MEASURE(b), 0)`, after its inputs); the view is `
_semantic_model` whatever the model is called; every column reference is `source.` (`
.` too), a unit or type keyword (`EXTRACT(YEAR ...)`, `INTERVAL 1 DAY`, `::date`) stays one; the time grains (`TRUNC` on a `DATE`, `DATE_TRUNC` otherwise); 42 refusal cases (names, collisions, expressions, quoting, calls that would lend the owner's rights, text limits). The hooks: a first apply creates the view with its `fluid.*` properties and grants it to the table's readers; a second runs nothing; a changed or hand-edited definition is an `ALTER VIEW` that keeps the grants; `SELECT` and `ALL PRIVILEGES` are revoked from anyone else, before the apply and first in it, so a failed grant or update leaves no removed reader; a view the contract no longer declares, in a schema the product's tables are in, is dropped before the apply, another product's is left and another environment's (same id, another schema of the catalog) is kept; a table, a view without this product's property or one another identity owns at the name is refused before the apply; a failed statement is `databricks-metric-view-failed` naming the view; without databricks-sdk a declared view is refused; when a restricted table's binding fails, `SELECT` on its views is revoked. Through the CLI: apply, a second apply `+0 ~0 -0` with no statement, dry-run prints and runs nothing, dropping the semantics drops the view, `fluid validate` refuses a derived metric. All 19 industry-model tower-management contracts and the 4 forge templates that bind a table compile with no refusal. | What Databricks accepts and answers (the 2026-11-12 live row); Genie and AI/BI dashboards themselves. A schema the contract stops using: the next row. | +| Stage 1: Databricks, a schema the contract stops using | `tests/iac/test_iac_databricks_vacated.py` (the rules, against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_apply_engine_destroys.py` (the engine, `tofu` stubbed), 2 of `tests/iac/test_iac_databricks_metrics_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against the stand-in) | ✅ 9 + 19 + 2. A table with a metric view, a mask and a row filter moves from `main.sales` to `main.other`: `--mode dry-run` prints the `DROP VIEW` before the apply and the `DROP FUNCTION`s after the table's destroy, and runs none; the apply drops the view before `tofu apply`, `tofu apply` destroys the old table and is refused the schema (the stand-in, like Unity Catalog, deletes no schema that holds a table, view, function or volume), the functions are dropped once no table uses them, and a second, destroy-only plan (`+0 ~0 -1`) destroys the schema; the run exits 0 with the first plan's totals, and the next apply is `+0 ~0 -0`. Removing a schema's last expose does the same. Only schemas the saved plan destroys and the contract does not name are looked in, so a schema the contract still uses and another environment's (same product id, same identity, another schema) are never read. In the destroyed schema, a view of another product, one another identity owns and one over a table the plan keeps are left, and so are a function a kept table uses, one another identity owns and one of another name; an unused function of this module's name shape that the deploy identity owns is dropped; each object left is named with why. A schema that cannot be read drops nothing; a failed drop, a plugin that clears nothing or raises, a failure the plan's deletes do not account for, and a second plan holding any other change all leave the apply's own failure. | Live Databricks: that the provider destroys the table before its schema and Unity Catalog then deletes the schema once its functions are dropped (the 2026-11-03 run measured only that a schema holding functions is not deleted); whether a function re-created under a dropped one's name would be bound again (why a function a table still uses is never dropped); a table in another schema whose mask names a function here; a volume or a model in the schema (neither dropped nor named). | | Manual live, Free Edition (founder-approved), 2026-11-12, metric views | the real `fluid` CLI, no test file: this branch, databricks-sdk 0.166.0, a scratch catalog `fluid_iactest_metrics` dropped `CASCADE` after | ✅ The semantics block of industry-model `dp019_site_360` verbatim (13 dimensions, 3 non-additive measures, 4 filtered metrics) on a table shaped like it: dry-run printed the `CREATE VIEW` and grants; apply `+77`, then the view created and `SELECT` granted to the 3 reader principals; `SELECT country_code, MEASURE(site_count), MEASURE(avg_tenancy_ratio), ... GROUP BY ALL` returned each market at its latest snapshot (8 rows seeded, 4 sites x 2 days); by date, each day's own values; a second apply `+0 ~0 -0` ran no statement. `information_schema.tables` lists it as `METRIC_VIEW` with its comment, `.columns` every dimension and measure with its comment, `.table_privileges` the 3 `SELECT` grants. Probes on the same warehouse (Databricks SQL 2026.45): `CREATE OR REPLACE VIEW` drops a view's grants and `ALTER VIEW ... AS` keeps them and its properties; the stored `view_definition` leaves out `comment`, `display_name` and `synonyms`; a dimension named like a column shadows it in a measure's filter and a window measure named like a column breaks the window measures that read it (`METRIC_VIEW_WINDOW_MEASURE_REFERENCES_WINDOW_MEASURE`), and `source.` reads the column; `semiadditive: last` is per output group; `FILTER (WHERE ...)` works on a window measure, `COUNT(DISTINCT ...)` and `PERCENTILE_CONT ... WITHIN GROUP`; `TBLPROPERTIES` combine with `WITH METRICS`; a metric view over a row-filtered table applies the filter to the reader; a table at the view's name fails `EXPECT_VIEW_NOT_TABLE`. | A query as a reader persona (the run had the deploy user's token only; the persona's effective `SELECT` was read from the permissions API); Genie and dashboards; a contract with column masks. | | Stage 1: Databricks data quality monitoring (live: the 2026-11-12 row) | `tests/iac/test_iac_databricks_quality.py` (what asks for it, the monitor per schema, the derived import, the notice, the engine), `tests/iac/test_iac_databricks_quality_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_tofu_validate.py::test_a_databricks_data_quality_monitor_passes_tofu_validate` | ✅ 37 + 6 + 1. A freshness or completeness SLO (`qos.freshnessSLO`, `qos.completenessTarget`, an `observability.defaultSLIs` freshness or completeness block, a `contract.dq.rules` entry of type `freshness`, `completeness` or `anomaly_detection`) puts one `databricks_data_quality_monitor` on the owned schema, watching it by `schema_id`, created after the schema's tables, the product's other tables there in `excluded_table_full_names`; `contract.dq.monitoring.enabled: false` removes only the monitor; a `dq.monitoring.window` other than one day is a `fluid validate` warning. Against the stand-in: a second plan is empty; a monitor turned on outside forge-cli is adopted by `schema,` taken from state after the schema's import (no create); the last exclusion leaving is a PATCH whose `update_mask` names the whole config (the stand-in honours the mask), and the plan converges; a monitor Databricks refuses fails the apply after the tables, its suggestions saying what it needs and naming the opt-out (`apply_failure_hints`, after the three generic ones); opting out deletes it, not gated; `tofu destroy` deletes it before the schema. | What Databricks scans and reports (the live row); the stand-in serves schema monitors only. | | Manual live, Free Edition (founder-approved) | the real `fluid` CLI, no test file: run 2026-11-01 from main dfd6f145, databricks-sdk 0.158.0, every `fluid_iactest_*` object swept after (1 schema, 1 table, 1 folder; 0 left) | ✅ validate, `generate iac --validate`, plan, dry-run and apply `+9`; a second plan is EMPTY against the real Jobs API; Databricks accepts the Quartz schedule, serverless environment 2 with `dbt-databricks>=1.8.0,<2.0.0` (dbt 1.12.6, adapter 1.13.0), the WORKSPACE `project_directory` and the commands as emitted; one `run-now` ends SUCCESS and dbt builds the table; `fluid verify` with the real SDK (columns, types, one job); `fluid diff`'s state check (its live check has no Databricks inspector); a model change `~1`; the build made local removes job, files and folders `-5` without `--allow-data-loss` (the empty `.fluid//` folders above `files/` stay: only `files/` and below are managed). It found the USE CATALOG check reading a group-conveyed privilege as none. After the fix, `fluid verify` on a fresh one-table product (`+4`) passes every dimension, catalog access included (1 schema and 1 table swept, 0 left). | A GIT source, profile or OAuth auth, a run-as service principal, table adoption and a larger project, live; the region residency check (the metastore precondition and `fluid verify`'s residency dimension), which was added after this run and not run live; dbt's CREATE OR REPLACE drops a contract's NOT NULL, so verify flags a `required` column a dbt model does not constrain itself. | diff --git a/fluid_build/cli/_apply_opentofu_engine.py b/fluid_build/cli/_apply_opentofu_engine.py index e9266f9e..8449364d 100644 --- a/fluid_build/cli/_apply_opentofu_engine.py +++ b/fluid_build/cli/_apply_opentofu_engine.py @@ -302,8 +302,10 @@ def _apply_via_opentofu( _enforce_data_loss_gate(args, logger, provider, changes, data_changes, lost_in_place) + destroyed = _planned_destroys(plugin, changes, str(workdir), env) if dry_run: _enforce_access(plugin, "pre_apply", contract, dry_run=True) + _prepare_destroys(plugin, contract, destroyed, dry_run=True) _converge_output_ports(plugin, contract, dry_run=True) cprint("\ndry-run: plan only — not applying.") info(logger, "opentofu_apply_dry_run", provider=provider, **changes) @@ -313,6 +315,7 @@ def _apply_via_opentofu( # grant the apply adds lands on a protected table. A failure stops the run here: # nothing has been applied. _enforce_access(plugin, "pre_apply", contract) + _prepare_destroys(plugin, contract, destroyed) imported.keep() # ``fluid apply`` is the headline state-changing command, so a receiver @@ -322,7 +325,7 @@ def _apply_via_opentofu( # (``resolve_lineage_emitter`` returns the null emitter), and every # emit call soft-fails — lineage never breaks an apply. with _apply_lineage(contract, provider=provider, planned=changes) as record: - apply_result = runner.tofu_apply(str(workdir), env=env) + apply_result, waited = _apply_saved_plan(plugin, contract, destroyed, str(workdir), env) if not apply_result.ok: # A table the failed apply created, its grants and the job that fills it can all # exist with no mask or filter: bind what exists (never drop) before failing. @@ -332,7 +335,7 @@ def _apply_via_opentofu( _tail(apply_result.stderr or apply_result.stdout), ) raise _apply_failure(apply_result, str(workdir), plugin) - applied = runner.change_summary(apply_result) + applied = _applied(changes, apply_result, waited) record(applied) cprint(f"\n tofu apply complete: +{applied['add']} ~{applied['change']} -{applied['remove']}") @@ -376,19 +379,25 @@ def _converge_output_ports( and the run exits non-zero; the tables the apply made stay, and the next apply converges the space. With ``dry_run`` the plugin prints what it would do and writes nothing. """ - run = getattr(plugin, "converge_output_ports", None) + # Its lines quote titles (anyone's, from the Genie list) and the ``[fluid-genie:...]`` + # marker, which Rich would read as markup. + _run_hook( + plugin, "converge_output_ports", "opentofu_output_port_failed", contract, dry_run=dry_run + ) + + +def _run_hook(plugin: Any, hook: str, event: str, *args: Any, **kwargs: Any) -> None: + """Run the plugin's optional ``hook``, if it has one, with what it prints going out + literally: it quotes the contract and the server (statements, names, a server's error text + with its ``[ERROR_CLASS]``), data, not markup, so a "[word]" in it neither vanishes nor + raises. A typed refusal is the run's error (:func:`_binding_error`, under ``event``).""" + run = getattr(plugin, hook, None) if not callable(run): return try: - # Literally: its lines quote titles (anyone's, from the Genie list) and the - # ``[fluid-genie:...]`` marker, which Rich would read as markup. - run(contract, say=cprint_literal, dry_run=dry_run) + run(*args, say=cprint_literal, **kwargs) except UnsupportedBindingError as exc: - raise CLIError( - 1, - "opentofu_output_port_failed", - {"kind": exc.kind, "error": str(exc), "remediation": list(exc.remediation)}, - ) from exc + raise _binding_error(exc, event) from exc def _enforce_access( @@ -403,22 +412,139 @@ def _enforce_access( plugin prints the statements and runs none. What a hook prints is data (table names, statements, a server's error text with its ``[ERROR_CLASS]``), so it goes out literally. """ - run = getattr(plugin, hook, None) - if not callable(run): - return - # What the hooks print quotes the contract and the server (statements, descriptions, errors): - # data, not markup, so a "[word]" in it neither vanishes nor raises. + # ``post_apply`` takes no ``dry_run``: it never runs in one. + flags = {"dry_run": True} if dry_run else {} + _run_hook(plugin, hook, "opentofu_access_enforcement_failed", contract, **flags) + + +def _planned_destroys( + plugin: Any, changes: Mapping[str, int], workdir: str, env: Mapping[str, str] +) -> List[Dict[str, Any]]: + """What the saved plan only deletes (``runner.planned_deletes``), for the plugin's + ``prepare_destroys`` and ``unblock_destroys``. ``[]`` for a plugin without + ``prepare_destroys`` or a plan that removes nothing; a plan that cannot be read back is + said, and nothing is cleared before the apply.""" + if not callable(getattr(plugin, "prepare_destroys", None)) or not changes.get("remove"): + return [] + shown = runner.tofu_show_plan(workdir, plan_file="tfplan", env=env) + if shown is None: + cprint( + " note: the saved plan could not be read back, so what the provider plugin made " + "outside OpenTofu in the resources it destroys is not cleared first." + ) + return [] + return runner.planned_deletes(shown) + + +def _prepare_destroys( + plugin: Any, + contract: Mapping[str, Any], + destroyed: List[Dict[str, Any]], + *, + dry_run: bool = False, +) -> None: + """Run the plugin's ``prepare_destroys`` hook, if it has one, before ``tofu apply``. + + Optional plugin capability, discovered like ``pre_apply``: the Databricks plugin drops what + it made outside OpenTofu (metric views, column mask and row filter functions) in the + schemas the plan destroys, which Unity Catalog would otherwise refuse to delete + (``iac/providers/databricks_vacated.py``). It gets the plan's deletes + (:func:`_planned_destroys`). With ``dry_run`` it prints the statements and runs none. What + it prints quotes live objects, so it goes out literally; a refusal stops the run before the + apply. + """ + if destroyed: + _run_hook( + plugin, + "prepare_destroys", + "opentofu_destroy_preparation_failed", + contract, + destroyed, + dry_run=dry_run, + ) + + +def _apply_saved_plan( + plugin: Any, + contract: Mapping[str, Any], + destroyed: List[Dict[str, Any]], + workdir: str, + env: Mapping[str, str], +) -> Tuple[Any, int]: + """``tofu apply`` of the saved plan, with the destroys it could not make yet resumed + (:func:`_resume_blocked_destroys`): the result that stands, and how many destroys waited + for a second apply (0 when none did).""" + result = runner.tofu_apply(workdir, env=env) + if result.ok: + return result, 0 + return _resume_blocked_destroys(plugin, contract, result, destroyed, workdir, env) or ( + result, + 0, + ) + + +def _applied(changes: Mapping[str, int], result: Any, waited: int) -> Dict[str, int]: + """What the apply made: its summary, or after a resume the plan's own changes (the first + apply made all but the destroys that waited, the second plan held nothing else), with the + second apply's destroys in place of those that waited.""" + applied = runner.change_summary(result) + if not waited: + return applied + return {**changes, "remove": int(changes["remove"]) - waited + applied["remove"]} + + +def _resume_blocked_destroys( + plugin: Any, + contract: Mapping[str, Any], + result: Any, + destroyed: List[Dict[str, Any]], + workdir: str, + env: Mapping[str, str], +) -> Optional[Tuple[Any, int]]: + """Finish the destroys a ``tofu apply`` could not make yet: ``(the second apply's result, + how many waited)``, or ``None`` and the apply's own failure stands. + + OpenTofu destroys a resource and then what holds it, inside one apply. Unity Catalog keeps + the functions of a table's column masks and row filter until the table is gone, and refuses + to delete a schema that holds them, so a schema the contract stopped using fails to destroy + in the very apply that destroyed its tables. Optional plugin capability + ``unblock_destroys(contract, blocked, say) -> bool``: the Databricks plugin drops those + functions once no table uses them, and says what it dropped + (``iac/providers/databricks_vacated.py``). Fails closed: it resumes only when every + resource the apply failed on is a delete of the saved plan and no error is left + unaccounted for, the plugin cleared something, and the plan made after it holds nothing but + some of those deletes, which the data-loss gate already passed. + """ + unblock = getattr(plugin, "unblock_destroys", None) + errored = runner.errored_resources(result) + planned = {entry["address"]: entry for entry in destroyed} + if not callable(unblock) or not errored or len(_diagnostics(result.events)) != len(errored): + return None # nothing to resume, or an error no failed resource accounts for + if any(action != "delete" or address not in planned for address, action in errored): + return None + waited = sorted({address for address, _action in errored}) try: - if dry_run: - run(contract, say=cprint_literal, dry_run=True) - else: - run(contract, say=cprint_literal) - except UnsupportedBindingError as exc: - raise CLIError( - 1, - getattr(exc, "event", None) or "opentofu_access_enforcement_failed", - {"kind": exc.kind, "error": str(exc), "remediation": list(exc.remediation)}, - ) from exc + cleared = unblock(contract, [planned[address] for address in waited], say=cprint_literal) + except Exception as exc: # noqa: BLE001 — the apply's own failure is what the run raises + cprint_literal( + f" what holds {', '.join(waited)} could not be cleared: {type(exc).__name__}" + ) + return None + if not cleared: + return None + plan = runner.tofu_plan(workdir, env=env) + removals = {address for address, _kind in runner.planned_removals(plan)} + only_those = {"add": 0, "change": 0, "remove": len(removals)} + if not (plan.ok and removals and removals <= set(waited)) or ( + runner.change_summary(plan) != only_those + ): + cprint_literal( + " not destroyed again: the plan made after the apply holds more than " + + ", ".join(waited) + ) + return None + cprint(f" tofu plan (the destroys that waited): +0 ~0 -{len(removals)}") + return runner.tofu_apply(workdir, env=env), len(waited) def build_after_apply( @@ -740,10 +866,12 @@ def emit_module( return module, actions -def _binding_error(exc: UnsupportedBindingError) -> CLIError: +def _binding_error(exc: UnsupportedBindingError, event: str = "unsupported_binding") -> CLIError: + """A typed refusal as the run's error, under ``event``, or the refusal's own when its class + names one (a Genie space's ``opentofu_output_port_failed``).""" return CLIError( 1, - "unsupported_binding", + getattr(exc, "event", None) or event, {"kind": exc.kind, "error": str(exc), "remediation": list(exc.remediation)}, ) diff --git a/fluid_build/iac/providers/databricks.py b/fluid_build/iac/providers/databricks.py index b2cf333b..687c0299 100644 --- a/fluid_build/iac/providers/databricks.py +++ b/fluid_build/iac/providers/databricks.py @@ -106,6 +106,13 @@ values pick the tables and are not thresholds the monitor checks. Its results are the Quality indicator on the table in Catalog Explorer and ``system.data_quality_monitoring.table_results``; removing it is not gated. +* A schema the contract stops using (its table moved, its last expose removed) is destroyed by + the plan. The metric views and the mask and row filter functions this product made there are + no OpenTofu resource, and Unity Catalog does not delete a schema that holds them, so + :meth:`DatabricksIacPlugin.prepare_destroys` drops them before ``tofu apply`` and + :meth:`~DatabricksIacPlugin.unblock_destroys` drops the functions the apply's destroyed tables + used, once they are gone, before the engine resumes that schema's destroy + (``databricks_vacated.py``). * A dbt build that runs on Databricks (``execution.runtime.platform: databricks``) becomes a serverless ``databricks_job`` with a ``dbt_task``, and a local project its ``databricks_workspace_file`` uploads @@ -205,6 +212,7 @@ from ..provider_match import canonical_cloud, is_cloud from ..versions import required_providers from . import databricks_metrics as _metric_views +from . import databricks_vacated as _vacated from .databricks_access import TableEnforcement, compile_enforcement from .databricks_auth import tofu_env_overlay from .databricks_enforce import ( @@ -2537,6 +2545,31 @@ def post_apply( _declare_keys(contract, say, client_factory) _metric_views.post_apply(views, say, client_factory) + def prepare_destroys( + self, + contract: Mapping[str, Any], + destroyed: Iterable[Mapping[str, Any]], + say: Optional[Callable[[str], None]] = None, + dry_run: bool = False, + client_factory: Optional[Callable[[], Any]] = None, + ) -> None: + """Before ``tofu apply``: clear what this product made outside OpenTofu in the schemas + the plan destroys (``databricks_vacated.prepare``); ``dry_run`` prints, runs none.""" + _vacated.prepare( + _vacated_of(contract, destroyed), say or print, client_factory, dry_run=dry_run + ) + + def unblock_destroys( + self, + contract: Mapping[str, Any], + blocked: Iterable[Mapping[str, Any]], + say: Optional[Callable[[str], None]] = None, + client_factory: Optional[Callable[[], Any]] = None, + ) -> bool: + """After an apply that failed only at destroying ``blocked``: drop what its destroyed + tables used there (``databricks_vacated.unblock``); ``True`` to destroy them again.""" + return _vacated.unblock(_vacated_of(contract, blocked), say or print, client_factory) + def post_build( self, contract: Mapping[str, Any], @@ -2948,6 +2981,18 @@ def validate_databricks_binding( return errors, warnings +def _vacated_of( + contract: Mapping[str, Any], destroyed: Iterable[Mapping[str, Any]] +) -> _vacated.Vacated: + """The schemas among a plan's deletes that ``contract`` no longer uses + (``databricks_vacated.vacated``); a contract that does not compile raises its refusal.""" + compiled = _compile(contract, os.environ) + if compiled.refusals: + raise compiled.refusals[0] + declared = {f"{table.catalog}.{table.schema}" for table in compiled.tables} + return _vacated.vacated(destroyed, declared, compiled.metric_views.product_id) + + def _genie(contract: Mapping[str, Any], compiled: _Compiled, env: Mapping[str, str]) -> Any: """The Genie space ``contract`` asks for, from this compile's tables and grants (``databricks_genie.compile_space``); imported here, the Light CLI's ``--help`` never diff --git a/fluid_build/iac/providers/databricks_enforce.py b/fluid_build/iac/providers/databricks_enforce.py index e8cbd38b..63ed2c8e 100644 --- a/fluid_build/iac/providers/databricks_enforce.py +++ b/fluid_build/iac/providers/databricks_enforce.py @@ -17,7 +17,10 @@ function carries this module's name prefix. A filter an admin set by hand is left alone. Dropping widens access, so it waits for the apply to succeed and is printed as such. After an apply that succeeded it also drops this module's superseded functions that no table in the - schema uses and this run's identity owns (:func:`drop_orphan_functions`). + schema uses and this run's identity owns (:func:`drop_orphan_functions`). Those are the + contract's schemas; in a schema the contract stops using, which the plan destroys, the + functions go by the same rules, once OpenTofu has destroyed the tables that used them + (``databricks_vacated.py``). Both read the live bindings first (``GET /api/2.1/unity-catalog/tables/{name}``: ``row_filter.function_name`` / ``input_column_names`` and ``columns[].mask.function_name``, diff --git a/fluid_build/iac/providers/databricks_metrics.py b/fluid_build/iac/providers/databricks_metrics.py index 5df2d2d8..704f479a 100644 --- a/fluid_build/iac/providers/databricks_metrics.py +++ b/fluid_build/iac/providers/databricks_metrics.py @@ -82,12 +82,12 @@ (its table renamed, its semantics or its expose removed), is dropped before ``tofu apply``: dropping a view only narrows who reads. It is found by name (``*_semantic_model``, ``table-summaries``) and dropped only with this product's ``fluid.product_id`` and the deploy - identity as owner. Only those schemas are looked in, so another environment of the product (an - overlay that changes the schema, in the same catalog, under the same identity) keeps its views. - The cost is the one the column masks' functions share: a view left in a schema the contract - stops using (its table moved, its last expose removed) is not found, and OpenTofu's destroy of - that schema (never ``force_destroy``) then fails while the schema holds it; drop it by hand - (``DROP VIEW ``) and re-run. + identity as owner. Only those schemas are looked in here, so another environment of the + product (an overlay that changes the schema, in the same catalog, under the same identity) + keeps its views. A schema the contract stops using (its table moved, its last expose removed) + is the plan's to name: OpenTofu's state holds it and the saved plan destroys it, and + ``databricks_vacated.py`` drops this product's views there, by the same checks, before the + apply, so the destroy (never ``force_destroy``) is not refused. * A metric view reads its table with its owner's rights, and Unity Catalog applies the table's row filter and column masks to whoever queries the view (a row filter measured live). When binding them fails, :func:`lock_restricted` revokes ``SELECT`` on the views over restricted diff --git a/fluid_build/iac/providers/databricks_vacated.py b/fluid_build/iac/providers/databricks_vacated.py new file mode 100644 index 00000000..f599b9b0 --- /dev/null +++ b/fluid_build/iac/providers/databricks_vacated.py @@ -0,0 +1,391 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""Clear what this product made outside OpenTofu in a schema a plan destroys, so the destroy runs. + +A contract stops using a schema when its table moves to another schema or catalog, or its last +expose there is removed. The previous apply's state holds that schema (the product owns every +schema its tables are in, ``databricks.py``), so the saved plan destroys it, never with +``force_destroy``, and Unity Catalog refuses to delete a schema that holds anything (a function +measured live, the 2026-11-03 run in HONESTLY_TESTED.md). Two kinds of objects the sibling +modules create there are no OpenTofu resource, so nothing in the plan removes them: the metric +views (``databricks_metrics.py``) and the functions of the column masks and row filters +(``databricks_enforce.py``). + +The schemas come from the saved plan, never from a search of the catalog: each +``databricks_schema`` the plan only deletes and the contract no longer names (:func:`vacated`). +So a schema is looked in only when this environment's OpenTofu state holds it, and another +environment of the product (an overlay that changes the schema, in the same catalog, under the +same deploy identity) keeps its objects: its schemas are in its own state, not in this plan. + +* :func:`prepare` runs before ``tofu apply``. A metric view is dropped when it carries this + product's ``fluid.product_id``, the deploy identity owns it, and the table it reads is one the + plan destroys or no longer exists: dropping a view only narrows who reads, and one over a + table the plan keeps is left. A function is dropped when its name has the shape this module's + siblings give one (``fluid_rf_
_<8 hex>``, ``fluid_mask_
__<8 hex>``, + ``databricks_access.py``), the deploy identity owns it, and no table or view of the schema has + it as its row filter or a column mask (every one is read first; if the schema cannot be read, + nothing in it is dropped). A function no table uses grants nothing, so dropping it changes no + one's access. Unity Catalog names a mask's or a row filter's function in the table + (``SET MASK ``), so a function a table still uses is never dropped while the + table exists: the table would read through whatever function someone with ``CREATE FUNCTION`` + on the schema then made under that name. +* OpenTofu destroys a table before its schema, both inside one ``tofu apply``, so the functions + of a destroyed table's masks and row filter are still in the schema when OpenTofu comes to + destroy it, and that destroy fails. :func:`prepare` names them. The engine then calls + :func:`unblock` (``cli/_apply_opentofu_engine._resume_blocked_destroys``): their tables are + gone, no table uses them, and they are dropped by the same rules; the engine plans again and + applies only those destroys. +* Anything else the schema holds (a table or view the plan keeps, another product's view, an + object another identity owns, a function a kept table uses or this module did not name) is + named and left, and OpenTofu's destroy of the schema fails until it is moved or dropped. + +Not looked for: a table in another schema whose mask or row filter names a function here (this +module's siblings only ever bind a function of the table's own schema, the assumption +``databricks_enforce.drop_orphan_functions`` makes too), and a volume or a model in the schema. + +``fluid apply --mode dry-run`` prints what :func:`prepare` would drop before the apply and what +waits for the tables, and runs none of it. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any, Dict, FrozenSet, Iterable, List, Mapping, Optional, Sequence, Tuple + +import yaml + +from . import databricks_enforce as _enforce +from . import databricks_metrics as _metrics +from .databricks_access import MASK_PREFIX, ROW_FILTER_PREFIX, quote_table +from .databricks_enforce import ( + ClientFactory, + Say, + Session, + StatementFailed, + _bare, + _list_named, + _live_columns, + owner_matches, + read_function, + read_table, + scrub, +) + +#: The shape of a function this module's siblings name (``databricks_access._function_name``): +#: the prefix, the table (and column) it is for, and 8 hex digits of its definition's digest. +_FUNCTION_RE = re.compile( + rf"(?:{re.escape(ROW_FILTER_PREFIX)}|{re.escape(MASK_PREFIX)})[a-z0-9_]+_[0-9a-f]{{8}}" +) + + +@dataclass(frozen=True) +class Vacated: + """The schemas a saved plan destroys that the contract no longer names, and the tables the + plan destroys in them.""" + + product_id: str = "" + #: ``catalog.schema`` of each, as the plan names it. + schemas: Tuple[str, ...] = () + #: Lower-case full names of the tables the plan destroys in those schemas. + tables: FrozenSet[str] = frozenset() + + +def _name(before: Any, *keys: str) -> str: + """``catalog.schema[.table]`` from a planned delete's ``before`` attributes, else its ``id``.""" + if not isinstance(before, Mapping): + return "" + parts = [str(before.get(key) or "").strip() for key in keys] + if all(parts): + return ".".join(parts) + found = str(before.get("id") or "").strip() + return found if found.count(".") == len(keys) - 1 else "" + + +def vacated( + destroyed: Iterable[Mapping[str, Any]], declared: Iterable[str], product_id: str +) -> Vacated: + """The schemas among a plan's deletes that ``declared`` (the contract's ``catalog.schema`` + names) does not name, and the tables the plan destroys in them. + + ``destroyed`` is what the engine reads from the saved plan (``tofu show -json``): one + ``{"address", "type", "before"}`` per resource whose only planned action is ``delete``. + """ + entries = [entry for entry in destroyed if isinstance(entry, Mapping)] + keep = {str(name).lower() for name in declared} + schemas: Dict[str, str] = {} + for entry in entries: + if entry.get("type") == "databricks_schema": + name = _name(entry.get("before"), "catalog_name", "name") + if name and name.lower() not in keep: + schemas.setdefault(name.lower(), name) + tables = frozenset( + full.lower() + for full in ( + _name(entry.get("before"), "catalog_name", "schema_name", "name") + for entry in entries + if entry.get("type") == "databricks_sql_table" + ) + if full and full.lower().rsplit(".", 1)[0] in schemas + ) + return Vacated(str(product_id or ""), tuple(schemas[key] for key in sorted(schemas)), tables) + + +@dataclass +class _Outcome: + dropped: int = 0 + failed: int = 0 + #: Functions only the tables the plan destroys use: they go once those tables are gone. + waiting: List[str] = field(default_factory=list) + #: What the schema keeps, each with why; OpenTofu's destroy of it fails while it does. + left: List[str] = field(default_factory=list) + #: The schema could not be read, so nothing in it was dropped. + unread: bool = False + + +def _contents( + session: Session, where: str +) -> Optional[Tuple[Dict[str, Mapping[str, Any]], List[str]]]: + """Every table and view of ``where``, read, and the names of its functions; ``None`` when the + schema no longer exists.""" + catalog, schema = where.split(".", 1) + query = {"catalog_name": catalog, "schema_name": schema} + try: + # include_browse: an object the run may only browse is listed too, and reading it fails, + # so nothing is dropped rather than a binding being missed (as drop_orphan_functions) + rows = _list_named( + session.client, + "/api/2.1/unity-catalog/tables", + {**query, "include_browse": "true"}, + "tables", + ) + listed = _list_named(session.client, "/api/2.1/unity-catalog/functions", query, "functions") + except Exception as exc: # noqa: BLE001 — a missing schema is an answer; the rest is raised on + if _enforce.is_not_found(exc): + return None + raise + objects: Dict[str, Mapping[str, Any]] = {} + for row in rows: + full = str(row.get("full_name") or f"{where}.{row.get('name')}") + info = read_table(session.client, full) + if info is not None: + objects[full] = info + names = { + str(row.get("name") or str(row.get("full_name") or "").rsplit(".", 1)[-1]) for row in listed + } + return objects, sorted(name for name in names if name) + + +def _source(info: Mapping[str, Any]) -> str: + """The table a metric view reads (its definition's ``source``), or ``""``.""" + try: + document = yaml.safe_load(str(info.get("view_definition") or "")) + except yaml.YAMLError: + return "" + return str(document.get("source") or "").strip() if isinstance(document, Mapping) else "" + + +def _exists(session: Session, full: str, present: Iterable[str]) -> bool: + """Whether a table exists; one that cannot be read counts as existing (keep the view).""" + if full.lower() in present: + return True + try: + return read_table(session.client, full) is not None + except Exception: # noqa: BLE001 — unknown: the view stays + return True + + +def _holders(objects: Mapping[str, Mapping[str, Any]]) -> Dict[str, List[str]]: + """``{function (bare, lower case): [objects whose row filter or a mask names it]}``.""" + holders: Dict[str, List[str]] = {} + for full, info in sorted(objects.items()): + bound = [(info.get("row_filter") or {}).get("function_name")] + bound += [(c.get("mask") or {}).get("function_name") for c in _live_columns(info).values()] + for name in filter(None, bound): + holders.setdefault(_bare(name), []).append(full) + return holders + + +class _Clearing: + """One pass of the rules over one schema: drop what may go now, name what waits for its + tables and what the schema keeps.""" + + def __init__(self, session: Session, vac: Vacated, where: str, say: Say, dry_run: bool): + self.session, self.vac, self.where = session, vac, where + self.say, self.dry_run = say, dry_run + self.out = _Outcome() + self.me = "" + self.present: FrozenSet[str] = frozenset() + + def run(self) -> _Outcome: + try: + read = _contents(self.session, self.where) + self.me = self.session.identity if read is not None else "" + except Exception as exc: # noqa: BLE001 — unknown contents: drop nothing + self.say( + f" not checked: {self.where} could not be read ({type(exc).__name__}: " + f"{scrub(exc)}), so nothing in it was dropped" + ) + self.out.unread = True + return self.out + if read is None: + return self.out # the schema is gone already + objects, functions = read + self.present = frozenset(full.lower() for full in objects) + for full, info in sorted(objects.items()): + kind = str(info.get("table_type") or "").upper() + if kind == _metrics.METRIC_VIEW: + self._view(full, info) + elif full.lower() not in self.vac.tables: + what = _metrics._KINDS.get(kind, "a " + (kind.lower() or "table")) + self.out.left.append(f"{full} ({what} this apply keeps)") + holders = _holders(objects) + for name in functions: + full = f"{self.where}.{name}" + self._function(full, holders.get(_bare(full), [])) + return self.out + + def _view(self, full: str, info: Mapping[str, Any]) -> None: + found = (info.get("properties") or {}).get(_metrics.PRODUCT_PROPERTY) + source = _source(info) + if not self.vac.product_id or found != self.vac.product_id: + whose = f"of product {found!r}" if found else "this module did not create" + self.out.left.append(f"{full} (a metric view {whose})") + elif not source: + self.out.left.append(f"{full} (a metric view whose definition names no table)") + elif source.lower() in self.vac.tables: + self._drop_owned(full, info, "VIEW", f"it reads {source}, which this apply destroys") + elif not _exists(self.session, source, self.present): + self._drop_owned(full, info, "VIEW", f"it reads {source}, which no longer exists") + else: + self.out.left.append(f"{full} (a metric view over {source}, which this apply keeps)") + + def _function(self, full: str, holders: Sequence[str]) -> None: + kept = [table for table in holders if table.lower() not in self.vac.tables] + if not _FUNCTION_RE.fullmatch(full.rsplit(".", 1)[-1].lower()): + self.out.left.append(f"{full} (a function this module did not create)") + elif kept: + self.out.left.append(f"{full} (the row filter or a mask of {', '.join(kept)})") + else: + try: + info = read_function(self.session.client, full) or {} + except Exception: # noqa: BLE001 — unknown owner: leave it + info = {} + # Still the mask or row filter of a table the plan destroys: it goes once that is gone. + self._drop_owned(full, info, "FUNCTION", "no table uses it", waits_for=holders) + + def _drop_owned( + self, + full: str, + info: Mapping[str, Any], + kind: str, + why: str, + waits_for: Sequence[str] = (), + ) -> None: + """Drop ``full`` (a ``VIEW`` or a ``FUNCTION``) when the deploy identity owns it; one + that ``waits_for`` tables is only named, to go once they are gone.""" + owner = str(info.get("owner") or "") + what = f"{'metric view' if kind == 'VIEW' else 'function'} {full}" + try: + statement = f"DROP {kind} IF EXISTS {quote_table(full)}" + except ValueError: + self.out.left.append(f"{full} (its name is no plain identifier; drop it by hand)") + return + if not owner_matches(owner, {self.me}): + whose = ( + f"owned by {owner}, not the deploy identity" if owner else "its owner is unknown" + ) + self.out.left.append(f"{full} ({whose})") + elif waits_for: + self.out.waiting.append(full) + self.say(f" after tofu destroys {', '.join(waits_for)}: drop {what}") + if self.dry_run: + self.say(f" {statement}") + elif self.dry_run: + self.say(f" before apply: drop {what}: {why}") + self.say(f" {statement}") + else: + self._run(statement, what, why) + + def _run(self, statement: str, what: str, why: str) -> None: + try: + self.session.runner.run(statement) + except StatementFailed as exc: + self.out.failed += 1 + self.say(f" FAILED: drop {what}: {exc}") + return + self.out.dropped += 1 + self.say(f" done: drop {what}: {why}") + + +def _session(vac: Vacated, say: Say, client_factory: Optional[ClientFactory]) -> Optional[Session]: + session = _metrics._session(client_factory, lambda _line: None, required=False) + if session is None: + say( + f" note: {', '.join(vac.schemas)}: what this product made outside OpenTofu there " + "(metric views, mask and row filter functions) was not checked: no Databricks " + "client, or no DATABRICKS_WAREHOUSE_ID. OpenTofu's destroy of a schema fails while " + "it holds one." + ) + return session + + +def _say_left(where: str, out: _Outcome, say: Say) -> None: + if out.left: + say( + f" note: OpenTofu cannot destroy {where} while it holds " + + "; ".join(out.left) + + ". Move or drop them, then re-run fluid apply." + ) + + +def prepare( + vac: Vacated, say: Say, client_factory: Optional[ClientFactory] = None, *, dry_run: bool = False +) -> None: + """Before ``tofu apply``: in each schema the plan destroys, drop what may go now, and name + what waits for its tables and what keeps the schema from being destroyed. With ``dry_run`` + it prints the statements and runs none.""" + if not vac.schemas: + return + session = _session(vac, say, client_factory) + if session is None: + return + shown = False + for where in vac.schemas: + lines: List[str] = [] + out = _Clearing(session, vac, where, lines.append, dry_run).run() + _say_left(where, out, lines.append) + if not lines: + continue + if not shown: + say("\n Schemas this apply destroys: what this product made there outside OpenTofu") + shown = True + say(f" {where}") + for line in lines: + say(line) + if shown and dry_run: + say(" dry-run: none of these statements ran.") + + +def unblock(vac: Vacated, say: Say, client_factory: Optional[ClientFactory] = None) -> bool: + """After a ``tofu apply`` that failed only at destroying ``vac.schemas``: drop what this + product made there that the apply's own destroys freed (the functions of the tables it + destroyed), by :func:`prepare`'s rules. ``True`` when something was dropped, no drop failed + and nothing it can see is left, so destroying the schemas again can succeed.""" + if not vac.schemas: + return False + session = _session(vac, say, client_factory) + if session is None: + return False + dropped, clear = 0, True + for where in vac.schemas: + say(f"\n {where}: tofu apply could not destroy it yet; what this product made there") + out = _Clearing(session, Vacated(vac.product_id, (where,)), where, say, False).run() + _say_left(where, out, say) + dropped += out.dropped + clear = clear and not (out.failed or out.left or out.unread or out.waiting) + return dropped > 0 and clear + + +__all__ = ["Vacated", "prepare", "unblock", "vacated"] diff --git a/fluid_build/iac/runner.py b/fluid_build/iac/runner.py index 813b8c84..675bf16a 100644 --- a/fluid_build/iac/runner.py +++ b/fluid_build/iac/runner.py @@ -23,7 +23,7 @@ import subprocess from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, List, Mapping, Optional, Tuple +from typing import Any, Dict, Iterator, List, Mapping, Optional, Tuple # Per-command wall-clock cap. A hung ``tofu`` (e.g., an unauthenticated # interactive auth prompt that ``-input=false`` did not catch, or a cloud @@ -521,6 +521,18 @@ def tofu_state_rm( return _run(["state", "rm", *addresses], workdir=workdir, env=env, command="state-rm") +def _resource_events( + result: TofuResult, kind: str, body: str +) -> Iterator[Tuple[str, Mapping[str, Any]]]: + """``(action, resource)`` of each ``kind`` event of a ``-json`` stream (OpenTofu's + machine-readable UI): a ``planned_change`` carries them under ``change``, an + ``apply_*`` hook event under ``hook``.""" + for event in result.events: + if event.get("type") == kind: + detail = event.get(body) or {} + yield str(detail.get("action") or ""), detail.get("resource") or {} + + def planned_removals(result: TofuResult) -> List[Tuple[str, str]]: """``(address, resource type)`` of every resource a plan deletes or replaces. @@ -529,16 +541,41 @@ def planned_removals(result: TofuResult) -> List[Tuple[str, str]]: that remove an object). The ``change_summary`` event's ``remove`` counts the same resources, so a caller can tell which of them hold data. """ - out: List[Tuple[str, str]] = [] - for event in result.events: - if event.get("type") != "planned_change": + return [ + (str(resource.get("addr") or ""), str(resource.get("resource_type") or "")) + for action, resource in _resource_events(result, "planned_change", "change") + if action in ("delete", "replace") + ] + + +def errored_resources(result: TofuResult) -> List[Tuple[str, str]]: + """``(address, action)`` of every resource ``tofu apply -json`` failed on, from its + ``apply_errored`` events (the diagnostic that follows one carries no address).""" + return [ + (str(resource.get("addr") or ""), action) + for action, resource in _resource_events(result, "apply_errored", "hook") + ] + + +def planned_deletes(shown: Mapping[str, Any]) -> List[Dict[str, Any]]: + """``{"address", "type", "before"}`` of each managed resource a saved plan, as ``tofu + show -json`` prints it, only deletes (a replaced one comes back, so it is not one).""" + deletes: List[Dict[str, Any]] = [] + for change in shown.get("resource_changes") or []: + detail = change.get("change") if isinstance(change, Mapping) else None + if not isinstance(detail, Mapping) or change.get("mode", "managed") != "managed": continue - change = event.get("change") or {} - if change.get("action") not in ("delete", "replace"): + if list(detail.get("actions") or []) != ["delete"]: continue - resource = change.get("resource") or {} - out.append((str(resource.get("addr") or ""), str(resource.get("resource_type") or ""))) - return out + before = detail.get("before") + deletes.append( + { + "address": str(change.get("address") or ""), + "type": str(change.get("type") or ""), + "before": before if isinstance(before, Mapping) else {}, + } + ) + return deletes def change_summary(result: TofuResult) -> Dict[str, int]: diff --git a/tests/iac/test_iac_apply_engine_destroys.py b/tests/iac/test_iac_apply_engine_destroys.py new file mode 100644 index 00000000..dcb1422d --- /dev/null +++ b/tests/iac/test_iac_apply_engine_destroys.py @@ -0,0 +1,274 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""The engine's side of a destroy that must wait for the apply's own (``_planned_destroys``, +``_prepare_destroys``, ``_resume_blocked_destroys``), with ``tofu`` stubbed. + +Pinned: the plugin is handed only what the saved plan deletes outright; a failed apply is +resumed only when every resource it failed on is one of those deletes and no error is left +unaccounted for, the plugin cleared something, and the plan made after it holds nothing but those +deletes. Not proven here: a real ``tofu`` run (``test_iac_databricks_metrics_fake_e2e.py``). +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import pytest + +from fluid_build.cli import _apply_opentofu_engine as engine +from fluid_build.cli._common import CLIError +from fluid_build.iac.base import UnsupportedBindingError + +pytestmark = [pytest.mark.unit] + +SCHEMA = "databricks_schema.p_main_sales" +DESTROYED = [ + {"address": SCHEMA, "type": "databricks_schema", "before": {"id": "main.sales"}}, + { + "address": "databricks_sql_table.p_main_sales_orders", + "type": "databricks_sql_table", + "before": {"id": "main.sales.orders"}, + }, +] + + +def _result(command: str, rc: int = 0, events: Optional[List[Dict[str, Any]]] = None) -> Any: + return engine.runner.TofuResult(command, rc, "", "", events=events or []) + + +def _errored(address: str, action: str = "delete") -> Dict[str, Any]: + return {"type": "apply_errored", "hook": {"resource": {"addr": address}, "action": action}} + + +DIAGNOSTIC = { + "type": "diagnostic", + "diagnostic": {"severity": "error", "summary": "cannot delete schema: Schema is not empty"}, +} + + +def _plan(*removed: str, add: int = 0, change: int = 0) -> Any: + events: List[Dict[str, Any]] = [ + { + "type": "planned_change", + "change": {"action": "delete", "resource": {"addr": a, "resource_type": "x"}}, + } + for a in removed + ] + events.append( + { + "type": "change_summary", + "changes": {"add": add, "change": change, "remove": len(removed)}, + } + ) + return _result("plan", events=events) + + +class Plugin: + def __init__(self, cleared: Any = True): + self.cleared = cleared + self.calls: List[List[Dict[str, Any]]] = [] + self.prepared: List[Dict[str, Any]] = [] + + def prepare_destroys(self, contract, destroyed, say=None, dry_run=False): + self.prepared.append({"destroyed": destroyed, "dry_run": dry_run}) + + def unblock_destroys(self, contract, blocked, say=None): + self.calls.append(list(blocked)) + if isinstance(self.cleared, Exception): + raise self.cleared + return self.cleared + + +@pytest.fixture +def tofu(monkeypatch): + calls: Dict[str, Any] = { + "plan": 0, + "apply": 0, + "next_plan": _plan(SCHEMA), + "shown": None, + "applies": [], + } + + def plan(*a, **k): + calls["plan"] += 1 + return calls["next_plan"] + + def apply(*a, **k): + calls["apply"] += 1 + if calls["applies"]: + return calls["applies"].pop(0) + return _result("apply", events=[{"type": "change_summary", "changes": {"remove": 1}}]) + + def shown(*a, **k): + calls["show"] = calls.get("show", 0) + 1 + return calls["shown"] + + monkeypatch.setattr(engine.runner, "tofu_plan", plan) + monkeypatch.setattr(engine.runner, "tofu_apply", apply) + monkeypatch.setattr(engine.runner, "tofu_show_plan", shown) + return calls + + +def _said(capfd) -> str: + """What the engine printed, whitespace collapsed (Rich wraps at the terminal width).""" + return " ".join(capfd.readouterr().out.split()) + + +def _resume(plugin: Any, *events: Dict[str, Any]) -> Any: + failed = _result("apply", 1, list(events)) + return engine._resume_blocked_destroys(plugin, {}, failed, DESTROYED, "/wd", {}) + + +def test_the_plugin_gets_only_what_the_plan_deletes_outright(tofu): + def change(address: str, actions: List[str], mode: str = "managed") -> Dict[str, Any]: + return { + "address": address, + "mode": mode, + "type": address.split(".")[0], + "change": {"actions": actions, "before": {"id": address}}, + } + + tofu["shown"] = { + "resource_changes": [ + change("databricks_schema.gone", ["delete"]), + change("databricks_sql_table.replaced", ["delete", "create"]), + change("databricks_schema.new", ["create"]), + change("databricks_grant.kept", ["no-op"]), + change("data.databricks_catalog.c", ["delete"], mode="data"), + ] + } + found = engine._planned_destroys(Plugin(), {"remove": 2}, "/wd", {}) + assert found == [ + { + "address": "databricks_schema.gone", + "type": "databricks_schema", + "before": {"id": "databricks_schema.gone"}, + } + ] + + +def test_nothing_is_read_for_a_plugin_without_the_hook_or_a_plan_that_removes_nothing(tofu): + assert engine._planned_destroys(object(), {"remove": 3}, "/wd", {}) == [] + assert engine._planned_destroys(Plugin(), {"remove": 0}, "/wd", {}) == [] + assert "show" not in tofu + + +def test_a_plan_that_cannot_be_read_back_is_said(tofu, capfd): + assert engine._planned_destroys(Plugin(), {"remove": 1}, "/wd", {}) == [] + assert "the saved plan could not be read back" in _said(capfd) + + +def test_prepare_gets_the_deletes_and_the_dry_run_flag(): + plugin = Plugin() + engine._prepare_destroys(plugin, {}, DESTROYED, dry_run=True) + engine._prepare_destroys(plugin, {}, [], dry_run=False) # nothing destroyed: not called + assert plugin.prepared == [{"destroyed": DESTROYED, "dry_run": True}] + + +def test_a_refusal_in_prepare_stops_the_run(): + class Refusing(Plugin): + def prepare_destroys(self, contract, destroyed, say=None, dry_run=False): + raise UnsupportedBindingError("databricks-x", "refused", ("fix it",)) + + with pytest.raises(CLIError) as exc: + engine._prepare_destroys(Refusing(), {}, DESTROYED) + assert exc.value.event == "opentofu_destroy_preparation_failed" + + +def test_a_destroy_that_waited_is_resumed_once_the_plugin_cleared_it(tofu, capfd): + plugin = Plugin() + resumed = _resume(plugin, _errored(SCHEMA), DIAGNOSTIC) + assert resumed is not None + second, waited = resumed + assert waited == 1 and second.ok + assert plugin.calls == [[DESTROYED[0]]] + assert tofu["plan"] == 1 and tofu["apply"] == 1 + assert "tofu plan (the destroys that waited): +0 ~0 -1" in _said(capfd) + + +@pytest.mark.parametrize( + "events", + [ + pytest.param([_errored("databricks_schema.unplanned"), DIAGNOSTIC], id="not-planned"), + pytest.param([_errored(SCHEMA, "create"), DIAGNOSTIC], id="a-create-failed"), + pytest.param([_errored(SCHEMA), DIAGNOSTIC, DIAGNOSTIC], id="an-unaccounted-error"), + pytest.param([DIAGNOSTIC], id="no-resource-failed"), + pytest.param( + [_errored(SCHEMA), _errored("databricks_grant.g", "create"), DIAGNOSTIC, DIAGNOSTIC], + id="another-resource-failed", + ), + ], +) +def test_any_other_failure_is_left_as_it_is(tofu, events): + plugin = Plugin() + assert _resume(plugin, *events) is None + assert plugin.calls == [] and tofu["plan"] == 0 and tofu["apply"] == 0 + + +@pytest.mark.parametrize("cleared", [False, RuntimeError("boom")], ids=["nothing", "raised"]) +def test_nothing_cleared_is_not_resumed(tofu, cleared): + assert _resume(Plugin(cleared), _errored(SCHEMA), DIAGNOSTIC) is None + assert tofu["plan"] == 0 and tofu["apply"] == 0 + + +@pytest.mark.parametrize( + "plan", + [ + pytest.param(lambda: _plan(SCHEMA, add=1), id="adds"), + pytest.param(lambda: _plan(SCHEMA, change=1), id="changes"), + pytest.param(lambda: _plan("databricks_schema.other"), id="removes-another"), + pytest.param(lambda: _plan(), id="removes-nothing"), + pytest.param(lambda: _result("plan", 1), id="fails"), + ], +) +def test_a_second_plan_holding_anything_else_is_not_applied(tofu, capfd, plan): + tofu["next_plan"] = plan() + assert _resume(Plugin(), _errored(SCHEMA), DIAGNOSTIC) is None + assert tofu["plan"] == 1 and tofu["apply"] == 0 + assert "not destroyed again: the plan made after the apply holds more than" in _said(capfd) + + +def test_a_plugin_without_the_hook_is_never_resumed(tofu): + class Bare: + pass + + failed = _result("apply", 1, [_errored(SCHEMA), DIAGNOSTIC]) + assert engine._resume_blocked_destroys(Bare(), {}, failed, DESTROYED, "/wd", {}) is None + assert tofu["plan"] == 0 + + +def test_a_clean_apply_is_not_resumed(tofu): + result, waited = engine._apply_saved_plan(Plugin(), {}, DESTROYED, "/wd", {}) + assert result.ok and waited == 0 + assert tofu["apply"] == 1 and tofu["plan"] == 0 + + +def test_an_apply_that_failed_only_at_a_destroy_that_waited_is_resumed(tofu): + tofu["applies"].append(_result("apply", 1, [_errored(SCHEMA), DIAGNOSTIC])) + result, waited = engine._apply_saved_plan(Plugin(), {}, DESTROYED, "/wd", {}) + assert result.ok and waited == 1 + assert tofu["apply"] == 2 and tofu["plan"] == 1 + + +def test_an_apply_that_failed_otherwise_keeps_its_own_failure(tofu): + failed = _result("apply", 1, [DIAGNOSTIC]) + tofu["applies"].append(failed) + result, waited = engine._apply_saved_plan(Plugin(), {}, DESTROYED, "/wd", {}) + assert result is failed and waited == 0 + assert tofu["apply"] == 1 and tofu["plan"] == 0 + + +def test_after_a_resume_the_totals_are_the_plans_own(): + def summary(**changes: int) -> Any: + return _result("apply", events=[{"type": "change_summary", "changes": changes}]) + + planned = {"add": 11, "change": 0, "remove": 11} + second = summary(add=0, change=0, remove=1) + assert engine._applied(planned, second, 1) == planned + alone = summary(add=2, change=1, remove=0) + assert engine._applied({"add": 9, "change": 9, "remove": 9}, alone, 0) == { + "add": 2, + "change": 1, + "remove": 0, + } diff --git a/tests/iac/test_iac_databricks_enforce.py b/tests/iac/test_iac_databricks_enforce.py index 745604dc..6001360f 100644 --- a/tests/iac/test_iac_databricks_enforce.py +++ b/tests/iac/test_iac_databricks_enforce.py @@ -612,16 +612,25 @@ def test_a_refusal_from_a_hook_is_a_typed_error_with_its_fix(hook): assert exc.value.context["remediation"] == ["fix it"] +def _source(function) -> str: + return __import__("inspect").getsource(function) + + def test_the_engine_runs_the_hooks_around_tofu_apply_in_this_order(): - source = __import__("inspect").getsource(engine._apply_via_opentofu) + source = _source(engine._apply_via_opentofu) pre = source.index('_enforce_access(plugin, "pre_apply", contract)') + prepare = source.index("_prepare_destroys(plugin, contract, destroyed)") keep = source.index("imported.keep()") - apply = source.index("runner.tofu_apply(") + apply = source.index("_apply_saved_plan(") post = source.index('_enforce_access(plugin, "post_apply", contract)') gate = source.index("_enforce_data_loss_gate(") dry = source.index('_enforce_access(plugin, "pre_apply", contract, dry_run=True)') - assert gate < dry < source.index('cprint("\\ndry-run: plan only') - assert gate < pre < keep < apply < post + dry_prepare = source.index("_prepare_destroys(plugin, contract, destroyed, dry_run=True)") + assert gate < dry < dry_prepare < source.index('cprint("\\ndry-run: plan only') + assert gate < pre < prepare < keep < apply < post + # The saved plan is applied first; a destroy that waited is resumed only after it failed. + saved = _source(engine._apply_saved_plan) + assert saved.index("runner.tofu_apply(") < saved.index("_resume_blocked_destroys(") # ── after a tofu apply that failed part way ───────────────────────────────── @@ -712,10 +721,10 @@ def test_the_engine_prints_an_enforcement_failure_after_a_failed_apply_and_never def test_the_engine_binds_before_it_raises_the_apply_failure(): - source = __import__("inspect").getsource(engine._apply_via_opentofu) + source = _source(engine._apply_via_opentofu) bind = source.index("_enforce_after_failed_apply(plugin, contract)") failed = source.index("raise _apply_failure(") - assert source.index("runner.tofu_apply(") < bind < failed + assert source.index("_apply_saved_plan(") < bind < failed # ── a function someone else made under the hashed name ────────────────────── diff --git a/tests/iac/test_iac_databricks_metrics.py b/tests/iac/test_iac_databricks_metrics.py index 1f43a2cd..658a75b4 100644 --- a/tests/iac/test_iac_databricks_metrics.py +++ b/tests/iac/test_iac_databricks_metrics.py @@ -709,7 +709,8 @@ def test_a_view_the_contract_no_longer_declares_is_dropped_before_the_apply(fake run.pre(doc) dropped = [s for s in run.statements(before) if s.startswith("DROP VIEW")] assert dropped == ["DROP VIEW IF EXISTS `main`.`sales`.`site_360_v0_semantic_model`"] - # Only the product's own schemas are looked in: one it no longer uses is not (a known limit). + # pre_apply looks only in the schemas the contract uses; one it stopped using is the plan's + # to name (prepare_destroys, test_iac_databricks_vacated.py). assert ours_moved in fake.tables assert theirs in fake.tables and VIEW in fake.tables del doc["exposes"][0]["semantics"] diff --git a/tests/iac/test_iac_databricks_metrics_fake_e2e.py b/tests/iac/test_iac_databricks_metrics_fake_e2e.py index 427897a9..dfa98ee0 100644 --- a/tests/iac/test_iac_databricks_metrics_fake_e2e.py +++ b/tests/iac/test_iac_databricks_metrics_fake_e2e.py @@ -8,9 +8,13 @@ with OpenTofu and then the metric view of its ``semantics`` on the warehouse, with ``SELECT`` for the table's readers; a second apply plans nothing and runs no metric view statement; ``--mode dry-run`` prints the ``CREATE VIEW`` and runs none; a contract that drops the semantics -drops the view; ``fluid validate`` refuses what metric view YAML cannot express; the product's -Genie space (``extensions.databricksGenie``, converged after the metric views) lists the view the -same apply made among its tables. +drops the view; a table that moves to another schema, with its view, masks and row filter, leaves +nothing in the old schema and OpenTofu destroys it in the same apply (the view dropped before the +apply, the functions once the table is gone, then the schema's destroy resumed), which a dry run +prints without running, and so does removing the last expose in a schema; ``fluid validate`` +refuses what metric view YAML cannot express; the product's Genie space +(``extensions.databricksGenie``, converged after the metric views) lists the view the same apply +made among its tables. Not proven here: what real Databricks accepts and answers (the Free Edition run in the PR, HONESTLY_TESTED.md). @@ -20,6 +24,7 @@ import copy import json +import re from pathlib import Path from typing import Any, Dict, Iterator, List @@ -149,6 +154,94 @@ def test_validate_refuses_what_the_yaml_cannot_express(workspace, fake, capfd): assert fake.sql_log == [] +RESTRICTED: Dict[str, Any] = { + "columnRestrictions": [ + { + "principal": "group:analysts", + "columns": ["region"], + "access": "mask", + "mask": "last_four", + } + ], + "rowFilters": [{"principal": "group:analysts", "where": "region = 'eu'"}], +} + + +def _governed(schema: str = "sales") -> Dict[str, Any]: + """The metric view contract with ``region`` masked and a row filter for the analysts, its + table in ``main.``.""" + contract = _contract() + expose = contract["exposes"][0] + expose["binding"]["location"]["schema"] = schema + expose["policy"]["authz"].update(copy.deepcopy(RESTRICTED)) + return contract + + +def _held(fake: FakeDatabricks, schema: str) -> List[str]: + return sorted(n for n in [*fake.tables, *fake.functions] if n.rsplit(".", 1)[0] == schema) + + +def _totals(out: str, what: str) -> str: + found = re.search(rf"{what}: (\+\d+ ~\d+ -\d+)", out) + assert found, out + return found.group(1) + + +def test_a_table_that_moves_schema_takes_its_view_and_functions_and_its_old_schema_goes( + workspace, fake, capfd +): + rc, out = apply_contract(capfd, write_contract(workspace, _governed())) + assert rc == 0, out + old = _held(fake, "main.sales") + assert VIEW in old and sum(n.split(".")[2].startswith("fluid_") for n in old) == 2 + + moved = write_contract(workspace, _governed("other")) + before = len(fake.sql_log) + rc, out = apply_contract(capfd, moved, "--allow-data-loss", "--mode", "dry-run") + assert rc == 0, out + assert ( + f"before apply: drop metric view {VIEW}: it reads main.sales.orders, which this apply " + "destroys DROP VIEW IF EXISTS `main`.`sales`.`orders_semantic_model`" + ) in out + assert "after tofu destroys main.sales.orders: drop function main.sales.fluid_rf_orders_" in out + assert "DROP FUNCTION IF EXISTS `main`.`sales`.`fluid_mask_orders_region_" in out + assert "dry-run: none of these statements ran." in out + assert _held(fake, "main.sales") == old + assert not [s for s in fake.sql_log[before:] if s.startswith("DROP")] + + rc, out = apply_contract(capfd, moved, "--allow-data-loss") + assert rc == 0, out + assert "main.sales" not in fake.schemas and _held(fake, "main.sales") == [] + assert "tofu plan (the destroys that waited): +0 ~0 -1" in out + assert _totals(out, "tofu apply complete") == _totals(out, "tofu plan") + assert fake.tables["main.other.orders_semantic_model"]["table_type"] == "METRIC_VIEW" + bound = fake.tables["main.other.orders"]["row_filter"]["function_name"] + assert bound.startswith("main.other.fluid_rf_orders_") + + before = len(fake.sql_log) + rc, out = apply_contract(capfd, moved) + assert rc == 0, out + assert "tofu plan: +0 ~0 -0" in out + assert not [s for s in fake.sql_log[before:] if s.startswith(("DROP", "CREATE"))] + + +def test_removing_the_last_expose_in_a_schema_destroys_that_schema(workspace, fake, capfd): + contract = _governed() + crm = copy.deepcopy(contract["exposes"][0]) + crm["exposeId"] = "orders_crm" + crm["binding"]["location"]["schema"] = "crm" + contract["exposes"].append(crm) + rc, out = apply_contract(capfd, write_contract(workspace, contract)) + assert rc == 0, out + assert "main.crm.orders_semantic_model" in fake.tables + + del contract["exposes"][1] + rc, out = apply_contract(capfd, write_contract(workspace, contract), "--allow-data-loss") + assert rc == 0, out + assert "main.crm" not in fake.schemas and _held(fake, "main.crm") == [] + assert VIEW in fake.tables and "main.sales" in fake.schemas + + def test_the_products_genie_space_holds_the_metric_view_the_apply_made(workspace, fake, capfd): # Genie reads the contract's semantic model through the metric view: the space is converged # after the metric views, so the first apply already lists the view beside its table. diff --git a/tests/iac/test_iac_databricks_vacated.py b/tests/iac/test_iac_databricks_vacated.py new file mode 100644 index 00000000..e08058a1 --- /dev/null +++ b/tests/iac/test_iac_databricks_vacated.py @@ -0,0 +1,342 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""What a product made outside OpenTofu in a schema its contract stopped using, against a stand-in. + +Pinned (``iac/providers/databricks_vacated.py`` through the plugin's ``prepare_destroys`` and +``unblock_destroys``, against ``_fake_databricks.py``): a table that moved from ``main.sales`` to +``main.other`` leaves its metric view, dropped before the apply, and its mask and row filter +functions, named before the apply and dropped once the table is gone, after which the schema can +be deleted; a dry run prints the statements and runs none; only this product's views and the +deploy identity's objects are dropped, never a function a table still uses, nor anything in a +schema the contract still names or that another environment of the product uses; a schema that +cannot be read, or a drop that fails, drops nothing more and does not ask for the destroy again. + +Not proven here: the engine's resume of the destroy (``test_iac_apply_engine_destroys.py``) and +the whole apply through ``tofu`` (``test_iac_databricks_metrics_fake_e2e.py``). +""" + +from __future__ import annotations + +import copy +from typing import Any, Dict, Iterator, List + +import pytest + +from fluid_build.iac import IAC_PLUGINS +from fluid_build.iac.providers import databricks_vacated as vacated_mod +from fluid_build.iac.providers.databricks_enforce import WAREHOUSE_ENV + +from ._databricks_cli import make_contract +from ._fake_databricks import DEPLOY_USER, FakeDatabricks, _execute + +pytestmark = [pytest.mark.unit, pytest.mark.databricks] + +PLUGIN = IAC_PLUGINS["databricks"] +OLD = "main.sales.orders" +VIEW = "main.sales.orders_semantic_model" +LIVE = [("id", "bigint", False), ("amount", "double", True), ("region", "string", True)] +SEMANTICS: Dict[str, Any] = { + "name": "orders_metrics", + "dimensions": [{"name": "region", "type": "categorical", "expr": "region"}], + "measures": [{"name": "revenue", "agg": "sum", "expr": "amount"}], +} +#: The moved contract's plan, as the engine reads it (``tofu show -json``): the old schema and its +#: table, destroyed outright. +SCHEMA_DELETE = { + "address": "databricks_schema.dbx_fake_orders_main_sales", + "type": "databricks_schema", + "before": {"id": "main.sales", "catalog_name": "main", "name": "sales"}, +} +DESTROYED = [ + SCHEMA_DELETE, + { + "address": "databricks_sql_table.dbx_fake_orders_main_sales_orders", + "type": "databricks_sql_table", + "before": {"id": OLD, "catalog_name": "main", "schema_name": "sales", "name": "orders"}, + }, +] + + +def _contract(schema: str = "sales") -> Dict[str, Any]: + """``orders`` with a metric view, ``region`` masked and a row filter for the analysts.""" + contract = make_contract() + expose = contract["exposes"][0] + expose["binding"]["location"]["schema"] = schema + expose["semantics"] = copy.deepcopy(SEMANTICS) + authz = expose["policy"]["authz"] + authz["columnRestrictions"] = [ + { + "principal": "group:analysts", + "columns": ["region"], + "access": "mask", + "mask": "last_four", + } + ] + authz["rowFilters"] = [{"principal": "group:analysts", "where": "region = 'eu'"}] + return contract + + +@pytest.fixture +def fake(monkeypatch) -> Iterator[FakeDatabricks]: + server = FakeDatabricks().start() + monkeypatch.setenv(WAREHOUSE_ENV, server.warehouse_id) + monkeypatch.setenv("DATABRICKS_TOKEN", "dapi-secret-token-value") # pragma: allowlist secret + try: + yield server + finally: + server.stop() + + +class Run: + """The plugin's hooks against ``fake``, with what they print kept.""" + + def __init__(self, fake: FakeDatabricks): + self.fake = fake + self.said: List[str] = [] + + def deploy(self, doc: Dict[str, Any]) -> None: + """What an apply of ``doc`` leaves: the table (OpenTofu's), then its masks, row filter + and metric view (the hooks').""" + location = doc["exposes"][0]["binding"]["location"] + table = f"{location['database']}.{location['schema']}.{location['table']}" + self.fake.create_table(table, LIVE) + PLUGIN.post_apply(doc, say=self.said.append, client_factory=self.fake.client) + + def prepare(self, destroyed: List[Dict[str, Any]] = DESTROYED, **kw: Any) -> None: + PLUGIN.prepare_destroys( + _contract("other"), + destroyed, + say=self.said.append, + client_factory=kw.pop("client_factory", self.fake.client), + **kw, + ) + + def unblock(self, **kw: Any) -> bool: + return PLUGIN.unblock_destroys( + _contract("other"), + [SCHEMA_DELETE], + say=self.said.append, + client_factory=kw.pop("client_factory", self.fake.client), + ) + + def drops(self, since: int) -> List[str]: + return [s for s in self.fake.sql_log[since:] if s.startswith("DROP")] + + def destroy_table(self, full: str) -> None: + """What ``tofu apply`` does to the old table before it comes to the schema.""" + ok, err = _execute(self.fake, "DROP TABLE " + ".".join(f"`{p}`" for p in full.split("."))) + assert ok, err + + def delete_schema(self, full: str) -> str: + """OpenTofu's destroy of the schema (the Schemas API); the error, or ``""``.""" + try: + self.fake.client().api_client.do("DELETE", f"/api/2.1/unity-catalog/schemas/{full}") + except Exception as exc: # noqa: BLE001 — the stand-in's error is the answer + return str(exc) + return "" + + def held(self, schema: str) -> List[str]: + objects = [*self.fake.tables, *self.fake.functions] + return sorted(n for n in objects if n.rsplit(".", 1)[0] == schema) + + @property + def text(self) -> str: + return "\n".join(self.said) + + +def _metric_view(fake: FakeDatabricks, full: str, product: str, source: str = OLD) -> None: + ok, err = _execute( + fake, + f"CREATE VIEW {full} WITH METRICS LANGUAGE YAML TBLPROPERTIES ('fluid.product_id' = " + f"'{product}') AS $$version: 1.1\nsource: {source}\nmeasures:\n- name: n\n expr: " + "COUNT(1)\n$$", + ) + assert ok, err + + +def test_the_view_goes_before_the_apply_and_the_functions_once_their_table_is_gone(fake): + run = Run(fake) + run.deploy(_contract()) + functions = [n for n in run.held("main.sales") if n.split(".")[2].startswith("fluid_")] + assert len(functions) == 2 and VIEW in fake.tables + + before = len(fake.sql_log) + run.prepare() + assert run.drops(before) == ["DROP VIEW IF EXISTS `main`.`sales`.`orders_semantic_model`"] + assert VIEW not in fake.tables and ("table", VIEW) not in fake.grants + for name in functions: # still the old table's masks and row filter + assert f"after tofu destroys {OLD}: drop function {name}" in run.text + assert name in fake.functions + + # tofu apply destroys the table, then is refused the schema: the functions are still in it. + run.destroy_table(OLD) + assert "Schema is not empty" in run.delete_schema("main.sales") + before = len(fake.sql_log) + assert run.unblock() is True + assert sorted(run.drops(before)) == sorted( + "DROP FUNCTION IF EXISTS " + ".".join(f"`{p}`" for p in name.split(".")) + for name in functions + ) + assert f"done: drop function {functions[0]}: no table uses it" in run.text + assert run.held("main.sales") == [] + assert run.delete_schema("main.sales") == "" and "main.sales" not in fake.schemas + + +def test_a_dry_run_prints_the_drops_and_runs_none(fake): + run = Run(fake) + run.deploy(_contract()) + before, run.said[:] = len(fake.sql_log), [] + run.prepare(dry_run=True) + assert fake.sql_log[before:] == [] + assert VIEW in fake.tables + assert run.said[1:3] == [ + " main.sales", + f" before apply: drop metric view {VIEW}: it reads {OLD}, which this apply destroys", + ] + assert "Schemas this apply destroys: what this product made there outside OpenTofu" in run.text + assert " DROP VIEW IF EXISTS `main`.`sales`.`orders_semantic_model`" in run.text + assert f" after tofu destroys {OLD}: drop function main.sales.fluid_rf_orders_" in run.text + assert " DROP FUNCTION IF EXISTS `main`.`sales`.`fluid_mask_orders_region_" in run.text + assert run.said[-1] == " dry-run: none of these statements ran." + + +def test_only_this_products_and_this_identitys_objects_go_and_never_a_bound_function(fake): + run = Run(fake) + run.deploy(_contract()) + ours = set(run.held("main.sales")) & set(fake.functions) # the old table's mask and filter + # Another environment's table in the same schema, its row filter bound, and its view. + fake.create_table("main.sales.orders_stg", LIVE) + kept_filter = "main.sales.fluid_rf_orders_stg_12345678" + fake.add_function(kept_filter, [("region", "string")], "BOOLEAN", "true", owner=DEPLOY_USER) + ok, err = _execute( + fake, + "ALTER TABLE `main`.`sales`.`orders_stg` SET ROW FILTER " + "`main`.`sales`.`fluid_rf_orders_stg_12345678` ON (`region`)", + ) + assert ok, err + _metric_view( + fake, "main.sales.orders_stg_semantic_model", "dbx.fake.orders", "main.sales.orders_stg" + ) + _metric_view(fake, "main.sales.theirs_semantic_model", "other.product") + _metric_view(fake, "main.sales.handed_semantic_model", "dbx.fake.orders") + fake.tables["main.sales.handed_semantic_model"]["owner"] = "someone@example.test" + fake.add_function( + "main.sales.normalize", [("x", "string")], "STRING", "upper(x)", owner=DEPLOY_USER + ) + planted = "main.sales.fluid_mask_orders_amount_0badc0de" + fake.add_function(planted, [("val", "double")], "DOUBLE", "val") # attacker-owned + orphan = "main.sales.fluid_rf_orders_00c0ffee" + fake.add_function(orphan, [("region", "string")], "BOOLEAN", "true", owner=DEPLOY_USER) + + before = len(fake.sql_log) + run.prepare() + assert sorted(run.drops(before)) == [ + "DROP FUNCTION IF EXISTS `main`.`sales`.`fluid_rf_orders_00c0ffee`", + "DROP VIEW IF EXISTS `main`.`sales`.`orders_semantic_model`", + ] + note = next(line for line in run.said if "OpenTofu cannot destroy main.sales" in line) + for left in ( + "main.sales.orders_stg (a managed table this apply keeps)", + "main.sales.orders_stg_semantic_model (a metric view over main.sales.orders_stg, which " + "this apply keeps)", + "main.sales.theirs_semantic_model (a metric view of product 'other.product')", + "main.sales.handed_semantic_model (owned by someone@example.test, not the deploy " + "identity)", + "main.sales.normalize (a function this module did not create)", + f"{planted} (owned by attacker@example.test, not the deploy identity)", + f"{kept_filter} (the row filter or a mask of main.sales.orders_stg)", + ): + assert left in note + + run.destroy_table(OLD) + before = len(fake.sql_log) + assert run.unblock() is False # the schema still holds what is not this run's to drop + assert len(ours) == 2 and len(run.drops(before)) == 2 and not ours & set(fake.functions) + assert {kept_filter, planted, "main.sales.normalize"} <= set(fake.functions) + assert fake.tables["main.sales.orders_stg"]["row_filter"]["function_name"] == kept_filter + for view in ("orders_stg", "theirs", "handed"): + assert f"main.sales.{view}_semantic_model" in fake.tables + + +def test_a_schema_the_contract_still_names_is_not_looked_in(fake): + run = Run(fake) + run.deploy(_contract()) + before = len(fake.queries) + PLUGIN.prepare_destroys(_contract(), DESTROYED, say=run.said.append, client_factory=fake.client) + assert fake.queries[before:] == [] + assert VIEW in fake.tables + + +def test_another_environment_of_the_product_keeps_its_view_and_functions(fake): + """prod and staging of one product: an overlay that changes only the schema, one catalog and + one deploy identity. Prod's plan destroys prod's old schema, never staging's.""" + run = Run(fake) + run.deploy(_contract()) + run.deploy(_contract("sales_stg")) + staging = run.held("main.sales_stg") + assert "main.sales_stg.orders_semantic_model" in staging and len(staging) == 4 + + run.prepare() + run.destroy_table(OLD) + assert run.unblock() is True + assert run.held("main.sales") == [] + assert run.held("main.sales_stg") == staging + + +def test_a_schema_that_cannot_be_read_drops_nothing(fake): + run = Run(fake) + run.deploy(_contract()) + fake.fail_route("GET", "/unity-catalog/functions", status=503, message="unavailable") + before = len(fake.sql_log) + run.prepare() + assert run.drops(before) == [] and VIEW in fake.tables + assert "not checked: main.sales could not be read" in run.text + run.destroy_table(OLD) + assert run.unblock() is False + + +def test_a_drop_that_fails_is_named_and_the_destroy_is_not_asked_for_again(fake): + run = Run(fake) + run.deploy(_contract()) + run.prepare() + run.destroy_table(OLD) + fake.fail_on("DROP FUNCTION", "[INTERNAL_ERROR] injected failure") + assert run.unblock() is False + assert "FAILED: drop function main.sales.fluid_" in run.text + + +def test_without_a_client_nothing_is_checked_and_it_says_so(fake): + def broken() -> Any: + raise RuntimeError("no credential") + + run = Run(fake) + run.prepare(client_factory=broken) + assert "main.sales: what this product made outside OpenTofu there" in run.text + assert "was not checked: no Databricks client" in run.text + assert run.unblock(client_factory=broken) is False + + +def test_vacated_takes_the_schemas_the_plan_destroys_that_the_contract_does_not_name(): + destroyed = [ + *DESTROYED, + { + "address": "databricks_schema.k", + "type": "databricks_schema", + "before": {"id": "main.kept"}, + }, + { + "address": "databricks_sql_table.k", + "type": "databricks_sql_table", + "before": {"catalog_name": "main", "schema_name": "kept", "name": "t"}, + }, + {"address": "databricks_grant.g", "type": "databricks_grant", "before": {"id": "main.x"}}, + {"address": "databricks_schema.odd", "type": "databricks_schema", "before": None}, + ] + found = vacated_mod.vacated(destroyed, {"main.kept"}, "dbx.fake.orders") + assert found.schemas == ("main.sales",) + assert found.tables == frozenset({OLD}) + assert found.product_id == "dbx.fake.orders" + # Without catalog_name and name, the id names the schema. + only_id = [{"type": "databricks_schema", "before": {"id": "main.legacy"}}] + assert vacated_mod.vacated(only_id, (), "p").schemas == ("main.legacy",) From 4b4de0ffe3e90578b5ccfdf77a8da16860ece06c Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:02:22 +0100 Subject: [PATCH 3/5] fix(databricks): a function a table names without its schema still counts as bound The Tables API names a row filter's or a mask's function in full, and the rules that clear a schema the plan destroys read it that way. A shorter name is now read against the table's own catalog and schema, so no form of it can make a function a table still uses look unused and be dropped while the table exists (raised by the branch's security review as the one assumption that would change its verdict). The resume's lead line says why the schema waited, and HONESTLY_TESTED counts the tests the branch now has (10 + 23 + 2). --- HONESTLY_TESTED.md | 2 +- fluid_build/iac/providers/databricks_vacated.py | 11 ++++++++--- tests/iac/test_iac_databricks_vacated.py | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/HONESTLY_TESTED.md b/HONESTLY_TESTED.md index 966ca9aa..6a6fde61 100644 --- a/HONESTLY_TESTED.md +++ b/HONESTLY_TESTED.md @@ -93,7 +93,7 @@ Markers declared in `pyproject.toml`. | Stage 1 unit: Databricks retention (`lifecycle.expire`) | `tests/iac/test_iac_databricks_retention.py`, `tests/cli/test_verify_databricks_retention.py` | ✅ 195 + 18. ISO 8601 durations (years and months at their shortest, P1Y = 365 and P1M = 28 days; a fraction, a sign, a lower-case letter, `PT`, `P1DT`, out-of-order and over-long numbers refused), Delta `interval` properties (rounded up, months refused) and the history as the larger of the two properties with Delta's defaults. A dbt model's `materialized` resolved from `config()` in the SQL, the model's YAML `config:` and `dbt_project.yml` (the `+` prefix or none, root, project, folder and model-name levels, the most specific wins, SQL over YAML over project), the default `view`, the model found by alias or file name, commented-out and `raw` configs ignored, a `config()` inside a Jinja block, `var()`, `**`, a custom schema or database, a Python model, a versioned or twin model, a linked file and a Git repository refused as undeterminable. The build's `--select` must include the resolved model (name, `fqn`, `path:`, `*`, unions and intersections; a graph operator on another node, a tag, a config or run state is refused as not evaluable). Accepted: a dbt `table` model and a DuckDB acquisition with P30D, P365D, P6Y; refused: P7D, P29D, P4W, P1M, a history set longer than the retention, `incremental`, `snapshot`, `view`, another acquisition engine, no writer, a mixed set of writers. `fluid validate` and the emitter agree on kind and message. `fluid verify`'s retention dimension against a stubbed client and, over HTTP, the stand-in workspace (pass, fail with the `ALTER TABLE`, the runtime property over the declared one, skip, predictive optimization off as a warning). | Live behaviour: no real Databricks endpoint was called. Whether Unity Catalog's GetTable serves a table's `delta.logRetentionDuration` and `delta.deletedFileRetentionDuration` in `properties` or `delta_runtime_properties_kvpairs` (read from databricks-sdk-go v0.190.0 `model.go`, not observed), whether `effective_predictive_optimization_flag` is filled for a managed table, that predictive optimization runs `VACUUM` within the retention (a Databricks guarantee, not tested), and that dbt-databricks `table` replaces the table each run. Delta history is only half the promise: the acquisition acceptance rests on the landing being an `INSERT OVERWRITE` of the whole table in the one mode it runs (`full_refresh`, read from the landing's own `LOAD_MODES`; another source mode is refused as `incremental`, none as `unmapped`), which the bronze landing row covers against the stand-in, not live. A GetTable answer with no properties is a verify warning (Delta's defaults assumed). Embedded-logic DuckDB builds, `dlt`, `airbyte` and the other acquisition engines are refused. Selection is matched on names and paths only, not on dbt's graph: `+other+` is refused even if `other` is upstream of the model. | | Stage 1: Databricks Unity Catalog metadata (comments, product tags, column tags; live: the 2026-11-04 row) | `tests/iac/test_iac_databricks_metadata.py` (the emitter), `tests/iac/test_iac_databricks_metadata_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against `tests/iac/_fake_databricks.py`), `tests/cli/test_verify_databricks.py` (the comment and column-tag cases) | ✅ 43 + 5 + 7. The table's `comment` is the expose's description, else the product's (its `purpose` when structured), each described column's `comment` its schema description, stripped; none on a table a dbt build writes (dbt's CREATE OR REPLACE owns them; `fluid validate` warns). The product's identity and owner team are tags on the table only (`fluid_product_id`, `fluid_product_name`, `fluid_domain`, `fluid_layer`, `fluid_product_type`, `fluid_product_version`, `fluid_version`, `fluid_owner_team`), an unstated field makes none, and they count with the governance labels toward 50; `lifecycle.state: deprecated` (the expose's, else the product's) sets `system.certification_status = deprecated`, else the label `certification: certified` (the expose's, else the product's) sets `certified` and is no plain tag (a product-level label certifies every expose's table, as intended; an expose-level one only its own); another label value is `databricks-certification-invalid`; a plan notice says the identity needs ASSIGN on that governed tag. Against the stand-in: certified, then deprecated in place, then removed. A classified column (`sensitivity`) carries `fluid_sensitivity` as a `columns` tag named `catalog.schema.table.column`, imported best-effort by the provider's id. Refused through `validate` and `emit` alike: a backslash in any comment (`databricks-comment-unsafe`; the provider's `parseComment` escapes only quotes) and a `${` (Spark variable substitution), a label meeting a product key (`databricks-tag-collision`), a value over 256 characters (`databricks-tag-value-too-long`), over 1,000 column tags (`databricks-tags-too-many`); a `${...}` in a tag value is rendered as a literal. Against the stand-in: apply writes the comments (an apostrophe included, through the provider's escaping), the product tags and the column tag, never an `OWNER TO`; a second plan is empty; a changed description is an in-place `COMMENT ON TABLE`, never a replace; removing a classification destroys only its column tag; a table made before this is adopted and gains its comments in place (`+5 ~1` for the product tags and the table update). `fluid verify` passes after the apply and fails a comment or column tag changed by hand; against a stubbed client it reads each column's tag list to its last page, ignores a tag or comment someone else added, and checks no comment on a dbt-written table. | **Not run live.** What real Unity Catalog stores and returns for a table or column comment (whitespace, newlines, length limits, which the docs do not state), whether it accepts a tag value with `, . : / = -` (the docs restrict only keys), whether `entity-tag-assignments/columns/{catalog.schema.table.column}` resolves as the SDK documents, and whether a dbt CREATE OR REPLACE keeps column tags (undocumented; `fluid verify` is the gate), and whether the Entity Tag Assignments API sets the system tag `system.certification_status` (its docs show the UI and `SET TAG`; the SDK documents governed tags through the API, needing ASSIGN; the stand-in checks no permission). A contract that adds a column while comments change for the first time hits the provider's mixed-change refusal (its fix-it says to apply in two steps). | | Stage 1: Databricks metric views (`exposes[].semantics`) | `tests/iac/test_iac_databricks_metrics.py` (the YAML, the refusals and the hooks against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_databricks_metrics_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against the stand-in) | ✅ 91 + 4. A site-360-shaped semantics block compiles to the metric view YAML it should (the entity as its key column and categorical and time dimensions as dimensions, with comments, display names and synonyms from the columns; non-additive measures as `semiadditive: last` windows; filtered simple metrics as `FILTER (WHERE ...)`; a ratio as `MEASURE(a) / NULLIF(MEASURE(b), 0)`, after its inputs); the view is `
_semantic_model` whatever the model is called; every column reference is `source.` (`
.` too), a unit or type keyword (`EXTRACT(YEAR ...)`, `INTERVAL 1 DAY`, `::date`) stays one; the time grains (`TRUNC` on a `DATE`, `DATE_TRUNC` otherwise); 42 refusal cases (names, collisions, expressions, quoting, calls that would lend the owner's rights, text limits). The hooks: a first apply creates the view with its `fluid.*` properties and grants it to the table's readers; a second runs nothing; a changed or hand-edited definition is an `ALTER VIEW` that keeps the grants; `SELECT` and `ALL PRIVILEGES` are revoked from anyone else, before the apply and first in it, so a failed grant or update leaves no removed reader; a view the contract no longer declares, in a schema the product's tables are in, is dropped before the apply, another product's is left and another environment's (same id, another schema of the catalog) is kept; a table, a view without this product's property or one another identity owns at the name is refused before the apply; a failed statement is `databricks-metric-view-failed` naming the view; without databricks-sdk a declared view is refused; when a restricted table's binding fails, `SELECT` on its views is revoked. Through the CLI: apply, a second apply `+0 ~0 -0` with no statement, dry-run prints and runs nothing, dropping the semantics drops the view, `fluid validate` refuses a derived metric. All 19 industry-model tower-management contracts and the 4 forge templates that bind a table compile with no refusal. | What Databricks accepts and answers (the 2026-11-12 live row); Genie and AI/BI dashboards themselves. A schema the contract stops using: the next row. | -| Stage 1: Databricks, a schema the contract stops using | `tests/iac/test_iac_databricks_vacated.py` (the rules, against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_apply_engine_destroys.py` (the engine, `tofu` stubbed), 2 of `tests/iac/test_iac_databricks_metrics_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against the stand-in) | ✅ 9 + 19 + 2. A table with a metric view, a mask and a row filter moves from `main.sales` to `main.other`: `--mode dry-run` prints the `DROP VIEW` before the apply and the `DROP FUNCTION`s after the table's destroy, and runs none; the apply drops the view before `tofu apply`, `tofu apply` destroys the old table and is refused the schema (the stand-in, like Unity Catalog, deletes no schema that holds a table, view, function or volume), the functions are dropped once no table uses them, and a second, destroy-only plan (`+0 ~0 -1`) destroys the schema; the run exits 0 with the first plan's totals, and the next apply is `+0 ~0 -0`. Removing a schema's last expose does the same. Only schemas the saved plan destroys and the contract does not name are looked in, so a schema the contract still uses and another environment's (same product id, same identity, another schema) are never read. In the destroyed schema, a view of another product, one another identity owns and one over a table the plan keeps are left, and so are a function a kept table uses, one another identity owns and one of another name; an unused function of this module's name shape that the deploy identity owns is dropped; each object left is named with why. A schema that cannot be read drops nothing; a failed drop, a plugin that clears nothing or raises, a failure the plan's deletes do not account for, and a second plan holding any other change all leave the apply's own failure. | Live Databricks: that the provider destroys the table before its schema and Unity Catalog then deletes the schema once its functions are dropped (the 2026-11-03 run measured only that a schema holding functions is not deleted); whether a function re-created under a dropped one's name would be bound again (why a function a table still uses is never dropped); a table in another schema whose mask names a function here; a volume or a model in the schema (neither dropped nor named). | +| Stage 1: Databricks, a schema the contract stops using | `tests/iac/test_iac_databricks_vacated.py` (the rules, against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_apply_engine_destroys.py` (the engine, `tofu` stubbed), 2 of `tests/iac/test_iac_databricks_metrics_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against the stand-in) | ✅ 10 + 23 + 2. A table with a metric view, a mask and a row filter moves from `main.sales` to `main.other`: `--mode dry-run` prints the `DROP VIEW` before the apply and the `DROP FUNCTION`s after the table's destroy, and runs none; the apply drops the view before `tofu apply`, `tofu apply` destroys the old table and is refused the schema (the stand-in, like Unity Catalog, deletes no schema that holds a table, view, function or volume), the functions are dropped once no table uses them, and a second, destroy-only plan (`+0 ~0 -1`) destroys the schema; the run exits 0 with the first plan's totals, and the next apply is `+0 ~0 -0`. Removing a schema's last expose does the same. Only schemas the saved plan destroys and the contract does not name are looked in, so a schema the contract still uses and another environment's (same product id, same identity, another schema) are never read. In the destroyed schema, a view of another product, one another identity owns and one over a table the plan keeps are left, and so are a function a kept table uses (named in full or not), one another identity owns and one of another name; an unused function of this module's name shape that the deploy identity owns is dropped; each object left is named with why. A schema that cannot be read drops nothing; a failed drop, a plugin that clears nothing or raises, a failure the plan's deletes do not account for, and a second plan holding any other change all leave the apply's own failure. | Live Databricks: that the provider destroys the table before its schema and Unity Catalog then deletes the schema once its functions are dropped (the 2026-11-03 run measured only that a schema holding functions is not deleted); whether a function re-created under a dropped one's name would be bound again (why a function a table still uses is never dropped); a table in another schema whose mask names a function here; a volume or a model in the schema (neither dropped nor named). | | Manual live, Free Edition (founder-approved), 2026-11-12, metric views | the real `fluid` CLI, no test file: this branch, databricks-sdk 0.166.0, a scratch catalog `fluid_iactest_metrics` dropped `CASCADE` after | ✅ The semantics block of industry-model `dp019_site_360` verbatim (13 dimensions, 3 non-additive measures, 4 filtered metrics) on a table shaped like it: dry-run printed the `CREATE VIEW` and grants; apply `+77`, then the view created and `SELECT` granted to the 3 reader principals; `SELECT country_code, MEASURE(site_count), MEASURE(avg_tenancy_ratio), ... GROUP BY ALL` returned each market at its latest snapshot (8 rows seeded, 4 sites x 2 days); by date, each day's own values; a second apply `+0 ~0 -0` ran no statement. `information_schema.tables` lists it as `METRIC_VIEW` with its comment, `.columns` every dimension and measure with its comment, `.table_privileges` the 3 `SELECT` grants. Probes on the same warehouse (Databricks SQL 2026.45): `CREATE OR REPLACE VIEW` drops a view's grants and `ALTER VIEW ... AS` keeps them and its properties; the stored `view_definition` leaves out `comment`, `display_name` and `synonyms`; a dimension named like a column shadows it in a measure's filter and a window measure named like a column breaks the window measures that read it (`METRIC_VIEW_WINDOW_MEASURE_REFERENCES_WINDOW_MEASURE`), and `source.` reads the column; `semiadditive: last` is per output group; `FILTER (WHERE ...)` works on a window measure, `COUNT(DISTINCT ...)` and `PERCENTILE_CONT ... WITHIN GROUP`; `TBLPROPERTIES` combine with `WITH METRICS`; a metric view over a row-filtered table applies the filter to the reader; a table at the view's name fails `EXPECT_VIEW_NOT_TABLE`. | A query as a reader persona (the run had the deploy user's token only; the persona's effective `SELECT` was read from the permissions API); Genie and dashboards; a contract with column masks. | | Stage 1: Databricks data quality monitoring (live: the 2026-11-12 row) | `tests/iac/test_iac_databricks_quality.py` (what asks for it, the monitor per schema, the derived import, the notice, the engine), `tests/iac/test_iac_databricks_quality_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_tofu_validate.py::test_a_databricks_data_quality_monitor_passes_tofu_validate` | ✅ 37 + 6 + 1. A freshness or completeness SLO (`qos.freshnessSLO`, `qos.completenessTarget`, an `observability.defaultSLIs` freshness or completeness block, a `contract.dq.rules` entry of type `freshness`, `completeness` or `anomaly_detection`) puts one `databricks_data_quality_monitor` on the owned schema, watching it by `schema_id`, created after the schema's tables, the product's other tables there in `excluded_table_full_names`; `contract.dq.monitoring.enabled: false` removes only the monitor; a `dq.monitoring.window` other than one day is a `fluid validate` warning. Against the stand-in: a second plan is empty; a monitor turned on outside forge-cli is adopted by `schema,` taken from state after the schema's import (no create); the last exclusion leaving is a PATCH whose `update_mask` names the whole config (the stand-in honours the mask), and the plan converges; a monitor Databricks refuses fails the apply after the tables, its suggestions saying what it needs and naming the opt-out (`apply_failure_hints`, after the three generic ones); opting out deletes it, not gated; `tofu destroy` deletes it before the schema. | What Databricks scans and reports (the live row); the stand-in serves schema monitors only. | | Manual live, Free Edition (founder-approved) | the real `fluid` CLI, no test file: run 2026-11-01 from main dfd6f145, databricks-sdk 0.158.0, every `fluid_iactest_*` object swept after (1 schema, 1 table, 1 folder; 0 left) | ✅ validate, `generate iac --validate`, plan, dry-run and apply `+9`; a second plan is EMPTY against the real Jobs API; Databricks accepts the Quartz schedule, serverless environment 2 with `dbt-databricks>=1.8.0,<2.0.0` (dbt 1.12.6, adapter 1.13.0), the WORKSPACE `project_directory` and the commands as emitted; one `run-now` ends SUCCESS and dbt builds the table; `fluid verify` with the real SDK (columns, types, one job); `fluid diff`'s state check (its live check has no Databricks inspector); a model change `~1`; the build made local removes job, files and folders `-5` without `--allow-data-loss` (the empty `.fluid//` folders above `files/` stay: only `files/` and below are managed). It found the USE CATALOG check reading a group-conveyed privilege as none. After the fix, `fluid verify` on a fresh one-table product (`+4`) passes every dimension, catalog access included (1 schema and 1 table swept, 0 left). | A GIT source, profile or OAuth auth, a run-as service principal, table adoption and a larger project, live; the region residency check (the metastore precondition and `fluid verify`'s residency dimension), which was added after this run and not run live; dbt's CREATE OR REPLACE drops a contract's NOT NULL, so verify flags a `required` column a dbt model does not constrain itself. | diff --git a/fluid_build/iac/providers/databricks_vacated.py b/fluid_build/iac/providers/databricks_vacated.py index f599b9b0..b4e7f1e6 100644 --- a/fluid_build/iac/providers/databricks_vacated.py +++ b/fluid_build/iac/providers/databricks_vacated.py @@ -197,13 +197,18 @@ def _exists(session: Session, full: str, present: Iterable[str]) -> bool: def _holders(objects: Mapping[str, Mapping[str, Any]]) -> Dict[str, List[str]]: - """``{function (bare, lower case): [objects whose row filter or a mask names it]}``.""" + """``{function (full name, lower case): [objects whose row filter or a mask names it]}``. + + The Tables API names the function in full; a shorter name is read against the object's own + catalog and schema, so no form of it can make a bound function look unbound.""" holders: Dict[str, List[str]] = {} for full, info in sorted(objects.items()): bound = [(info.get("row_filter") or {}).get("function_name")] bound += [(c.get("mask") or {}).get("function_name") for c in _live_columns(info).values()] for name in filter(None, bound): - holders.setdefault(_bare(name), []).append(full) + parts = _bare(name).split(".") + home = full.lower().split(".")[: max(0, 3 - len(parts))] + holders.setdefault(".".join([*home, *parts]), []).append(full) return holders @@ -380,7 +385,7 @@ def unblock(vac: Vacated, say: Say, client_factory: Optional[ClientFactory] = No return False dropped, clear = 0, True for where in vac.schemas: - say(f"\n {where}: tofu apply could not destroy it yet; what this product made there") + say(f"\n {where}: tofu apply could not destroy it yet: it held what this product made") out = _Clearing(session, Vacated(vac.product_id, (where,)), where, say, False).run() _say_left(where, out, say) dropped += out.dropped diff --git a/tests/iac/test_iac_databricks_vacated.py b/tests/iac/test_iac_databricks_vacated.py index e08058a1..a70ddf9e 100644 --- a/tests/iac/test_iac_databricks_vacated.py +++ b/tests/iac/test_iac_databricks_vacated.py @@ -259,6 +259,21 @@ def test_only_this_products_and_this_identitys_objects_go_and_never_a_bound_func assert f"main.sales.{view}_semantic_model" in fake.tables +def test_a_function_a_table_names_without_its_schema_still_counts_as_used(fake): + run = Run(fake) + fake.create_table("main.sales.orders_stg", LIVE) + bound = "main.sales.fluid_rf_orders_stg_12345678" + fake.add_function(bound, [("region", "string")], "BOOLEAN", "true", owner=DEPLOY_USER) + fake.tables["main.sales.orders_stg"]["row_filter"] = { + "function_name": "fluid_rf_orders_stg_12345678", + "input_column_names": ["region"], + } + before = len(fake.sql_log) + run.prepare() + assert run.drops(before) == [] and bound in fake.functions + assert f"{bound} (the row filter or a mask of main.sales.orders_stg)" in run.text + + def test_a_schema_the_contract_still_names_is_not_looked_in(fake): run = Run(fake) run.deploy(_contract()) From e5be2f57be40359d8e085ecfa0fb8b49cd888d25 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:21:13 +0100 Subject: [PATCH 4/5] fix(databricks): clearing a schema the plan destroys never raises, and resumes only what it cleared From the branch's defect scan and inspection: - A metric view definition PyYAML cannot load (an impossible date is a ValueError, not a YAMLError) or a Tables API row of an unexpected shape no longer escapes prepare_destroys as a traceback before tofu apply: the definition names no table, the row is read defensively, and anything else that breaks while one schema is read leaves that schema as it is, with a note. - The product id comes from the contract (databricks_metrics.product_id_of, shared with compile_metric_views), so a contract with no Databricks expose left, which compiles no metric views, still names its views. - unblock_destroys returns the plan addresses it cleared, and the engine resumes only when they are every resource the apply failed on: a delete that failed for another reason is no longer retried behind the cleared ones. - The schema is read through table-summaries as well as the Tables list, so a metric view is found whichever listing Unity Catalog gives it in. - The dry-run heading fits a schema holding only objects this product did not make; iac/base.py lists the two new optional capabilities; HONESTLY_TESTED counts the tests (13 + 24 + 2) and, from the #865 docs check, the metric-view row's 5 stand-in tests and that CREATE OR REPLACE VIEW makes the replacer the view's owner. --- HONESTLY_TESTED.md | 6 +- fluid_build/cli/_apply_opentofu_engine.py | 19 ++-- fluid_build/iac/base.py | 8 ++ fluid_build/iac/providers/databricks.py | 11 ++- .../iac/providers/databricks_metrics.py | 9 +- .../iac/providers/databricks_vacated.py | 94 +++++++++++++------ tests/iac/test_iac_apply_engine_destroys.py | 12 ++- tests/iac/test_iac_databricks_vacated.py | 75 +++++++++++++-- 8 files changed, 178 insertions(+), 56 deletions(-) diff --git a/HONESTLY_TESTED.md b/HONESTLY_TESTED.md index 6a6fde61..d829a4b2 100644 --- a/HONESTLY_TESTED.md +++ b/HONESTLY_TESTED.md @@ -92,9 +92,9 @@ Markers declared in `pyproject.toml`. | Stage 1 unit + manual live: contract DQ rules as dbt tests in a Databricks dbt job | `tests/util/test_dq_checks.py` (the reader), `tests/iac/test_iac_databricks_dbt_tests.py` (the tests, each rendered in a dbt-shaped Jinja context, at parse and at execute), `tests/build_runners/test_databricks_job_run.py` (the run's report) | ✅ 51 + 65 + 4. Every rule type read once (`util/dq_checks.py`): selectors split on commas; a `valid_values` rule read as `fluid test`'s `value_domain` reads it (the rule's own list, else the column's first `enum`, split on `|` and `,`, `regex` and `range`, and `labels.references` through `consumes[]`); shares below 1.0, the severity map, unique names; each refusal with its fix. Each check rendered: dbt's own `unique`, `not_null`, `accepted_values`, `relationships` through `adapter.dispatch`, forge-cli's regex, bounds, freshness, schema and share SQL; hostile values (`'`, `\\`, `}}`, `{%`, `#}`, a character outside the BMP) stay inside their Jinja string and SQL literal; every test refs its model while dbt parses, and a real dbt (dbt-core with dbt-duckdb) parses the upload and `dbt ls --select +orders+` lists every rule's test. Refused at plan: a test name the project's own tests use, a selection that leaves the model out, a threshold no float holds, an empty range, an unpaired surrogate (in a rule, a value or the test path; one from the project's YAML is named escaped in a refusal); an `anomaly_detection` rule is left to data quality monitoring (#864) unless the expose turns it off, and one monitoring could not read either (a bad severity) is still refused. The `fluid_keys` task (#860) runs with `run_if: ALL_DONE`, so a failing rule does not leave the table without its keys. **Live, Databricks Free Edition, 2026-11-12** (scratch catalog `fluid_iactest_dq`, swept after: job, upload folder, catalog; 0 left): one dbt product, 9 rules as 11 tests, `fluid apply --mode amend-and-build` (`tofu` +27). Run 1, with a seeded bad row, ended `RUN_EXECUTION_ERROR`: completeness on `status` and the `amount` range `FAIL 1` (error), the country regex and the customer reference `WARN 1`, six `PASS`, each printed by `fluid apply`; it also showed the schema test was never selected (its `ref` sat behind `if execute`: dbt found 11 tests, ran 10), fixed. Run 2, clean: `SUCCESS`, the model and all 11 tests `PASS`; the next plan `+0 ~0 -0`. Offline: the demo's 14 dbt products compile to 230 tests with no refusal, and `dbt parse` / `dbt ls --select +model+` (dbt-core 1.12.5) select all 20 of DP005. | A share below 1.0 that fails, a reference share, and a regex or bounds share were not run live (only a passing completeness share). A Git-sourced build with rules is refused by design (no file can be added to a checkout). Not handled: a versioned model, and a `dbt_project.yml` that sets `+where` or `+limit` on tests, which would change the generated tests too. dbt's result lines are read from the task's output: lines Databricks cut from a long output are reported as not in it. The run page was read through the Jobs API (`runs/get-output`), not in a browser. | | Stage 1 unit: Databricks retention (`lifecycle.expire`) | `tests/iac/test_iac_databricks_retention.py`, `tests/cli/test_verify_databricks_retention.py` | ✅ 195 + 18. ISO 8601 durations (years and months at their shortest, P1Y = 365 and P1M = 28 days; a fraction, a sign, a lower-case letter, `PT`, `P1DT`, out-of-order and over-long numbers refused), Delta `interval` properties (rounded up, months refused) and the history as the larger of the two properties with Delta's defaults. A dbt model's `materialized` resolved from `config()` in the SQL, the model's YAML `config:` and `dbt_project.yml` (the `+` prefix or none, root, project, folder and model-name levels, the most specific wins, SQL over YAML over project), the default `view`, the model found by alias or file name, commented-out and `raw` configs ignored, a `config()` inside a Jinja block, `var()`, `**`, a custom schema or database, a Python model, a versioned or twin model, a linked file and a Git repository refused as undeterminable. The build's `--select` must include the resolved model (name, `fqn`, `path:`, `*`, unions and intersections; a graph operator on another node, a tag, a config or run state is refused as not evaluable). Accepted: a dbt `table` model and a DuckDB acquisition with P30D, P365D, P6Y; refused: P7D, P29D, P4W, P1M, a history set longer than the retention, `incremental`, `snapshot`, `view`, another acquisition engine, no writer, a mixed set of writers. `fluid validate` and the emitter agree on kind and message. `fluid verify`'s retention dimension against a stubbed client and, over HTTP, the stand-in workspace (pass, fail with the `ALTER TABLE`, the runtime property over the declared one, skip, predictive optimization off as a warning). | Live behaviour: no real Databricks endpoint was called. Whether Unity Catalog's GetTable serves a table's `delta.logRetentionDuration` and `delta.deletedFileRetentionDuration` in `properties` or `delta_runtime_properties_kvpairs` (read from databricks-sdk-go v0.190.0 `model.go`, not observed), whether `effective_predictive_optimization_flag` is filled for a managed table, that predictive optimization runs `VACUUM` within the retention (a Databricks guarantee, not tested), and that dbt-databricks `table` replaces the table each run. Delta history is only half the promise: the acquisition acceptance rests on the landing being an `INSERT OVERWRITE` of the whole table in the one mode it runs (`full_refresh`, read from the landing's own `LOAD_MODES`; another source mode is refused as `incremental`, none as `unmapped`), which the bronze landing row covers against the stand-in, not live. A GetTable answer with no properties is a verify warning (Delta's defaults assumed). Embedded-logic DuckDB builds, `dlt`, `airbyte` and the other acquisition engines are refused. Selection is matched on names and paths only, not on dbt's graph: `+other+` is refused even if `other` is upstream of the model. | | Stage 1: Databricks Unity Catalog metadata (comments, product tags, column tags; live: the 2026-11-04 row) | `tests/iac/test_iac_databricks_metadata.py` (the emitter), `tests/iac/test_iac_databricks_metadata_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against `tests/iac/_fake_databricks.py`), `tests/cli/test_verify_databricks.py` (the comment and column-tag cases) | ✅ 43 + 5 + 7. The table's `comment` is the expose's description, else the product's (its `purpose` when structured), each described column's `comment` its schema description, stripped; none on a table a dbt build writes (dbt's CREATE OR REPLACE owns them; `fluid validate` warns). The product's identity and owner team are tags on the table only (`fluid_product_id`, `fluid_product_name`, `fluid_domain`, `fluid_layer`, `fluid_product_type`, `fluid_product_version`, `fluid_version`, `fluid_owner_team`), an unstated field makes none, and they count with the governance labels toward 50; `lifecycle.state: deprecated` (the expose's, else the product's) sets `system.certification_status = deprecated`, else the label `certification: certified` (the expose's, else the product's) sets `certified` and is no plain tag (a product-level label certifies every expose's table, as intended; an expose-level one only its own); another label value is `databricks-certification-invalid`; a plan notice says the identity needs ASSIGN on that governed tag. Against the stand-in: certified, then deprecated in place, then removed. A classified column (`sensitivity`) carries `fluid_sensitivity` as a `columns` tag named `catalog.schema.table.column`, imported best-effort by the provider's id. Refused through `validate` and `emit` alike: a backslash in any comment (`databricks-comment-unsafe`; the provider's `parseComment` escapes only quotes) and a `${` (Spark variable substitution), a label meeting a product key (`databricks-tag-collision`), a value over 256 characters (`databricks-tag-value-too-long`), over 1,000 column tags (`databricks-tags-too-many`); a `${...}` in a tag value is rendered as a literal. Against the stand-in: apply writes the comments (an apostrophe included, through the provider's escaping), the product tags and the column tag, never an `OWNER TO`; a second plan is empty; a changed description is an in-place `COMMENT ON TABLE`, never a replace; removing a classification destroys only its column tag; a table made before this is adopted and gains its comments in place (`+5 ~1` for the product tags and the table update). `fluid verify` passes after the apply and fails a comment or column tag changed by hand; against a stubbed client it reads each column's tag list to its last page, ignores a tag or comment someone else added, and checks no comment on a dbt-written table. | **Not run live.** What real Unity Catalog stores and returns for a table or column comment (whitespace, newlines, length limits, which the docs do not state), whether it accepts a tag value with `, . : / = -` (the docs restrict only keys), whether `entity-tag-assignments/columns/{catalog.schema.table.column}` resolves as the SDK documents, and whether a dbt CREATE OR REPLACE keeps column tags (undocumented; `fluid verify` is the gate), and whether the Entity Tag Assignments API sets the system tag `system.certification_status` (its docs show the UI and `SET TAG`; the SDK documents governed tags through the API, needing ASSIGN; the stand-in checks no permission). A contract that adds a column while comments change for the first time hits the provider's mixed-change refusal (its fix-it says to apply in two steps). | -| Stage 1: Databricks metric views (`exposes[].semantics`) | `tests/iac/test_iac_databricks_metrics.py` (the YAML, the refusals and the hooks against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_databricks_metrics_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against the stand-in) | ✅ 91 + 4. A site-360-shaped semantics block compiles to the metric view YAML it should (the entity as its key column and categorical and time dimensions as dimensions, with comments, display names and synonyms from the columns; non-additive measures as `semiadditive: last` windows; filtered simple metrics as `FILTER (WHERE ...)`; a ratio as `MEASURE(a) / NULLIF(MEASURE(b), 0)`, after its inputs); the view is `
_semantic_model` whatever the model is called; every column reference is `source.` (`
.` too), a unit or type keyword (`EXTRACT(YEAR ...)`, `INTERVAL 1 DAY`, `::date`) stays one; the time grains (`TRUNC` on a `DATE`, `DATE_TRUNC` otherwise); 42 refusal cases (names, collisions, expressions, quoting, calls that would lend the owner's rights, text limits). The hooks: a first apply creates the view with its `fluid.*` properties and grants it to the table's readers; a second runs nothing; a changed or hand-edited definition is an `ALTER VIEW` that keeps the grants; `SELECT` and `ALL PRIVILEGES` are revoked from anyone else, before the apply and first in it, so a failed grant or update leaves no removed reader; a view the contract no longer declares, in a schema the product's tables are in, is dropped before the apply, another product's is left and another environment's (same id, another schema of the catalog) is kept; a table, a view without this product's property or one another identity owns at the name is refused before the apply; a failed statement is `databricks-metric-view-failed` naming the view; without databricks-sdk a declared view is refused; when a restricted table's binding fails, `SELECT` on its views is revoked. Through the CLI: apply, a second apply `+0 ~0 -0` with no statement, dry-run prints and runs nothing, dropping the semantics drops the view, `fluid validate` refuses a derived metric. All 19 industry-model tower-management contracts and the 4 forge templates that bind a table compile with no refusal. | What Databricks accepts and answers (the 2026-11-12 live row); Genie and AI/BI dashboards themselves. A schema the contract stops using: the next row. | -| Stage 1: Databricks, a schema the contract stops using | `tests/iac/test_iac_databricks_vacated.py` (the rules, against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_apply_engine_destroys.py` (the engine, `tofu` stubbed), 2 of `tests/iac/test_iac_databricks_metrics_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against the stand-in) | ✅ 10 + 23 + 2. A table with a metric view, a mask and a row filter moves from `main.sales` to `main.other`: `--mode dry-run` prints the `DROP VIEW` before the apply and the `DROP FUNCTION`s after the table's destroy, and runs none; the apply drops the view before `tofu apply`, `tofu apply` destroys the old table and is refused the schema (the stand-in, like Unity Catalog, deletes no schema that holds a table, view, function or volume), the functions are dropped once no table uses them, and a second, destroy-only plan (`+0 ~0 -1`) destroys the schema; the run exits 0 with the first plan's totals, and the next apply is `+0 ~0 -0`. Removing a schema's last expose does the same. Only schemas the saved plan destroys and the contract does not name are looked in, so a schema the contract still uses and another environment's (same product id, same identity, another schema) are never read. In the destroyed schema, a view of another product, one another identity owns and one over a table the plan keeps are left, and so are a function a kept table uses (named in full or not), one another identity owns and one of another name; an unused function of this module's name shape that the deploy identity owns is dropped; each object left is named with why. A schema that cannot be read drops nothing; a failed drop, a plugin that clears nothing or raises, a failure the plan's deletes do not account for, and a second plan holding any other change all leave the apply's own failure. | Live Databricks: that the provider destroys the table before its schema and Unity Catalog then deletes the schema once its functions are dropped (the 2026-11-03 run measured only that a schema holding functions is not deleted); whether a function re-created under a dropped one's name would be bound again (why a function a table still uses is never dropped); a table in another schema whose mask names a function here; a volume or a model in the schema (neither dropped nor named). | -| Manual live, Free Edition (founder-approved), 2026-11-12, metric views | the real `fluid` CLI, no test file: this branch, databricks-sdk 0.166.0, a scratch catalog `fluid_iactest_metrics` dropped `CASCADE` after | ✅ The semantics block of industry-model `dp019_site_360` verbatim (13 dimensions, 3 non-additive measures, 4 filtered metrics) on a table shaped like it: dry-run printed the `CREATE VIEW` and grants; apply `+77`, then the view created and `SELECT` granted to the 3 reader principals; `SELECT country_code, MEASURE(site_count), MEASURE(avg_tenancy_ratio), ... GROUP BY ALL` returned each market at its latest snapshot (8 rows seeded, 4 sites x 2 days); by date, each day's own values; a second apply `+0 ~0 -0` ran no statement. `information_schema.tables` lists it as `METRIC_VIEW` with its comment, `.columns` every dimension and measure with its comment, `.table_privileges` the 3 `SELECT` grants. Probes on the same warehouse (Databricks SQL 2026.45): `CREATE OR REPLACE VIEW` drops a view's grants and `ALTER VIEW ... AS` keeps them and its properties; the stored `view_definition` leaves out `comment`, `display_name` and `synonyms`; a dimension named like a column shadows it in a measure's filter and a window measure named like a column breaks the window measures that read it (`METRIC_VIEW_WINDOW_MEASURE_REFERENCES_WINDOW_MEASURE`), and `source.` reads the column; `semiadditive: last` is per output group; `FILTER (WHERE ...)` works on a window measure, `COUNT(DISTINCT ...)` and `PERCENTILE_CONT ... WITHIN GROUP`; `TBLPROPERTIES` combine with `WITH METRICS`; a metric view over a row-filtered table applies the filter to the reader; a table at the view's name fails `EXPECT_VIEW_NOT_TABLE`. | A query as a reader persona (the run had the deploy user's token only; the persona's effective `SELECT` was read from the permissions API); Genie and dashboards; a contract with column masks. | +| Stage 1: Databricks metric views (`exposes[].semantics`) | `tests/iac/test_iac_databricks_metrics.py` (the YAML, the refusals and the hooks against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_databricks_metrics_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against the stand-in) | ✅ 91 + 5. A site-360-shaped semantics block compiles to the metric view YAML it should (the entity as its key column and categorical and time dimensions as dimensions, with comments, display names and synonyms from the columns; non-additive measures as `semiadditive: last` windows; filtered simple metrics as `FILTER (WHERE ...)`; a ratio as `MEASURE(a) / NULLIF(MEASURE(b), 0)`, after its inputs); the view is `
_semantic_model` whatever the model is called; every column reference is `source.` (`
.` too), a unit or type keyword (`EXTRACT(YEAR ...)`, `INTERVAL 1 DAY`, `::date`) stays one; the time grains (`TRUNC` on a `DATE`, `DATE_TRUNC` otherwise); 42 refusal cases (names, collisions, expressions, quoting, calls that would lend the owner's rights, text limits). The hooks: a first apply creates the view with its `fluid.*` properties and grants it to the table's readers; a second runs nothing; a changed or hand-edited definition is an `ALTER VIEW` that keeps the grants; `SELECT` and `ALL PRIVILEGES` are revoked from anyone else, before the apply and first in it, so a failed grant or update leaves no removed reader; a view the contract no longer declares, in a schema the product's tables are in, is dropped before the apply, another product's is left and another environment's (same id, another schema of the catalog) is kept; a table, a view without this product's property or one another identity owns at the name is refused before the apply; a failed statement is `databricks-metric-view-failed` naming the view; without databricks-sdk a declared view is refused; when a restricted table's binding fails, `SELECT` on its views is revoked. Through the CLI: apply, a second apply `+0 ~0 -0` with no statement, dry-run prints and runs nothing, dropping the semantics drops the view, `fluid validate` refuses a derived metric, and the product's Genie space lists the view the same apply made. All 19 industry-model tower-management contracts and the 4 forge templates that bind a table compile with no refusal. | What Databricks accepts and answers (the 2026-11-12 live row); Genie and AI/BI dashboards themselves. A schema the contract stops using: the next row. | +| Stage 1: Databricks, a schema the contract stops using | `tests/iac/test_iac_databricks_vacated.py` (the rules, against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_apply_engine_destroys.py` (the engine, `tofu` stubbed), 2 of `tests/iac/test_iac_databricks_metrics_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against the stand-in) | ✅ 13 + 24 + 2. A table with a metric view, a mask and a row filter moves from `main.sales` to `main.other`: `--mode dry-run` prints the `DROP VIEW` before the apply and the `DROP FUNCTION`s after the table's destroy, and runs none; the apply drops the view before `tofu apply`, `tofu apply` destroys the old table and is refused the schema (the stand-in, like Unity Catalog, deletes no schema that holds a table, view, function or volume), the functions are dropped once no table uses them, and a second, destroy-only plan (`+0 ~0 -1`) destroys the schema; the run exits 0 with the first plan's totals, and the next apply is `+0 ~0 -0`. Removing a schema's last expose does the same. Only schemas the saved plan destroys and the contract does not name are looked in, so a schema the contract still uses and another environment's (same product id, same identity, another schema) are never read. In the destroyed schema, a view of another product, one another identity owns and one over a table the plan keeps are left, and so are a function a kept table uses (named in full or not), one another identity owns and one of another name; an unused function of this module's name shape that the deploy identity owns is dropped; each object left is named with why. A schema that cannot be read, a view definition YAML cannot load (an impossible date) and a row of an unexpected shape drop nothing and raise nothing; a metric view only the table-summaries listing gives is found; a contract with no Databricks expose left still names its views by its id. A failed drop, a plugin that clears nothing or raises, a failed delete the plugin did not clear, a failure the plan's deletes do not account for, and a second plan holding any other change all leave the apply's own failure. | Live Databricks: that the provider destroys the table before its schema and Unity Catalog then deletes the schema once its functions are dropped (the 2026-11-03 run measured only that a schema holding functions is not deleted); whether a function re-created under a dropped one's name would be bound again (why a function a table still uses is never dropped); whether Unity Catalog's Tables list includes metric views (both it and table-summaries are read, so either suffices); a table in another schema whose mask names a function here; a volume or a model in the schema (neither dropped nor named). | +| Manual live, Free Edition (founder-approved), 2026-11-12, metric views | the real `fluid` CLI, no test file: this branch, databricks-sdk 0.166.0, a scratch catalog `fluid_iactest_metrics` dropped `CASCADE` after | ✅ The semantics block of industry-model `dp019_site_360` verbatim (13 dimensions, 3 non-additive measures, 4 filtered metrics) on a table shaped like it: dry-run printed the `CREATE VIEW` and grants; apply `+77`, then the view created and `SELECT` granted to the 3 reader principals; `SELECT country_code, MEASURE(site_count), MEASURE(avg_tenancy_ratio), ... GROUP BY ALL` returned each market at its latest snapshot (8 rows seeded, 4 sites x 2 days); by date, each day's own values; a second apply `+0 ~0 -0` ran no statement. `information_schema.tables` lists it as `METRIC_VIEW` with its comment, `.columns` every dimension and measure with its comment, `.table_privileges` the 3 `SELECT` grants. Probes on the same warehouse (Databricks SQL 2026.45): `CREATE OR REPLACE VIEW` drops a view's grants and makes whoever replaces it its owner, and `ALTER VIEW ... AS` keeps its grants and its properties; the stored `view_definition` leaves out `comment`, `display_name` and `synonyms`; a dimension named like a column shadows it in a measure's filter and a window measure named like a column breaks the window measures that read it (`METRIC_VIEW_WINDOW_MEASURE_REFERENCES_WINDOW_MEASURE`), and `source.` reads the column; `semiadditive: last` is per output group; `FILTER (WHERE ...)` works on a window measure, `COUNT(DISTINCT ...)` and `PERCENTILE_CONT ... WITHIN GROUP`; `TBLPROPERTIES` combine with `WITH METRICS`; a metric view over a row-filtered table applies the filter to the reader; a table at the view's name fails `EXPECT_VIEW_NOT_TABLE`. | A query as a reader persona (the run had the deploy user's token only; the persona's effective `SELECT` was read from the permissions API); Genie and dashboards; a contract with column masks. | | Stage 1: Databricks data quality monitoring (live: the 2026-11-12 row) | `tests/iac/test_iac_databricks_quality.py` (what asks for it, the monitor per schema, the derived import, the notice, the engine), `tests/iac/test_iac_databricks_quality_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_tofu_validate.py::test_a_databricks_data_quality_monitor_passes_tofu_validate` | ✅ 37 + 6 + 1. A freshness or completeness SLO (`qos.freshnessSLO`, `qos.completenessTarget`, an `observability.defaultSLIs` freshness or completeness block, a `contract.dq.rules` entry of type `freshness`, `completeness` or `anomaly_detection`) puts one `databricks_data_quality_monitor` on the owned schema, watching it by `schema_id`, created after the schema's tables, the product's other tables there in `excluded_table_full_names`; `contract.dq.monitoring.enabled: false` removes only the monitor; a `dq.monitoring.window` other than one day is a `fluid validate` warning. Against the stand-in: a second plan is empty; a monitor turned on outside forge-cli is adopted by `schema,` taken from state after the schema's import (no create); the last exclusion leaving is a PATCH whose `update_mask` names the whole config (the stand-in honours the mask), and the plan converges; a monitor Databricks refuses fails the apply after the tables, its suggestions saying what it needs and naming the opt-out (`apply_failure_hints`, after the three generic ones); opting out deletes it, not gated; `tofu destroy` deletes it before the schema. | What Databricks scans and reports (the live row); the stand-in serves schema monitors only. | | Manual live, Free Edition (founder-approved) | the real `fluid` CLI, no test file: run 2026-11-01 from main dfd6f145, databricks-sdk 0.158.0, every `fluid_iactest_*` object swept after (1 schema, 1 table, 1 folder; 0 left) | ✅ validate, `generate iac --validate`, plan, dry-run and apply `+9`; a second plan is EMPTY against the real Jobs API; Databricks accepts the Quartz schedule, serverless environment 2 with `dbt-databricks>=1.8.0,<2.0.0` (dbt 1.12.6, adapter 1.13.0), the WORKSPACE `project_directory` and the commands as emitted; one `run-now` ends SUCCESS and dbt builds the table; `fluid verify` with the real SDK (columns, types, one job); `fluid diff`'s state check (its live check has no Databricks inspector); a model change `~1`; the build made local removes job, files and folders `-5` without `--allow-data-loss` (the empty `.fluid//` folders above `files/` stay: only `files/` and below are managed). It found the USE CATALOG check reading a group-conveyed privilege as none. After the fix, `fluid verify` on a fresh one-table product (`+4`) passes every dimension, catalog access included (1 schema and 1 table swept, 0 left). | A GIT source, profile or OAuth auth, a run-as service principal, table adoption and a larger project, live; the region residency check (the metastore precondition and `fluid verify`'s residency dimension), which was added after this run and not run live; dbt's CREATE OR REPLACE drops a contract's NOT NULL, so verify flags a `required` column a dbt model does not constrain itself. | | Manual live, Free Edition (founder-approved), masks and row filters | the real `fluid` CLI, no test file: run 2026-11-03 at the tip of #785, a real workspace with an AWS us-west-2 metastore; every object swept after (0 left) | ✅ 10 steps passed. Functions are created and read back (owner is the user's email; `routine_definition` is the body without `RETURN`, backticks kept; `full_data_type` is STRING or BOOLEAN; a zero-parameter function's FunctionInfo has no `input_params`). The row filter and masks are bound and enforced on the restricted user's own SELECT: only the consented rows, `msisdn` as `XXXXX4567`, a 3-character value hashed, the denied column NULL. `SET ROW FILTER ... ON ()` is accepted and a `SET` replaces a live filter. Tags are present, `fluid verify` passes every dimension, a second apply is `+0 ~0 -0`, the residency precondition fails the plan and names both regions, `generate iac` refuses, and the sweep left 0. | Still open, as this run showed: superseded functions were left behind (orphans; the follow-up that drops them after a successful apply is not run live); `tofu destroy` of the module fails at a schema that still holds functions, so a teardown must drop them first; and the residency failure text was forge-cli's 800-character tail of the tofu JSON, which cut off the sentence naming the live region (the one-line explanation is not run live). A metastore in another cloud or region, a service principal as the run identity, and a group-conveyed grant on the live workspace are still not run. | diff --git a/fluid_build/cli/_apply_opentofu_engine.py b/fluid_build/cli/_apply_opentofu_engine.py index 8449364d..eb50cd15 100644 --- a/fluid_build/cli/_apply_opentofu_engine.py +++ b/fluid_build/cli/_apply_opentofu_engine.py @@ -508,12 +508,12 @@ def _resume_blocked_destroys( the functions of a table's column masks and row filter until the table is gone, and refuses to delete a schema that holds them, so a schema the contract stopped using fails to destroy in the very apply that destroyed its tables. Optional plugin capability - ``unblock_destroys(contract, blocked, say) -> bool``: the Databricks plugin drops those - functions once no table uses them, and says what it dropped - (``iac/providers/databricks_vacated.py``). Fails closed: it resumes only when every - resource the apply failed on is a delete of the saved plan and no error is left - unaccounted for, the plugin cleared something, and the plan made after it holds nothing but - some of those deletes, which the data-loss gate already passed. + ``unblock_destroys(contract, blocked, say)``: the addresses of the destroys it cleared (the + Databricks plugin drops those functions once no table uses them, and says what it dropped, + ``iac/providers/databricks_vacated.py``). Fails closed: it resumes only when every + resource the apply failed on is a delete of the saved plan that the plugin cleared, no + error is left unaccounted for, and the plan made after it holds nothing but some of those + deletes, which the data-loss gate already passed. """ unblock = getattr(plugin, "unblock_destroys", None) errored = runner.errored_resources(result) @@ -524,14 +524,15 @@ def _resume_blocked_destroys( return None waited = sorted({address for address, _action in errored}) try: - cleared = unblock(contract, [planned[address] for address in waited], say=cprint_literal) + blocked = [planned[address] for address in waited] + cleared = set(unblock(contract, blocked, say=cprint_literal) or ()) except Exception as exc: # noqa: BLE001 — the apply's own failure is what the run raises cprint_literal( f" what holds {', '.join(waited)} could not be cleared: {type(exc).__name__}" ) return None - if not cleared: - return None + if cleared != set(waited): + return None # a failure the plugin did not clear: the apply's own error stands plan = runner.tofu_plan(workdir, env=env) removals = {address for address, _kind in runner.planned_removals(plan)} only_those = {"add": 0, "change": 0, "remove": len(removals)} diff --git a/fluid_build/iac/base.py b/fluid_build/iac/base.py index c83ddb11..cacdcb92 100644 --- a/fluid_build/iac/base.py +++ b/fluid_build/iac/base.py @@ -77,6 +77,14 @@ class IacProviderPlugin(Protocol): the current state before the plan. * ``catalog_move_spec()``: the ``CatalogMoveSpec`` of an out-of-tree cloud whose emitter moved its catalog resources (``iac/catalog_moves.py``). + * ``prepare_destroys(contract, destroyed, say, dry_run)``: before ``tofu + apply``, clears what the plugin made outside OpenTofu in the resources the + saved plan only deletes (``destroyed``: ``{address, type, before}`` each), + so the cloud does not refuse their destroy. + * ``unblock_destroys(contract, blocked, say)``: after an apply that failed + at destroying ``blocked`` (some of those), clears what the apply's own + destroys freed, and returns the addresses it cleared; the engine plans and + applies those destroys again only when they are every resource that failed. """ #: Short cloud name, e.g. "gcp" / "aws" / "snowflake". diff --git a/fluid_build/iac/providers/databricks.py b/fluid_build/iac/providers/databricks.py index 687c0299..c996330e 100644 --- a/fluid_build/iac/providers/databricks.py +++ b/fluid_build/iac/providers/databricks.py @@ -2565,10 +2565,10 @@ def unblock_destroys( blocked: Iterable[Mapping[str, Any]], say: Optional[Callable[[str], None]] = None, client_factory: Optional[Callable[[], Any]] = None, - ) -> bool: - """After an apply that failed only at destroying ``blocked``: drop what its destroyed - tables used there (``databricks_vacated.unblock``); ``True`` to destroy them again.""" - return _vacated.unblock(_vacated_of(contract, blocked), say or print, client_factory) + ) -> List[str]: + """After an apply that failed at destroying ``blocked``: drop what its destroyed tables + used there (``databricks_vacated.unblock``); the addresses of those it cleared.""" + return list(_vacated.unblock(_vacated_of(contract, blocked), say or print, client_factory)) def post_build( self, @@ -2990,7 +2990,8 @@ def _vacated_of( if compiled.refusals: raise compiled.refusals[0] declared = {f"{table.catalog}.{table.schema}" for table in compiled.tables} - return _vacated.vacated(destroyed, declared, compiled.metric_views.product_id) + # From the contract, not the compiled views: one with no Databricks expose left compiles none. + return _vacated.vacated(destroyed, declared, _metric_views.product_id_of(contract)) def _genie(contract: Mapping[str, Any], compiled: _Compiled, env: Mapping[str, str]) -> Any: diff --git a/fluid_build/iac/providers/databricks_metrics.py b/fluid_build/iac/providers/databricks_metrics.py index 704f479a..0442c029 100644 --- a/fluid_build/iac/providers/databricks_metrics.py +++ b/fluid_build/iac/providers/databricks_metrics.py @@ -940,6 +940,12 @@ def _qualify(text: str, columns: Mapping[str, Any], table: str = "") -> Tuple[st return "".join(out), bare +def product_id_of(contract: Mapping[str, Any]) -> str: + """The id a contract's metric views carry as ``fluid.product_id``: its ``id``, else its + ``name``.""" + return str(contract.get("id") or contract.get("name") or "").strip() + + def compile_metric_views( contract: Mapping[str, Any], tables: Sequence[Any], @@ -953,7 +959,7 @@ def compile_metric_views( ``expose_id``, ``catalog``, ``schema``, ``name``, ``full_name``, ``columns``, ``exposure``, ``enforcement``); ``privileges`` is ``{table full name: {principal: privileges}}``. """ - product_id = str(contract.get("id") or contract.get("name") or "").strip() + product_id = product_id_of(contract) plan = MetricViews( product_id=product_id, schemas=tuple(sorted({f"{t.catalog}.{t.schema}" for t in tables})), @@ -1534,6 +1540,7 @@ def _close(session: Session, view: MetricView, say: Say) -> None: "PRODUCT_PROPERTY", "compile_metric_views", "lock_restricted", + "product_id_of", "post_apply", "pre_apply", "preview", diff --git a/fluid_build/iac/providers/databricks_vacated.py b/fluid_build/iac/providers/databricks_vacated.py index b4e7f1e6..c2b3ad7c 100644 --- a/fluid_build/iac/providers/databricks_vacated.py +++ b/fluid_build/iac/providers/databricks_vacated.py @@ -90,6 +90,8 @@ class Vacated: schemas: Tuple[str, ...] = () #: Lower-case full names of the tables the plan destroys in those schemas. tables: FrozenSet[str] = frozenset() + #: ``(catalog.schema, plan address)`` of each schema, as the plan names both. + addresses: Tuple[Tuple[str, str], ...] = () def _name(before: Any, *keys: str) -> str: @@ -115,11 +117,13 @@ def vacated( entries = [entry for entry in destroyed if isinstance(entry, Mapping)] keep = {str(name).lower() for name in declared} schemas: Dict[str, str] = {} + addresses: Dict[str, str] = {} for entry in entries: if entry.get("type") == "databricks_schema": name = _name(entry.get("before"), "catalog_name", "name") if name and name.lower() not in keep: schemas.setdefault(name.lower(), name) + addresses.setdefault(name, str(entry.get("address") or "")) tables = frozenset( full.lower() for full in ( @@ -129,7 +133,10 @@ def vacated( ) if full and full.lower().rsplit(".", 1)[0] in schemas ) - return Vacated(str(product_id or ""), tuple(schemas[key] for key in sorted(schemas)), tables) + ordered = tuple(schemas[key] for key in sorted(schemas)) + return Vacated( + str(product_id or ""), ordered, tables, tuple((name, addresses[name]) for name in ordered) + ) @dataclass @@ -160,14 +167,28 @@ def _contents( {**query, "include_browse": "true"}, "tables", ) + # Metric views are listed by table-summaries (databricks_metrics._schema_views reads + # them there): every name either listing gives is read. + summaries = _list_named( + session.client, + "/api/2.1/unity-catalog/table-summaries", + {"catalog_name": catalog, "schema_name_pattern": schema, "table_name_pattern": "%"}, + "tables", + ) listed = _list_named(session.client, "/api/2.1/unity-catalog/functions", query, "functions") except Exception as exc: # noqa: BLE001 — a missing schema is an answer; the rest is raised on if _enforce.is_not_found(exc): return None raise + names = {str(row.get("full_name") or f"{where}.{row.get('name')}") for row in rows} + # The pattern is a SQL LIKE, so `_` matches any character: keep this schema's rows only. + names |= { + str(row.get("full_name") or "") + for row in summaries + if str(row.get("full_name") or "").lower().rsplit(".", 1)[0] == where.lower() + } objects: Dict[str, Mapping[str, Any]] = {} - for row in rows: - full = str(row.get("full_name") or f"{where}.{row.get('name')}") + for full in sorted(name for name in names if name): info = read_table(session.client, full) if info is not None: objects[full] = info @@ -181,7 +202,7 @@ def _source(info: Mapping[str, Any]) -> str: """The table a metric view reads (its definition's ``source``), or ``""``.""" try: document = yaml.safe_load(str(info.get("view_definition") or "")) - except yaml.YAMLError: + except Exception: # noqa: BLE001 — YAMLError, or a ValueError for an impossible date return "" return str(document.get("source") or "").strip() if isinstance(document, Mapping) else "" @@ -196,6 +217,11 @@ def _exists(session: Session, full: str, present: Iterable[str]) -> bool: return True +def _named(binding: Any) -> str: + """The ``function_name`` of a row filter or mask the Tables API serves, or ``""``.""" + return str(binding.get("function_name") or "") if isinstance(binding, Mapping) else "" + + def _holders(objects: Mapping[str, Mapping[str, Any]]) -> Dict[str, List[str]]: """``{function (full name, lower case): [objects whose row filter or a mask names it]}``. @@ -203,8 +229,8 @@ def _holders(objects: Mapping[str, Mapping[str, Any]]) -> Dict[str, List[str]]: catalog and schema, so no form of it can make a bound function look unbound.""" holders: Dict[str, List[str]] = {} for full, info in sorted(objects.items()): - bound = [(info.get("row_filter") or {}).get("function_name")] - bound += [(c.get("mask") or {}).get("function_name") for c in _live_columns(info).values()] + bound = [_named(info.get("row_filter"))] + bound += [_named(c.get("mask")) for c in _live_columns(info).values()] for name in filter(None, bound): parts = _bare(name).split(".") home = full.lower().split(".")[: max(0, 3 - len(parts))] @@ -225,17 +251,21 @@ def __init__(self, session: Session, vac: Vacated, where: str, say: Say, dry_run def run(self) -> _Outcome: try: - read = _contents(self.session, self.where) - self.me = self.session.identity if read is not None else "" - except Exception as exc: # noqa: BLE001 — unknown contents: drop nothing + self._clear() + except Exception as exc: # noqa: BLE001 — what cannot be read is left as it is + more = " more" if self.out.dropped else "" self.say( f" not checked: {self.where} could not be read ({type(exc).__name__}: " - f"{scrub(exc)}), so nothing in it was dropped" + f"{scrub(exc)}), so nothing{more} in it was dropped" ) self.out.unread = True - return self.out + return self.out + + def _clear(self) -> None: + read = _contents(self.session, self.where) if read is None: - return self.out # the schema is gone already + return # the schema is gone already + self.me = self.session.identity objects, functions = read self.present = frozenset(full.lower() for full in objects) for full, info in sorted(objects.items()): @@ -249,15 +279,18 @@ def run(self) -> _Outcome: for name in functions: full = f"{self.where}.{name}" self._function(full, holders.get(_bare(full), [])) - return self.out def _view(self, full: str, info: Mapping[str, Any]) -> None: - found = (info.get("properties") or {}).get(_metrics.PRODUCT_PROPERTY) - source = _source(info) + properties = info.get("properties") + found = ( + properties.get(_metrics.PRODUCT_PROPERTY) if isinstance(properties, Mapping) else None + ) if not self.vac.product_id or found != self.vac.product_id: whose = f"of product {found!r}" if found else "this module did not create" self.out.left.append(f"{full} (a metric view {whose})") - elif not source: + return + source = _source(info) + if not source: self.out.left.append(f"{full} (a metric view whose definition names no table)") elif source.lower() in self.vac.tables: self._drop_owned(full, info, "VIEW", f"it reads {source}, which this apply destroys") @@ -364,7 +397,7 @@ def prepare( if not lines: continue if not shown: - say("\n Schemas this apply destroys: what this product made there outside OpenTofu") + say("\n Schemas this apply destroys, and what is in them outside OpenTofu") shown = True say(f" {where}") for line in lines: @@ -373,24 +406,27 @@ def prepare( say(" dry-run: none of these statements ran.") -def unblock(vac: Vacated, say: Say, client_factory: Optional[ClientFactory] = None) -> bool: - """After a ``tofu apply`` that failed only at destroying ``vac.schemas``: drop what this - product made there that the apply's own destroys freed (the functions of the tables it - destroyed), by :func:`prepare`'s rules. ``True`` when something was dropped, no drop failed - and nothing it can see is left, so destroying the schemas again can succeed.""" +def unblock( + vac: Vacated, say: Say, client_factory: Optional[ClientFactory] = None +) -> Tuple[str, ...]: + """After a ``tofu apply`` that failed at destroying ``vac.schemas``: drop what this product + made there that the apply's own destroys freed (the functions of the tables it destroyed), + by :func:`prepare`'s rules. The plan addresses of the schemas it cleared: something was + dropped there, no drop failed and nothing it can see is left, so destroying them again can + succeed.""" if not vac.schemas: - return False + return () session = _session(vac, say, client_factory) if session is None: - return False - dropped, clear = 0, True - for where in vac.schemas: + return () + cleared: List[str] = [] + for where, address in vac.addresses: say(f"\n {where}: tofu apply could not destroy it yet: it held what this product made") out = _Clearing(session, Vacated(vac.product_id, (where,)), where, say, False).run() _say_left(where, out, say) - dropped += out.dropped - clear = clear and not (out.failed or out.left or out.unread or out.waiting) - return dropped > 0 and clear + if out.dropped and not (out.failed or out.left or out.unread or out.waiting): + cleared.append(address) + return tuple(cleared) __all__ = ["Vacated", "prepare", "unblock", "vacated"] diff --git a/tests/iac/test_iac_apply_engine_destroys.py b/tests/iac/test_iac_apply_engine_destroys.py index dcb1422d..a86ac7cc 100644 --- a/tests/iac/test_iac_apply_engine_destroys.py +++ b/tests/iac/test_iac_apply_engine_destroys.py @@ -65,7 +65,7 @@ def _plan(*removed: str, add: int = 0, change: int = 0) -> Any: class Plugin: - def __init__(self, cleared: Any = True): + def __init__(self, cleared: Any = (SCHEMA,)): self.cleared = cleared self.calls: List[List[Dict[str, Any]]] = [] self.prepared: List[Dict[str, Any]] = [] @@ -206,7 +206,7 @@ def test_any_other_failure_is_left_as_it_is(tofu, events): assert plugin.calls == [] and tofu["plan"] == 0 and tofu["apply"] == 0 -@pytest.mark.parametrize("cleared", [False, RuntimeError("boom")], ids=["nothing", "raised"]) +@pytest.mark.parametrize("cleared", [(), RuntimeError("boom")], ids=["nothing", "raised"]) def test_nothing_cleared_is_not_resumed(tofu, cleared): assert _resume(Plugin(cleared), _errored(SCHEMA), DIAGNOSTIC) is None assert tofu["plan"] == 0 and tofu["apply"] == 0 @@ -272,3 +272,11 @@ def summary(**changes: int) -> Any: "change": 1, "remove": 0, } + + +def test_a_failed_destroy_the_plugin_did_not_clear_is_not_resumed(tofu): + table = DESTROYED[1]["address"] + plugin = Plugin(cleared=(SCHEMA,)) + assert _resume(plugin, _errored(SCHEMA), _errored(table), DIAGNOSTIC, DIAGNOSTIC) is None + assert plugin.calls == [DESTROYED[:2]] # the waited destroys, by address + assert tofu["plan"] == 0 and tofu["apply"] == 0 diff --git a/tests/iac/test_iac_databricks_vacated.py b/tests/iac/test_iac_databricks_vacated.py index a70ddf9e..c8eb6c78 100644 --- a/tests/iac/test_iac_databricks_vacated.py +++ b/tests/iac/test_iac_databricks_vacated.py @@ -173,7 +173,7 @@ def test_the_view_goes_before_the_apply_and_the_functions_once_their_table_is_go run.destroy_table(OLD) assert "Schema is not empty" in run.delete_schema("main.sales") before = len(fake.sql_log) - assert run.unblock() is True + assert run.unblock() == [SCHEMA_DELETE["address"]] assert sorted(run.drops(before)) == sorted( "DROP FUNCTION IF EXISTS " + ".".join(f"`{p}`" for p in name.split(".")) for name in functions @@ -194,7 +194,7 @@ def test_a_dry_run_prints_the_drops_and_runs_none(fake): " main.sales", f" before apply: drop metric view {VIEW}: it reads {OLD}, which this apply destroys", ] - assert "Schemas this apply destroys: what this product made there outside OpenTofu" in run.text + assert "Schemas this apply destroys, and what is in them outside OpenTofu" in run.text assert " DROP VIEW IF EXISTS `main`.`sales`.`orders_semantic_model`" in run.text assert f" after tofu destroys {OLD}: drop function main.sales.fluid_rf_orders_" in run.text assert " DROP FUNCTION IF EXISTS `main`.`sales`.`fluid_mask_orders_region_" in run.text @@ -251,7 +251,7 @@ def test_only_this_products_and_this_identitys_objects_go_and_never_a_bound_func run.destroy_table(OLD) before = len(fake.sql_log) - assert run.unblock() is False # the schema still holds what is not this run's to drop + assert run.unblock() == [] # the schema still holds what is not this run's to drop assert len(ours) == 2 and len(run.drops(before)) == 2 and not ours & set(fake.functions) assert {kept_filter, planted, "main.sales.normalize"} <= set(fake.functions) assert fake.tables["main.sales.orders_stg"]["row_filter"]["function_name"] == kept_filter @@ -294,7 +294,7 @@ def test_another_environment_of_the_product_keeps_its_view_and_functions(fake): run.prepare() run.destroy_table(OLD) - assert run.unblock() is True + assert run.unblock() == [SCHEMA_DELETE["address"]] assert run.held("main.sales") == [] assert run.held("main.sales_stg") == staging @@ -308,7 +308,7 @@ def test_a_schema_that_cannot_be_read_drops_nothing(fake): assert run.drops(before) == [] and VIEW in fake.tables assert "not checked: main.sales could not be read" in run.text run.destroy_table(OLD) - assert run.unblock() is False + assert run.unblock() == [] def test_a_drop_that_fails_is_named_and_the_destroy_is_not_asked_for_again(fake): @@ -317,7 +317,7 @@ def test_a_drop_that_fails_is_named_and_the_destroy_is_not_asked_for_again(fake) run.prepare() run.destroy_table(OLD) fake.fail_on("DROP FUNCTION", "[INTERNAL_ERROR] injected failure") - assert run.unblock() is False + assert run.unblock() == [] assert "FAILED: drop function main.sales.fluid_" in run.text @@ -329,7 +329,7 @@ def broken() -> Any: run.prepare(client_factory=broken) assert "main.sales: what this product made outside OpenTofu there" in run.text assert "was not checked: no Databricks client" in run.text - assert run.unblock(client_factory=broken) is False + assert run.unblock(client_factory=broken) == [] def test_vacated_takes_the_schemas_the_plan_destroys_that_the_contract_does_not_name(): @@ -355,3 +355,64 @@ def test_vacated_takes_the_schemas_the_plan_destroys_that_the_contract_does_not_ # Without catalog_name and name, the id names the schema. only_id = [{"type": "databricks_schema", "before": {"id": "main.legacy"}}] assert vacated_mod.vacated(only_id, (), "p").schemas == ("main.legacy",) + + +def test_a_definition_or_a_row_of_an_odd_shape_is_left_never_raised(fake, monkeypatch): + run = Run(fake) + run.deploy(_contract()) + # Unity Catalog keeps what YAML it was given: an impossible date is a ValueError to PyYAML. + _metric_view(fake, "main.sales.theirs_semantic_model", "other.product") + odd = "version: 1.1\nsource: main.sales.orders\nwhen: 2024-02-30\n" + fake.tables["main.sales.theirs_semantic_model"]["view_definition"] = odd + fake.tables[VIEW]["view_definition"] = odd + run.prepare() + assert VIEW in fake.tables and "main.sales.theirs_semantic_model" in fake.tables + assert f"{VIEW} (a metric view whose definition names no table)" in run.text + assert "main.sales.theirs_semantic_model (a metric view of product 'other.product')" in run.text + + # Anything else that breaks while one schema is read leaves it as it is, with a note. + def broken(objects): + raise RuntimeError("an unexpected row") + + monkeypatch.setattr(vacated_mod, "_holders", broken) + run.said.clear() + run.prepare() + assert "not checked: main.sales could not be read (RuntimeError: an unexpected row)" in run.text + run.destroy_table(OLD) + assert run.unblock() == [] + + +def test_a_contract_with_no_databricks_expose_left_still_names_its_views(fake): + run = Run(fake) + run.deploy(_contract()) + bare = _contract("other") + bare["exposes"] = [] + PLUGIN.prepare_destroys(bare, DESTROYED, say=run.said.append, client_factory=fake.client) + assert VIEW not in fake.tables + + +class _TablesListWithoutViews: + """The stand-in's client, its Tables list leaving metric views out (they stay listed by + table-summaries).""" + + def __init__(self, inner: Any): + self._inner = inner + self.api_client = self + + def do(self, method: str, path: str, **kw: Any) -> Any: + out = self._inner.api_client.do(method, path, **kw) + if method == "GET" and path == "/api/2.1/unity-catalog/tables": + out = { + **out, + "tables": [ + r for r in out.get("tables") or [] if r.get("table_type") != "METRIC_VIEW" + ], + } + return out + + +def test_a_metric_view_only_table_summaries_lists_is_still_found(fake): + run = Run(fake) + run.deploy(_contract()) + run.prepare(client_factory=lambda: _TablesListWithoutViews(fake.client())) + assert VIEW not in fake.tables From 3c4518c48db4d1b2f6c86841a7c2e72bfdaf3abd Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:49:31 +0100 Subject: [PATCH 5/5] fix(databricks): a table-summaries listing that is not found falls back to the Tables list From the re-check of the previous commit: a "not found" from table-summaries alone made a schema that exists look gone, so it was skipped with no note. Only the Tables list or the functions list answering "not found" now means the schema is gone; table-summaries answering it leaves the Tables list to name what the schema holds. Also reflows the stand-in's docstring line the rebase merged past 100 columns. --- HONESTLY_TESTED.md | 2 +- fluid_build/iac/providers/databricks_vacated.py | 15 ++++++++++----- tests/iac/_fake_databricks.py | 3 ++- tests/iac/test_iac_databricks_vacated.py | 9 +++++++++ 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/HONESTLY_TESTED.md b/HONESTLY_TESTED.md index d829a4b2..545f6e7d 100644 --- a/HONESTLY_TESTED.md +++ b/HONESTLY_TESTED.md @@ -93,7 +93,7 @@ Markers declared in `pyproject.toml`. | Stage 1 unit: Databricks retention (`lifecycle.expire`) | `tests/iac/test_iac_databricks_retention.py`, `tests/cli/test_verify_databricks_retention.py` | ✅ 195 + 18. ISO 8601 durations (years and months at their shortest, P1Y = 365 and P1M = 28 days; a fraction, a sign, a lower-case letter, `PT`, `P1DT`, out-of-order and over-long numbers refused), Delta `interval` properties (rounded up, months refused) and the history as the larger of the two properties with Delta's defaults. A dbt model's `materialized` resolved from `config()` in the SQL, the model's YAML `config:` and `dbt_project.yml` (the `+` prefix or none, root, project, folder and model-name levels, the most specific wins, SQL over YAML over project), the default `view`, the model found by alias or file name, commented-out and `raw` configs ignored, a `config()` inside a Jinja block, `var()`, `**`, a custom schema or database, a Python model, a versioned or twin model, a linked file and a Git repository refused as undeterminable. The build's `--select` must include the resolved model (name, `fqn`, `path:`, `*`, unions and intersections; a graph operator on another node, a tag, a config or run state is refused as not evaluable). Accepted: a dbt `table` model and a DuckDB acquisition with P30D, P365D, P6Y; refused: P7D, P29D, P4W, P1M, a history set longer than the retention, `incremental`, `snapshot`, `view`, another acquisition engine, no writer, a mixed set of writers. `fluid validate` and the emitter agree on kind and message. `fluid verify`'s retention dimension against a stubbed client and, over HTTP, the stand-in workspace (pass, fail with the `ALTER TABLE`, the runtime property over the declared one, skip, predictive optimization off as a warning). | Live behaviour: no real Databricks endpoint was called. Whether Unity Catalog's GetTable serves a table's `delta.logRetentionDuration` and `delta.deletedFileRetentionDuration` in `properties` or `delta_runtime_properties_kvpairs` (read from databricks-sdk-go v0.190.0 `model.go`, not observed), whether `effective_predictive_optimization_flag` is filled for a managed table, that predictive optimization runs `VACUUM` within the retention (a Databricks guarantee, not tested), and that dbt-databricks `table` replaces the table each run. Delta history is only half the promise: the acquisition acceptance rests on the landing being an `INSERT OVERWRITE` of the whole table in the one mode it runs (`full_refresh`, read from the landing's own `LOAD_MODES`; another source mode is refused as `incremental`, none as `unmapped`), which the bronze landing row covers against the stand-in, not live. A GetTable answer with no properties is a verify warning (Delta's defaults assumed). Embedded-logic DuckDB builds, `dlt`, `airbyte` and the other acquisition engines are refused. Selection is matched on names and paths only, not on dbt's graph: `+other+` is refused even if `other` is upstream of the model. | | Stage 1: Databricks Unity Catalog metadata (comments, product tags, column tags; live: the 2026-11-04 row) | `tests/iac/test_iac_databricks_metadata.py` (the emitter), `tests/iac/test_iac_databricks_metadata_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against `tests/iac/_fake_databricks.py`), `tests/cli/test_verify_databricks.py` (the comment and column-tag cases) | ✅ 43 + 5 + 7. The table's `comment` is the expose's description, else the product's (its `purpose` when structured), each described column's `comment` its schema description, stripped; none on a table a dbt build writes (dbt's CREATE OR REPLACE owns them; `fluid validate` warns). The product's identity and owner team are tags on the table only (`fluid_product_id`, `fluid_product_name`, `fluid_domain`, `fluid_layer`, `fluid_product_type`, `fluid_product_version`, `fluid_version`, `fluid_owner_team`), an unstated field makes none, and they count with the governance labels toward 50; `lifecycle.state: deprecated` (the expose's, else the product's) sets `system.certification_status = deprecated`, else the label `certification: certified` (the expose's, else the product's) sets `certified` and is no plain tag (a product-level label certifies every expose's table, as intended; an expose-level one only its own); another label value is `databricks-certification-invalid`; a plan notice says the identity needs ASSIGN on that governed tag. Against the stand-in: certified, then deprecated in place, then removed. A classified column (`sensitivity`) carries `fluid_sensitivity` as a `columns` tag named `catalog.schema.table.column`, imported best-effort by the provider's id. Refused through `validate` and `emit` alike: a backslash in any comment (`databricks-comment-unsafe`; the provider's `parseComment` escapes only quotes) and a `${` (Spark variable substitution), a label meeting a product key (`databricks-tag-collision`), a value over 256 characters (`databricks-tag-value-too-long`), over 1,000 column tags (`databricks-tags-too-many`); a `${...}` in a tag value is rendered as a literal. Against the stand-in: apply writes the comments (an apostrophe included, through the provider's escaping), the product tags and the column tag, never an `OWNER TO`; a second plan is empty; a changed description is an in-place `COMMENT ON TABLE`, never a replace; removing a classification destroys only its column tag; a table made before this is adopted and gains its comments in place (`+5 ~1` for the product tags and the table update). `fluid verify` passes after the apply and fails a comment or column tag changed by hand; against a stubbed client it reads each column's tag list to its last page, ignores a tag or comment someone else added, and checks no comment on a dbt-written table. | **Not run live.** What real Unity Catalog stores and returns for a table or column comment (whitespace, newlines, length limits, which the docs do not state), whether it accepts a tag value with `, . : / = -` (the docs restrict only keys), whether `entity-tag-assignments/columns/{catalog.schema.table.column}` resolves as the SDK documents, and whether a dbt CREATE OR REPLACE keeps column tags (undocumented; `fluid verify` is the gate), and whether the Entity Tag Assignments API sets the system tag `system.certification_status` (its docs show the UI and `SET TAG`; the SDK documents governed tags through the API, needing ASSIGN; the stand-in checks no permission). A contract that adds a column while comments change for the first time hits the provider's mixed-change refusal (its fix-it says to apply in two steps). | | Stage 1: Databricks metric views (`exposes[].semantics`) | `tests/iac/test_iac_databricks_metrics.py` (the YAML, the refusals and the hooks against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_databricks_metrics_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against the stand-in) | ✅ 91 + 5. A site-360-shaped semantics block compiles to the metric view YAML it should (the entity as its key column and categorical and time dimensions as dimensions, with comments, display names and synonyms from the columns; non-additive measures as `semiadditive: last` windows; filtered simple metrics as `FILTER (WHERE ...)`; a ratio as `MEASURE(a) / NULLIF(MEASURE(b), 0)`, after its inputs); the view is `
_semantic_model` whatever the model is called; every column reference is `source.` (`
.` too), a unit or type keyword (`EXTRACT(YEAR ...)`, `INTERVAL 1 DAY`, `::date`) stays one; the time grains (`TRUNC` on a `DATE`, `DATE_TRUNC` otherwise); 42 refusal cases (names, collisions, expressions, quoting, calls that would lend the owner's rights, text limits). The hooks: a first apply creates the view with its `fluid.*` properties and grants it to the table's readers; a second runs nothing; a changed or hand-edited definition is an `ALTER VIEW` that keeps the grants; `SELECT` and `ALL PRIVILEGES` are revoked from anyone else, before the apply and first in it, so a failed grant or update leaves no removed reader; a view the contract no longer declares, in a schema the product's tables are in, is dropped before the apply, another product's is left and another environment's (same id, another schema of the catalog) is kept; a table, a view without this product's property or one another identity owns at the name is refused before the apply; a failed statement is `databricks-metric-view-failed` naming the view; without databricks-sdk a declared view is refused; when a restricted table's binding fails, `SELECT` on its views is revoked. Through the CLI: apply, a second apply `+0 ~0 -0` with no statement, dry-run prints and runs nothing, dropping the semantics drops the view, `fluid validate` refuses a derived metric, and the product's Genie space lists the view the same apply made. All 19 industry-model tower-management contracts and the 4 forge templates that bind a table compile with no refusal. | What Databricks accepts and answers (the 2026-11-12 live row); Genie and AI/BI dashboards themselves. A schema the contract stops using: the next row. | -| Stage 1: Databricks, a schema the contract stops using | `tests/iac/test_iac_databricks_vacated.py` (the rules, against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_apply_engine_destroys.py` (the engine, `tofu` stubbed), 2 of `tests/iac/test_iac_databricks_metrics_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against the stand-in) | ✅ 13 + 24 + 2. A table with a metric view, a mask and a row filter moves from `main.sales` to `main.other`: `--mode dry-run` prints the `DROP VIEW` before the apply and the `DROP FUNCTION`s after the table's destroy, and runs none; the apply drops the view before `tofu apply`, `tofu apply` destroys the old table and is refused the schema (the stand-in, like Unity Catalog, deletes no schema that holds a table, view, function or volume), the functions are dropped once no table uses them, and a second, destroy-only plan (`+0 ~0 -1`) destroys the schema; the run exits 0 with the first plan's totals, and the next apply is `+0 ~0 -0`. Removing a schema's last expose does the same. Only schemas the saved plan destroys and the contract does not name are looked in, so a schema the contract still uses and another environment's (same product id, same identity, another schema) are never read. In the destroyed schema, a view of another product, one another identity owns and one over a table the plan keeps are left, and so are a function a kept table uses (named in full or not), one another identity owns and one of another name; an unused function of this module's name shape that the deploy identity owns is dropped; each object left is named with why. A schema that cannot be read, a view definition YAML cannot load (an impossible date) and a row of an unexpected shape drop nothing and raise nothing; a metric view only the table-summaries listing gives is found; a contract with no Databricks expose left still names its views by its id. A failed drop, a plugin that clears nothing or raises, a failed delete the plugin did not clear, a failure the plan's deletes do not account for, and a second plan holding any other change all leave the apply's own failure. | Live Databricks: that the provider destroys the table before its schema and Unity Catalog then deletes the schema once its functions are dropped (the 2026-11-03 run measured only that a schema holding functions is not deleted); whether a function re-created under a dropped one's name would be bound again (why a function a table still uses is never dropped); whether Unity Catalog's Tables list includes metric views (both it and table-summaries are read, so either suffices); a table in another schema whose mask names a function here; a volume or a model in the schema (neither dropped nor named). | +| Stage 1: Databricks, a schema the contract stops using | `tests/iac/test_iac_databricks_vacated.py` (the rules, against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_apply_engine_destroys.py` (the engine, `tofu` stubbed), 2 of `tests/iac/test_iac_databricks_metrics_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against the stand-in) | ✅ 14 + 24 + 2. A table with a metric view, a mask and a row filter moves from `main.sales` to `main.other`: `--mode dry-run` prints the `DROP VIEW` before the apply and the `DROP FUNCTION`s after the table's destroy, and runs none; the apply drops the view before `tofu apply`, `tofu apply` destroys the old table and is refused the schema (the stand-in, like Unity Catalog, deletes no schema that holds a table, view, function or volume), the functions are dropped once no table uses them, and a second, destroy-only plan (`+0 ~0 -1`) destroys the schema; the run exits 0 with the first plan's totals, and the next apply is `+0 ~0 -0`. Removing a schema's last expose does the same. Only schemas the saved plan destroys and the contract does not name are looked in, so a schema the contract still uses and another environment's (same product id, same identity, another schema) are never read. In the destroyed schema, a view of another product, one another identity owns and one over a table the plan keeps are left, and so are a function a kept table uses (named in full or not), one another identity owns and one of another name; an unused function of this module's name shape that the deploy identity owns is dropped; each object left is named with why. A schema that cannot be read, a view definition YAML cannot load (an impossible date) and a row of an unexpected shape drop nothing and raise nothing; a metric view only the table-summaries listing gives is found, and a table-summaries listing that answers not found falls back to the Tables list; a contract with no Databricks expose left still names its views by its id. A failed drop, a plugin that clears nothing or raises, a failed delete the plugin did not clear, a failure the plan's deletes do not account for, and a second plan holding any other change all leave the apply's own failure. | Live Databricks: that the provider destroys the table before its schema and Unity Catalog then deletes the schema once its functions are dropped (the 2026-11-03 run measured only that a schema holding functions is not deleted); whether a function re-created under a dropped one's name would be bound again (why a function a table still uses is never dropped); whether Unity Catalog's Tables list includes metric views (both it and table-summaries are read, so either suffices); a table in another schema whose mask names a function here; a volume or a model in the schema (neither dropped nor named). | | Manual live, Free Edition (founder-approved), 2026-11-12, metric views | the real `fluid` CLI, no test file: this branch, databricks-sdk 0.166.0, a scratch catalog `fluid_iactest_metrics` dropped `CASCADE` after | ✅ The semantics block of industry-model `dp019_site_360` verbatim (13 dimensions, 3 non-additive measures, 4 filtered metrics) on a table shaped like it: dry-run printed the `CREATE VIEW` and grants; apply `+77`, then the view created and `SELECT` granted to the 3 reader principals; `SELECT country_code, MEASURE(site_count), MEASURE(avg_tenancy_ratio), ... GROUP BY ALL` returned each market at its latest snapshot (8 rows seeded, 4 sites x 2 days); by date, each day's own values; a second apply `+0 ~0 -0` ran no statement. `information_schema.tables` lists it as `METRIC_VIEW` with its comment, `.columns` every dimension and measure with its comment, `.table_privileges` the 3 `SELECT` grants. Probes on the same warehouse (Databricks SQL 2026.45): `CREATE OR REPLACE VIEW` drops a view's grants and makes whoever replaces it its owner, and `ALTER VIEW ... AS` keeps its grants and its properties; the stored `view_definition` leaves out `comment`, `display_name` and `synonyms`; a dimension named like a column shadows it in a measure's filter and a window measure named like a column breaks the window measures that read it (`METRIC_VIEW_WINDOW_MEASURE_REFERENCES_WINDOW_MEASURE`), and `source.` reads the column; `semiadditive: last` is per output group; `FILTER (WHERE ...)` works on a window measure, `COUNT(DISTINCT ...)` and `PERCENTILE_CONT ... WITHIN GROUP`; `TBLPROPERTIES` combine with `WITH METRICS`; a metric view over a row-filtered table applies the filter to the reader; a table at the view's name fails `EXPECT_VIEW_NOT_TABLE`. | A query as a reader persona (the run had the deploy user's token only; the persona's effective `SELECT` was read from the permissions API); Genie and dashboards; a contract with column masks. | | Stage 1: Databricks data quality monitoring (live: the 2026-11-12 row) | `tests/iac/test_iac_databricks_quality.py` (what asks for it, the monitor per schema, the derived import, the notice, the engine), `tests/iac/test_iac_databricks_quality_fake_e2e.py` (real `fluid` + `tofu` + databricks/databricks 1.140.0 against `tests/iac/_fake_databricks.py`), `tests/iac/test_iac_tofu_validate.py::test_a_databricks_data_quality_monitor_passes_tofu_validate` | ✅ 37 + 6 + 1. A freshness or completeness SLO (`qos.freshnessSLO`, `qos.completenessTarget`, an `observability.defaultSLIs` freshness or completeness block, a `contract.dq.rules` entry of type `freshness`, `completeness` or `anomaly_detection`) puts one `databricks_data_quality_monitor` on the owned schema, watching it by `schema_id`, created after the schema's tables, the product's other tables there in `excluded_table_full_names`; `contract.dq.monitoring.enabled: false` removes only the monitor; a `dq.monitoring.window` other than one day is a `fluid validate` warning. Against the stand-in: a second plan is empty; a monitor turned on outside forge-cli is adopted by `schema,` taken from state after the schema's import (no create); the last exclusion leaving is a PATCH whose `update_mask` names the whole config (the stand-in honours the mask), and the plan converges; a monitor Databricks refuses fails the apply after the tables, its suggestions saying what it needs and naming the opt-out (`apply_failure_hints`, after the three generic ones); opting out deletes it, not gated; `tofu destroy` deletes it before the schema. | What Databricks scans and reports (the live row); the stand-in serves schema monitors only. | | Manual live, Free Edition (founder-approved) | the real `fluid` CLI, no test file: run 2026-11-01 from main dfd6f145, databricks-sdk 0.158.0, every `fluid_iactest_*` object swept after (1 schema, 1 table, 1 folder; 0 left) | ✅ validate, `generate iac --validate`, plan, dry-run and apply `+9`; a second plan is EMPTY against the real Jobs API; Databricks accepts the Quartz schedule, serverless environment 2 with `dbt-databricks>=1.8.0,<2.0.0` (dbt 1.12.6, adapter 1.13.0), the WORKSPACE `project_directory` and the commands as emitted; one `run-now` ends SUCCESS and dbt builds the table; `fluid verify` with the real SDK (columns, types, one job); `fluid diff`'s state check (its live check has no Databricks inspector); a model change `~1`; the build made local removes job, files and folders `-5` without `--allow-data-loss` (the empty `.fluid//` folders above `files/` stay: only `files/` and below are managed). It found the USE CATALOG check reading a group-conveyed privilege as none. After the fix, `fluid verify` on a fresh one-table product (`+4`) passes every dimension, catalog access included (1 schema and 1 table swept, 0 left). | A GIT source, profile or OAuth auth, a run-as service principal, table adoption and a larger project, live; the region residency check (the metastore precondition and `fluid verify`'s residency dimension), which was added after this run and not run live; dbt's CREATE OR REPLACE drops a contract's NOT NULL, so verify flags a `required` column a dbt model does not constrain itself. | diff --git a/fluid_build/iac/providers/databricks_vacated.py b/fluid_build/iac/providers/databricks_vacated.py index c2b3ad7c..363be0aa 100644 --- a/fluid_build/iac/providers/databricks_vacated.py +++ b/fluid_build/iac/providers/databricks_vacated.py @@ -167,6 +167,12 @@ def _contents( {**query, "include_browse": "true"}, "tables", ) + listed = _list_named(session.client, "/api/2.1/unity-catalog/functions", query, "functions") + except Exception as exc: # noqa: BLE001 — a missing schema is an answer; the rest is raised on + if _enforce.is_not_found(exc): + return None + raise + try: # Metric views are listed by table-summaries (databricks_metrics._schema_views reads # them there): every name either listing gives is read. summaries = _list_named( @@ -175,11 +181,10 @@ def _contents( {"catalog_name": catalog, "schema_name_pattern": schema, "table_name_pattern": "%"}, "tables", ) - listed = _list_named(session.client, "/api/2.1/unity-catalog/functions", query, "functions") - except Exception as exc: # noqa: BLE001 — a missing schema is an answer; the rest is raised on - if _enforce.is_not_found(exc): - return None - raise + except Exception as exc: # noqa: BLE001 — an answer only when it is "not found" + if not _enforce.is_not_found(exc): + raise + summaries = [] # the Tables list alone, which found the schema names = {str(row.get("full_name") or f"{where}.{row.get('name')}") for row in rows} # The pattern is a SQL LIKE, so `_` matches any character: keep this schema's rows only. names |= { diff --git a/tests/iac/_fake_databricks.py b/tests/iac/_fake_databricks.py index 714a4391..a2136aac 100644 --- a/tests/iac/_fake_databricks.py +++ b/tests/iac/_fake_databricks.py @@ -19,7 +19,8 @@ The bindings are served on the table (``row_filter``, ``columns[].mask``) as Unity Catalog serves them, and a CREATE OR REPLACE TABLE keeps them (Databricks docs: REPLACE preserves row filters and column masks). A schema that still holds a table, a view, a function or a volume is not -deleted (``Schema is not empty``), as Unity Catalog refuses it. The key constraints forge-cli declares (``databricks_keys.py``): ALTER TABLE +deleted (``Schema is not empty``), as Unity Catalog refuses it. The key constraints forge-cli +declares (``databricks_keys.py``): ALTER TABLE ADD CONSTRAINT ... PRIMARY KEY / FOREIGN KEY ... REFERENCES and DROP CONSTRAINT [IF EXISTS], served as ``table_constraints``, with the rules and error texts a warehouse showed (a key needs NOT NULL columns, a foreign key its parent's primary key, a referenced primary key does not diff --git a/tests/iac/test_iac_databricks_vacated.py b/tests/iac/test_iac_databricks_vacated.py index c8eb6c78..19e9f328 100644 --- a/tests/iac/test_iac_databricks_vacated.py +++ b/tests/iac/test_iac_databricks_vacated.py @@ -416,3 +416,12 @@ def test_a_metric_view_only_table_summaries_lists_is_still_found(fake): run.deploy(_contract()) run.prepare(client_factory=lambda: _TablesListWithoutViews(fake.client())) assert VIEW not in fake.tables + + +def test_a_table_summaries_listing_that_is_not_found_falls_back_to_the_tables_list(fake): + run = Run(fake) + run.deploy(_contract()) + fake.fail_route("GET", r"^/api/2\.1/unity-catalog/table-summaries$", 404, "not here") + run.prepare() + assert VIEW not in fake.tables + assert "not checked" not in run.text