From 6ec3010c74014185666c9a04de4946bb69597bd2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:56:30 +0100 Subject: [PATCH 01/26] chore(ci): bump pypa/gh-action-pypi-publish from 1.14.2 to 1.14.3 (#783) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.14.2 to 1.14.3. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/dc37677b2e1c63e2034f94d8a5b11f265b73ba33...7245906fb61fa33a13554f9d5fb856fe61758086) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-version: 1.14.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 897f71b34..d31e54caf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -347,7 +347,7 @@ jobs: path: dist/ - name: Publish to TestPyPI (with Sigstore attestations) - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 with: repository-url: https://test.pypi.org/legacy/ attestations: true @@ -459,7 +459,7 @@ jobs: path: dist/ - name: Publish to PyPI (with Sigstore attestations) - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 with: attestations: true From 9d873e286214f461e281d8f6b06227cbb4ff62ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:56:48 +0100 Subject: [PATCH 02/26] chore(ci): bump docker/login-action from 4.6.0 to 4.7.0 (#781) Bumps [docker/login-action](https://github.com/docker/login-action) from 4.6.0 to 4.7.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/dbcb813823bdd20940b903addbd779551569679f...81c6f93d4bc991e510b342a00b419b3bb4efc5c6) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.7.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d31e54caf..74f8734e2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -663,7 +663,7 @@ jobs: uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Log in to GHCR - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} From 6424a3de9d855a06dcc40032c41240db02b7820d Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:21:43 +0100 Subject: [PATCH 03/26] feat(databricks): column masks and row filters are enforced in Unity Catalog (#785) * feat(databricks): column restrictions and row filters compile to Unity Catalog SQL functions policy.authz.columnRestrictions and rowFilters were refused. They now compile to SQL functions named by a hash of their definition, with the semantics GCP and AWS give the same fields: deny is a mask that returns NULL, mask is last_four, first_four or NULL, one mask per column with a deny beating a mask, and one row filter per table that is the union of its principals' predicates for their members and every row for everyone else. A predicate is parsed as one boolean expression, checked node by node against an allowlist and rendered from the tree. access: allow, a principal SQL cannot test and a text mask on a non-string column are refused. Nothing runs them yet. * feat(databricks): fluid apply binds the masks and row filter around tofu apply, and revokes SELECT when it cannot The provider has no function resource and no mask or filter attribute, so the statements run on the SQL warehouse through the SQL Statement Execution API. pre_apply binds on the tables that exist before tofu apply runs, so a grant the apply adds lands on a protected table; post_apply binds on the tables the apply created, drops a mask or filter this module bound that the contract no longer asks for, and, for a table it cannot bind, revokes SELECT from the principals the contract restricts and exits non-zero. Live bindings are read from the Tables API and only what differs runs, so a second apply executes nothing. A dry run prints the statements and runs none. The stand-in executes the statements and serves the bindings. * feat(verify): fluid verify checks a Databricks table's row filter and column masks The contract's restrictions are Unity Catalog functions bound on the table, named by a hash of their definition. Verify compares the table's live row_filter and each column's mask, from the GetTable it already reads, with the names the contract expects: a binding that is missing, or is another function (a changed predicate is a new name), is CRITICAL and the fix is fluid apply. No restriction in the contract skips the dimension. * test(databricks): a real tofu apply binds the masks and row filter against the stand-in, and fails safe Apply binds the filter and masks and a second apply executes no statement; a table that exists is bound before the apply can grant on it; a changed predicate binds a new function and keeps the old; a removed restriction is dropped and said to widen access; a binding that fails revokes SELECT from the restricted principals only, fails the run, and the next apply binds first and grants again; a failure before the apply stops it with nothing applied. * docs(databricks): the changelog and tested-honestly rows say what is enforced, how, and what is not proven live * fix(databricks): a contract with no restriction needs no databricks-sdk at apply and says nothing without it * fix(databricks): a mask or filter tests every identity as a group and as a user, so a remapped principal is still restricted The test was chosen from the logical principal's prefix. binding.principals may map a group: principal to a service principal's application ID or a user's email, and the group test then matched no one while the grant to that identity still applied: the mask and filter never took effect. Every identity is tested as is_account_group_member(x) OR current_user() = x; a name that is both a group and a user is restricted either way, which errs closed. The prefix only refuses domain: and unprefixed principals now. * fix(databricks): a mask rule belongs to the principal that names it, and the strictest matching one applies One rules[column] kept the last restriction's rule for every masked member, so a nullify for one principal and a last_four for another gave the first the last four characters, a weaker mask than declared. Members are grouped by rule and the function has one WHEN branch per rule, strictest first: deny, nullify, first_four, last_four, so a member of several groups gets the strictest that matches. Each text rule is type-checked against the column. restrictions_for takes rule_per_principal for a platform whose mask tests each principal itself; GCP and AWS keep refusing two rules for one column. * fix(databricks): a tofu apply that fails part way still binds the masks and row filter of the tables that exist A failed apply raised before post_apply ran, so a table it created, its grants and the dbt job that fills it could exist with no mask or filter, and a scheduled run would expose unmasked rows. Before the apply error is raised, post_apply now runs best effort in a bind-only mode (after_failed_apply): it binds what exists, revokes SELECT where it cannot bind, skips a table that does not exist, and drops nothing, since dropping widens access and waits for an apply that succeeded. A failure there is printed; the run still raises opentofu_apply_failed. * docs(databricks): the changelog and tested-honestly rows describe per-identity tests, per-principal mask rules and the bind-only pass after a failed apply * fix(databricks): a function under a hashed name is read back and must be the one this module creates, before it is bound The name fluid_mask___ can be computed from the contract, so anyone with CREATE FUNCTION on the schema could create it first with a body that returns the value unmasked: CREATE FUNCTION IF NOT EXISTS kept theirs, SET MASK bound it and fluid verify, which compared names only, passed. After each create and before any bind the function is read (GET /api/2.1/unity-catalog/functions/{name}) and its owner, body, parameters and return type must be the expected ones, whitespace-normalised; the owner must be the identity the run authenticates as (userName of scim/v2/Me, who CREATE FUNCTION makes the owner). A mismatch is a failed bind, printed with what differs. fluid verify reads every bound function back and checks the same, accepting the table's owner as well since verify may run as someone else. * fix(databricks): a row filter or mask set by hand is never replaced silently SET ROW FILTER and SET MASK overwrite whatever is bound, so a filter or mask an admin set (a function without the fluid_ prefix) was lost without a word when the contract asked for its own. Every table is read and planned before any statement runs, and a live binding this module did not create, where the contract asks for one, is refused naming the table, the column and the live function, with the remedy: remove it, or move its rule into the contract. Nothing is changed on any table. After an apply it is a failed bind (the fail-safe runs). * fix(databricks): a Tables API read that fails for one table is a failed bind for it, and the other tables are still tried _read raised outside the try that triggers the fail-safe, so a transient 429 or 503 on one table aborted post_apply with that table granted and unbound and the rest never tried. Read failures are caught per table, after an apply and after a failed one: a table with restrictions is a failed bind (the fail-safe runs, the error names it) and the loop goes on; one without is a note. A read that fails before the apply still stops the run. * fix(databricks): a failed bind binds a deny-all row filter first, and only then locks the table by revoking The fail-safe revoked SELECT from the restricted principals alone, which misses a user in another granted group and SELECT inherited from the schema or catalog, and the message claimed none of them could read unmasked rows. A failed bind now binds a fixed-purpose deny-all row filter for the restricted identities (a zero-column function, RETURN NOT , content-hashed and checked for owner and body like every function): it blocks them whatever the grant path. If that cannot be bound, or would replace a row filter this module did not create, SELECT (and ALL PRIVILEGES) is revoked on the table from every grantee but its owner, listed through the permissions API. The message says exactly what was done and names each grant path still open, a schema or catalog grant a table-level REVOKE cannot close, with the statement to run. * fix(databricks): a row filter that needs a column the apply adds is held by a deny-all filter until then pre_apply skipped a filter whose columns the table does not have yet, and the table exists with rows, so the restricted principals read every row until post_apply. When the real filter cannot bind and the table exists, pre_apply now binds the interim deny-all row filter (the fail-safe's own: a zero-column function, SET ROW FILTER ... ON (), owner and body checked) for the restricted identities, and post_apply replaces it with the real one. A dry run shows it; a hostile function under its name, or a row filter this module did not create, stops the run before the apply. SET replaces a bound filter directly: a DROP first would leave a window with no filter. * fix(databricks): generate iac refuses a module a plain tofu apply would leave unmasked The masks and row filters are bound on the SQL warehouse around tofu apply, not in the module, so fluid generate iac wrote a module that creates the table and grants SELECT with none of them and told the reader to review and apply it with OpenTofu. generate iac now asks the plugin (standalone_module_refusal) before it writes one, and refuses a Databricks contract that compiles any mask or row filter as databricks-enforcement-outside-module, saying that a plain tofu apply would grant unmasked access and to use fluid apply. emit stays pure; fluid apply, plan and validate are unaffected. * fix(databricks): a dbt job that writes a table not yet protected is created paused A continuous or frequent cron job created in the same tofu apply as its restricted table and the grant on it could run and fill the table before post_apply binds the masks and row filter. DatabricksIacPlugin.emit (the apply path; the pure compile validate uses is unchanged) now reads the live bindings of each restricted table a job writes and sets pause_status to PAUSED on the job's schedule or continuous block unless the table exists, its row filter and masks are the expected functions on the expected columns, and each function is the one this module creates. An unreadable workspace counts as not protected. The next fluid apply, once they are bound, unpauses it in place, with no drift from unpausing by hand. A manual job has nothing to pause. plan_notices says why a job is created paused and that run-now works meanwhile. The engine comment that claimed binding happens before any build can fill a table is corrected. * docs(databricks): the changelog and tested-honestly rows describe the verified functions, the deny-all fail-safe, the paused jobs and what is still unproven live * chore(tests): the unused names the code-quality review found are gone An unused import of copy and Optional, two unused helper imports and an unused unpacking target in the Databricks tests; the stand-in's deploy identity is now the one constant the module itself uses. --- CHANGELOG.md | 106 ++ HONESTLY_TESTED.md | 2 + fluid_build/cli/_apply_opentofu_engine.py | 65 + fluid_build/cli/_verify_databricks.py | 190 ++- fluid_build/cli/generate_iac.py | 6 + fluid_build/cli/verify.py | 1 + fluid_build/iac/column_access.py | 11 +- fluid_build/iac/providers/databricks.py | 210 ++- .../iac/providers/databricks_access.py | 635 ++++++++ .../iac/providers/databricks_enforce.py | 1038 +++++++++++++ tests/cli/test_verify_databricks.py | 320 +++- tests/iac/_databricks_governed.py | 81 ++ tests/iac/_fake_databricks.py | 285 +++- tests/iac/test_iac_databricks.py | 6 +- tests/iac/test_iac_databricks_access.py | 721 +++++++++ tests/iac/test_iac_databricks_enforce.py | 1293 +++++++++++++++++ tests/iac/test_iac_databricks_fake_e2e.py | 446 +++++- 17 files changed, 5395 insertions(+), 21 deletions(-) create mode 100644 fluid_build/iac/providers/databricks_access.py create mode 100644 fluid_build/iac/providers/databricks_enforce.py create mode 100644 tests/iac/_databricks_governed.py create mode 100644 tests/iac/test_iac_databricks_access.py create mode 100644 tests/iac/test_iac_databricks_enforce.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 58c8ddb4c..4cb39e912 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,112 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 not Databricks, and when the checkout's project config named the endpoint (the runner's environment is not the checkout's to send, as for `{{ env.* }}` and the operator's credential). +- **A Databricks contract's column restrictions and row filters are enforced by Unity Catalog, + where they were refused.** `exposes[].policy.authz.columnRestrictions` and `rowFilters` were + `databricks-governance-unmapped`; they are now SQL functions bound on the table, so a query + by a restricted principal is rewritten by the engine whichever tool sends it, with the + semantics GCP and AWS give the same fields. `access: deny` becomes a mask that returns NULL + for the principal: Unity Catalog has no column-level grant, so a denied column stays listed + in the table's schema and reads NULL (the founder accepted that). `access: mask` returns + `last_four` (`XXXXX` and the last four characters), `first_four`, or NULL for `nullify`; a + value of four characters or fewer is its SHA-256 and a NULL stays NULL, as on AWS. One mask + per column (the Unity Catalog limit) combines every restriction on it, a principal in a deny + and a mask gets the deny, and everyone else reads the value. A mask rule belongs to the + principal that names it, not to the column (on Databricks a column may be masked by + `nullify` for one principal and `last_four` for another; GCP and AWS still refuse that): the + function has one branch per rule, strictest first (deny, `nullify`, `first_four`, + `last_four`), so a member of several groups gets the strictest that matches, and each text + rule is type-checked against the column. One row filter per table: its + principals' members read the union of those principals' predicates, everyone else every row. + Logical principals resolve through `binding.principals`, and every resolved identity is + tested both ways, `(is_account_group_member('') OR current_user() = '')`, + whatever the principal's prefix: a `group:` principal may be mapped to a service principal's + application ID or a user's email, and a test of the wrong kind would match no one while the + grant to that identity still applied (a name that is both a group and a user is restricted + either way, which errs closed). A principal with no prefix or a `domain:` one is + `databricks-principal-kind`, and `access: allow` (which would blind every reader it does not + name, the owner and the pipelines included), a text mask on a non-string column + (`databricks-mask-type`) and anything else this cannot express are refused rather than + dropped. A predicate is contract text that becomes a function body, so it is parsed as one + boolean expression (sqlglot, Databricks dialect), checked node by node against an allowlist + (comparisons, `AND` / `OR` / `NOT`, `IS NULL`, `IN`, `BETWEEN`, `LIKE`, literals, `CAST`, the + table's columns and `lower`, `upper`, `length`, `trim`, `coalesce`, `abs`) and written back + from the tree, never spliced: a `;`, a comment, a subquery, a session function such as + `current_user()`, an undeclared column or a predicate that reads no column is + `databricks-row-filter-invalid`. The functions are created once, as `CREATE FUNCTION IF NOT + EXISTS ... fluid_rf_
_` and `fluid_mask_
__`, named by a + hash of their definition: Unity Catalog refuses to replace a function a filter or mask uses, + so a changed predicate or rule is a new function bound in place of the old one, and the same + contract always names the same function. Superseded functions are not dropped (harmless + metadata; cleaning them up is a follow-up). The provider (1.140.0) has no function resource + and no mask or filter attribute on `databricks_sql_table` (terraform-provider-databricks#4550), + so `ALTER TABLE ... SET ROW FILTER ... ON (...)` and `ALTER COLUMN ... SET MASK` run on the + SQL warehouse (`DATABRICKS_WAREHOUSE_ID`) through the SQL Statement Execution API, outside + OpenTofu, in two hooks `fluid apply` calls around `tofu apply`: a `pre_apply` that binds the + tables that already exist, so a grant the apply adds lands on a protected table, and a + `post_apply` that binds the tables the apply created, empty. Databricks keeps a table's row + filter and masks across the `CREATE OR REPLACE TABLE` a dbt build runs (its docs say so), so + rebuilds keep them. The live bindings are read from the Tables API and only what differs + runs, so a second apply executes no statement; `--mode dry-run` prints the statements and + runs none. A restriction removed from the contract is dropped (`DROP ROW FILTER`, `DROP + MASK`) after the apply, printed as an access-widening change and not gated as data loss; only + a filter or mask whose function carries this module's name prefix is ever dropped, so one an + admin set by hand is left alone, and one is never replaced silently either: where the contract + asks for its own row filter or mask and the table has one whose function lacks the `fluid_` + prefix, the run stops before any change, on any table, naming the table, the column and the + live function (remove it, or move its rule into the contract). A content-hashed name can be + computed from the contract, so anyone with `CREATE FUNCTION` on the schema could take it first + with a body that returns the value unmasked, and `CREATE FUNCTION IF NOT EXISTS` would keep + theirs: after each create and before any bind the function is read + (`GET /api/2.1/unity-catalog/functions/{name}`) and its owner (the identity the run + authenticates as, `userName` of `scim/v2/Me`, who `CREATE FUNCTION` makes the owner), body, + parameters and return type must be the expected ones, whitespace-normalised; a mismatch is + not bound and is printed with what differs. It fails closed: a pre-apply failure stops the run + before `tofu apply`. When `post_apply` cannot bind a table (a failed statement, a hostile + function, a row filter set by hand, or a Tables API read that fails, which is tried per table + so one 503 does not stop the others), it binds a fixed-purpose deny-all row filter for the + restricted identities (a zero-column function, `RETURN NOT `, bound + `ON ()`, content-hashed and checked like the others), which blocks them whatever grant path + gives them the table: a group they are also in, a schema or catalog grant. If that cannot be + bound, or would replace a row filter not made here, it revokes `SELECT` (and `ALL PRIVILEGES`) + on the table from every grantee but its owner, listed through the permissions API, and names + every path still open, a grant on the schema or catalog that a table-level `REVOKE` cannot + close, with the statement to run. The message says exactly what was done, and the run exits + non-zero. The same bind-only pass runs when `tofu apply` itself fails part way, before the + apply's own error is raised: a table the apply created, its grants and the dbt job that fills + it can exist with no mask or filter. It binds what exists, closes what it cannot bind, skips a + table that was never created and drops nothing (dropping widens access and waits for an apply + that succeeded); a failure there is printed and the run still exits with + `opentofu_apply_failed`. When a row filter needs a column the apply adds, `pre_apply` holds the + table, which has rows, behind the same deny-all filter until `post_apply` binds the real one. + A dbt job that writes a restricted table which is not yet protected (the table is missing, a + binding is missing or differs, a bound function is not the expected one, or the live state + cannot be read) is created with `pause_status: PAUSED` on its schedule or continuous block, + so it cannot fill the table before it is bound; the next `fluid apply`, once the bindings + match, unpauses it in place, with no drift from unpausing it by hand, and the apply says so + (an explicit run-now works meanwhile). A manual job has nothing to pause, and `fluid validate` + and the pure compile never read the workspace. Re-running `fluid apply` once the cause of a + failed bind is fixed binds first (replacing a deny-all filter) and grants again. + `fluid generate iac` refuses a Databricks contract that compiles any mask or row filter + (`databricks-enforcement-outside-module`): they are bound by `fluid apply` around `tofu + apply`, not in the module, and a plain `tofu apply` of it would grant unmasked access; + `fluid apply`, `plan` and `validate` are unaffected. A contract with restrictions now needs + the `[databricks]` extra (databricks-sdk) at apply, and says so when it is missing. `fluid + verify` gains a row-filter-and-masks dimension: the expected function names (content-hashed) + against the live `row_filter` and each column's `mask` from the GetTable it already reads, and + each bound function read back for its owner (this verify's identity or the table's owner), + body, parameters and return type; a missing or different binding is CRITICAL with `fluid + apply` as the fix, and a contract that restricts nothing skips it. `policy.privacy.masking` + (landing-time masking), `rowLevelPolicy` and `binding.governance` are still refused. Proven + against the in-process stand-in with a real `tofu` apply, and the function bodies evaluated + in DuckDB; not run against a live workspace: that `CREATE FUNCTION IF NOT EXISTS`, `SET ROW + FILTER` (with `ON ()` as well) and `SET MASK` behave as the stand-in models them, that `SET` + replaces a binding already on the table (if it does not, a changed predicate or the interim + filter fails closed with the old binding in place; a `DROP` first was not used because it + leaves a window with no filter), that FunctionInfo's `routine_definition` is the body as + written, that `current_user()` is a service principal's application ID, and what a warehouse + of each type enforces are unproven. + ### Fixed - **A contract's governance labels were dropped silently on Databricks.** The Databricks emitter diff --git a/HONESTLY_TESTED.md b/HONESTLY_TESTED.md index da541306d..def4a5a6b 100644 --- a/HONESTLY_TESTED.md +++ b/HONESTLY_TESTED.md @@ -85,6 +85,8 @@ Markers declared in `pyproject.toml`. | Stage 1 unit | `tests/iac/test_iac_databricks.py`, `test_iac_databricks_jobs.py`, `test_iac_databricks_auth.py`, `test_iac_databricks_engine_phase2.py`, `tests/test_util_cron.py`, `tests/build_runners/test_databricks_job_builds.py`, `tests/engines/test_dbt_databricks.py`, `tests/cli/test_verify_databricks.py`, `tests/test_doctor_databricks.py`, `tests/test_welcome_scan_databricks.py` | ✅ 521 tests in the Databricks files above (139 tables, 149 jobs, 56 credentials, 47 engine hooks, 45 cron, 31 runner, 40 verify, 14 doctor and welcome scan), plus the dbt profile and type tests. The emitted module: a `databricks_catalog` data source, the owned schema, a managed Delta table, per-principal grants with best-effort imports, the provider's `retry_timeout_seconds`. A declared `binding.location.region` (validated, lower-cased; `databricks-region-invalid` and `databricks-region-conflict` refused) adds `data.databricks_current_metastore` and a precondition on each owned schema; with none the module is unchanged. A contract's governance labels (`iac/governance_labels.py`, the AWS and GCP source) are one `databricks_entity_tag_assignment` per tag on each owned schema (the contract's own) and each table (the expose's too), `-` read as `_` in a key; two labels meeting at one key are `databricks-tag-collision`, more than 50 tags on a securable `databricks-tags-too-many`; a contract without labels gets the module it always did; each assignment has a best-effort import (`entity_type,entity_name,tag_key`); removing one is not data loss. Databricks is a region-placed platform for the `sovereignty` checks (`tests/policy/test_sovereignty_enforcement_mode.py`). Each dbt build on Databricks becomes a serverless job with its workspace files, mapped from the build: Quartz cron, triggers, retries, notifications, timeout and run-as. The project upload's allowlist and refusals; the escaped file names. Every refusal `fluid validate` and the emitter share. The credential policy (host, profile, auth type, blanked variables) and the SDK kwargs. The in-place data-loss and protocol-upgrade readings of a saved plan. Job and file removals left ungated; the `USE CATALOG` notice; the drift report for dbt-written tables. `fluid verify`'s Databricks inspector against a stubbed client (residency: pass, mismatch, skipped; governance tags: pass, a missing tag, a wrong value, the paginated list read to its last page with an empty page in the middle, skipped); the doctor rows and welcome-scan detection; the dbt runner leaving a Databricks job build to its job. | What Databricks accepts (the manual live row). The real databricks-sdk is neither in the dev install nor in CI, so a stubbed client stands in for it; one test answers as the live workspace did, with the privilege under the group that conveys it. | | Stage 1 `tofu validate` | `tests/iac/test_iac_tofu_validate.py` `[databricks]`, `test_a_databricks_dbt_job_passes_tofu_validate[git\|workspace]` | ✅ Both job modules (GIT with schedule, retries, notifications and run-as; WORKSPACE with a continuous trigger and files) and the dbt-written table's `ignore_changes` list validate against provider 1.140.0's schema. | Anything the provider checks only at plan or apply. | | Stage 1 real tofu, stand-in workspace: tables | `tests/iac/test_iac_databricks_fake_e2e.py` (`tests/iac/_fake_databricks.py`) | ✅ 28. The real `fluid` CLI, `tofu` and databricks/databricks 1.140.0 against an in-process stand-in for the Unity Catalog, SQL statement and permissions APIs. validate / plan / `generate iac --validate` / dry-run route to OpenTofu, and a second plan is empty. A partition change (a replace), a dropped column and a positional rename are refused without `--allow-data-loss`; a grant revoke is not. A column reorder is refused even with the flag. An added column applies in place. A table that exists with column mapping is adopted, not replaced; without it the apply is refused (with or without the flag) until its owner runs the `ALTER`, and the refused run, like a dry-run, leaves nothing it imported in state. Existing grants are adopted with their table: the adoption plan is `+0 ~1 -0` (the provider imports neither the table's properties nor its warehouse id, so the first apply writes back the values it already has), and the plan after it is empty. A privilege the contract does not name shows as an update that removes it. A declared `location.region` plans and applies when the stand-in's metastore is in it (case-insensitively) and fails the plan, naming both regions and `DATABRICKS_HOST`, before anything is written when it is not. A type change and a missing catalog fail at plan or import, and a missing host or credential fails before `tofu` starts; each names the fix before any change. `fluid diff` initialises with the committed lock. A contract's governance labels are created as tag assignments on the schema and the table in the stand-in and a second plan is empty; a changed value updates the tag in place; a removed label destroys only its assignment, without `--allow-data-loss`; an assignment already in the workspace (state lost, or set by hand with another value) is adopted and its value corrected, never a failed create, and a tag someone else set is left alone. | The stand-in models the requests provider 1.140.0 sends: SQL dialect, warehouse state and privilege names are not checked. Its metastore region is whatever the test sets; whether a real workspace's `metastore_summary` reports the region in the spelling a contract declares (`eu-central-1`, `westeurope`) is not proven: the residency check and the verify dimension were not run against a live workspace; the governance tags and their verify dimension were not run live either. The tag endpoints are modelled on the SDK's source (databricks-sdk-go v0.190.0, the one provider 1.140.0 builds on), not on a live workspace: the live tier was not run for tags, so what real Databricks accepts as a tag key or value, whether it keeps a table's tags across a dbt CREATE OR REPLACE (undocumented), and whether the import id `entity_type,entity_name,tag_key` resolves a live assignment are unproven; `fluid verify`'s governance-tags dimension is the gate for the last. It stores the table properties it was sent, so the adoption check reads `properties` / `effective_properties` as a real workspace may not fill them. | +| Stage 1 unit: Databricks column masks and row filters | `tests/iac/test_iac_databricks_access.py` (the SQL), `tests/iac/test_iac_databricks_enforce.py` (the hooks and the engine's gate), `tests/cli/test_verify_databricks.py` (21 of its 61), `tests/iac/_databricks_governed.py`, `tests/iac/_fake_databricks.py` | ✅ 97 + 84 + 21. A Customer-Profile-360-shaped contract (six denied columns, a `msisdn` mask, a consent row filter, principals mapped to `fluid-demo-lab-*` groups) compiles to the exact `CREATE FUNCTION IF NOT EXISTS` / `ALTER TABLE ... SET ROW FILTER ... ON (...)` / `ALTER COLUMN ... SET MASK` statements. The function bodies are evaluated in DuckDB (transpiled from the Databricks dialect, `is_account_group_member` stubbed): a restricted group reads only the rows its predicate keeps, a member of several filtered principals reads the union, everyone else every row, a NULL predicate hides the row from a member; `last_four` / `first_four` / `nullify` and a deny give the BigQuery-matching values, a value of four characters or fewer is its SHA-256, a NULL stays NULL, a deny beats a mask, and a mask rule belongs to the principal that names it (nullify for one group and last_four for another on one column: strictest branch first, a member of both reads NULL). Every identity is tested as a group and as a user, so a `group:` principal mapped to a service principal's application ID or a user's email is still restricted (DuckDB's own session user stands in for a service principal that is in no group). Names carry a hash of the definition: the same contract names the same functions, a changed predicate or rule names a new one. Every refusal kind, 30 predicate injection attempts (`;`, comments, subqueries, session functions, unknown or qualified columns, a backslash-escaped quote) and a principal name with a quote or a backslash. The hooks against the stand-in (statements, Tables API reads): pre_apply binds existing tables and leaves a missing table to post_apply, a dry run prints and executes nothing, a second run executes nothing, a changed predicate binds a new function and keeps the old, a removal drops only what this module bound, a statement needing a column the apply adds waits for post_apply, a failed pre_apply stops the run and revokes nothing, a failed post_apply binds a deny-all row filter for the restricted identities (SELECT untouched), and when that cannot be bound, or would replace a filter not made here, revokes SELECT and ALL PRIVILEGES from every grantee but the owner, lists the schema and catalog grants still open with the statement to run, and reports a revoke that failed or a grant listing that failed; a function pre-created under any hashed name (row filter, mask, deny-all) by another owner, with another body, parameters or return type, is never bound; a row filter or mask set by hand is never replaced, with nothing changed on any table; a Tables API read that fails for one table (503, 429) is a failed bind for it and the next table is still tried; a filter needing a column the apply adds is held by the deny-all filter in pre_apply; a dbt job (cron or continuous) writing a table that does not exist, is unbound, has a wrong function or cannot be read is created PAUSED, and a manual job or a job writing no restricted table is left alone, after a `tofu apply` that failed part way, post_apply binds what exists, drops nothing, skips a table never created without a NOT REVOKED message, and the engine prints an enforcement failure there without raising it; the SQL statement poller (running, failed, cancelled, timed out, credentials scrubbed). The engine calls the hooks in order around `tofu apply`. `fluid generate iac` refuses a module with masks or filters (emit stays pure). `fluid verify`'s row-filter-and-masks dimension, including the owner, body, parameters and return type of every bound function (pass, no filter, another function, other input columns, missing and wrong mask, case and backticks ignored, skipped, the CLI output and `--strict`). | What Databricks accepts. The stand-in executes the statements as this module sends them and models the Unity Catalog refusal of `CREATE OR REPLACE FUNCTION` on a bound function, the preserved bindings across `CREATE OR REPLACE TABLE`, and the Tables API fields `row_filter.function_name` / `input_column_names` and `columns[].mask.function_name` (databricks-sdk-go v0.190.0), all from the docs and the SDK source, not from a live workspace. DuckDB evaluates the bodies, not Databricks: `sha2`, `right`, `left` and `is_account_group_member` are Databricks' own. | +| Stage 1 real tofu, stand-in workspace: masks and row filters | `tests/iac/test_iac_databricks_fake_e2e.py` (17 of its 45; needs databricks-sdk, the `[databricks]` extra, and skips without it) | ✅ 17. The real `fluid` CLI and `tofu` apply bind the filter and masks (after the apply, on a table it created); a second apply executes no statement; a table that already exists is bound before the apply's first grant on it; `--mode dry-run` prints the statements and executes none; a changed predicate binds a new function and keeps the old; a removed restriction is dropped and said to widen access without `--allow-data-loss`; a binding that fails revokes SELECT from the restricted principal only, fails the run (`opentofu_access_enforcement_failed`), and the next apply binds before it grants again; a failure before the apply stops it with nothing applied; an invalid predicate is refused before anything runs; an apply that fails after creating the table (the grant on it refused) still binds the filter and masks and the run exits with `opentofu_apply_failed`, an apply that fails before the table exists binds nothing and revokes nothing, and a failed apply drops no binding the contract no longer asks for (it is dropped by the next apply that succeeds); a function taken under the hashed name stops the apply before it runs; a row filter set by hand stops it and is left in place; a binding that fails with no deny-all possible revokes every grantee but the owner and the next apply binds first and grants again; a failed mask leaves the deny-all filter and every grant until the next apply; a filter needing a column the apply adds is held by the deny-all filter, then replaced, in that order around the added column; a dbt job created with its restricted table is paused and the next apply unpauses it in place; `generate iac` refuses the module. | Live behaviour: that `CREATE FUNCTION IF NOT EXISTS`, `SET ROW FILTER` and `SET MASK` run on a real warehouse as modelled, that a second `SET` replaces a binding already on the table (if it does not, a changed predicate fails closed with the old binding in place), that `current_user()` of a service principal is its application ID, that a Free Edition or classic warehouse can run them, and whether the SDK's statement API answers as the stand-in does; whether `SET ROW FILTER ... ON ()` (a zero-column function) is accepted; whether FunctionInfo's `routine_definition`, `input_params` and `full_data_type` come back in the form compared (whitespace-normalised, lower-case types), whose mismatch would refuse a good function; whether `scim/v2/Me` `userName` is the owner `CREATE FUNCTION` records for a service principal; that `SET` replaces a bound filter or mask (the stand-in does; none of the fixes uses a DROP first, which would leave a window with no filter); the permissions and effective-permissions fields used to lock a table; and the pause behaviour of a job on a live workspace. Unproven for ALL_PRIVILEGES grantees on a live workspace: a revoke of ALL PRIVILEGES removes more than read access, by design of the lock. A principal that is a metastore admin or the table's owner is never restricted by these bindings. | | Stage 1 real tofu, stand-in workspace: dbt jobs | `tests/iac/test_iac_databricks_jobs_fake_e2e.py` | ✅ 6. A local dbt build applies as one serverless job plus the workspace files dbt reads, each in a folder created before it (no upload meets a missing parent) (never `profiles.yml`, `target/` or a hidden file), and a second plan is empty. A changed model updates its file in place. A GIT build runs its tag on a continuous trigger and uploads nothing. Turning a job build local removes the job, its files and their folders without `--allow-data-loss`, and says so; a folder holding a file the apply did not put there is not deleted. The table a dbt build writes keeps what dbt's CREATE OR REPLACE made of it, and the apply reports the column drift instead of failing on a type change. | Whether Databricks accepts and runs the job: serverless environment version 2, the `dbt-databricks>=1.8.0,<2.0.0` range, the commands, a WORKSPACE `project_directory` under `/Workspace`, `run_as` permissions, a Git credential for a private repository, `dbt deps` with a local package (not uploaded). The manual live row proves the first four. Whether the jobs API echoes a server default the stand-in does not (the fake's builder found `performance_target` and an all-false `notification_settings` would each plan an update), which would make a second plan non-empty (live, it was empty). What dbt's CREATE OR REPLACE does to a table with column mapping (the module sets none on a dbt-written table). Not bound by a plan: an apply uploads the dbt project as it is when it runs (`planDigest` and `bundleDigest` cover the contract, not the project's files, as for a local dbt run), and a bundle does not carry the project. Known gap, by design: a job is not adopted. Its import id is a job id, which neither the contract nor `tofu` can look up by name, so an apply that lost its state creates a second job. `fluid verify` flags the duplicate (a stubbed client proves it). Workspace files need no import: the provider's create overwrites. | | Manual live, Free Edition (founder-approved) | the real `fluid` CLI, no test file: run 2026-11-01 from main dfd6f145, databricks-sdk 0.158.0, every `fluid_iactest_*` object swept after (1 schema, 1 table, 1 folder; 0 left) | ✅ validate, `generate iac --validate`, plan, dry-run and apply `+9`; a second plan is EMPTY against the real Jobs API; Databricks accepts the Quartz schedule, serverless environment 2 with `dbt-databricks>=1.8.0,<2.0.0` (dbt 1.12.6, adapter 1.13.0), the WORKSPACE `project_directory` and the commands as emitted; one `run-now` ends SUCCESS and dbt builds the table; `fluid verify` with the real SDK (columns, types, one job); `fluid diff`'s state check (its live check has no Databricks inspector); a model change `~1`; the build made local removes job, files and folders `-5` without `--allow-data-loss` (the empty `.fluid//` folders above `files/` stay: only `files/` and below are managed). It found the USE CATALOG check reading a group-conveyed privilege as none. After the fix, `fluid verify` on a fresh one-table product (`+4`) passes every dimension, catalog access included (1 schema and 1 table swept, 0 left). | A GIT source, profile or OAuth auth, a run-as service principal, table adoption and a larger project, live; the region residency check (the metastore precondition and `fluid verify`'s residency dimension), which was added after this run and not run live; dbt's CREATE OR REPLACE drops a contract's NOT NULL, so verify flags a `required` column a dbt model does not constrain itself. | | Stage 2 emulator (nightly, non-gating) | `tests/iac/test_iac_databricks_minilake.py` (job `nightly-databricks-minilake`) | ⚠️ first run 2026-10-30 (manual dispatch, run 41111425504, minilake v1.9.0): 1/4. Schema, table and grant creates work, and the replace gate holds. | A second plan is not empty and every in-place table update fails: minilake strips `TBLPROPERTIES` at create and its SQL parser rejects `ALTER TABLE ... SET TBLPROPERTIES`, which the provider sends before each column change. An upstream issue is drafted, not filed. Jobs and workspace files are not in this flow yet. | diff --git a/fluid_build/cli/_apply_opentofu_engine.py b/fluid_build/cli/_apply_opentofu_engine.py index 4032fceab..584d301f0 100644 --- a/fluid_build/cli/_apply_opentofu_engine.py +++ b/fluid_build/cli/_apply_opentofu_engine.py @@ -291,9 +291,15 @@ def _apply_via_opentofu(args, logger: logging.Logger, contract: Optional[Mapping _enforce_data_loss_gate(args, logger, provider, changes, data_changes, lost_in_place) if dry_run: + _enforce_access(plugin, "pre_apply", contract, dry_run=True) cprint("\ndry-run: plan only — not applying.") info(logger, "opentofu_apply_dry_run", provider=provider, **changes) return 0 + # Access enforcement the cloud keeps outside OpenTofu (Unity Catalog column masks + # and row filters) goes onto the tables that already exist BEFORE the apply, so a + # grant the apply adds lands on a protected table. A failure stops the run here: + # nothing has been applied. + _enforce_access(plugin, "pre_apply", contract) imported.keep() # ``fluid apply`` is the headline state-changing command, so a receiver @@ -305,6 +311,9 @@ def _apply_via_opentofu(args, logger: logging.Logger, contract: Optional[Mapping with _apply_lineage(contract, provider=provider, planned=changes) as record: apply_result = runner.tofu_apply(str(workdir), env=env) if not apply_result.ok: + # A table the failed apply created, its grants and the job that fills it can all + # exist with no mask or filter: bind what exists (never drop) before failing. + _enforce_after_failed_apply(plugin, contract) raise CLIError( 1, "opentofu_apply_failed", @@ -319,10 +328,66 @@ def _apply_via_opentofu(args, logger: logging.Logger, contract: Optional[Mapping planned=changes, applied=applied, resources=_module_addresses(module), dry_run=False ) + # The tables this apply created exist now, empty, and get their masks and row filter here. + # A dbt job that writes one was created paused in this same apply (the plugin's ``emit`` + # reads the live bindings), and the next apply starts it once they are in place, so no + # build fills a table first. A table that cannot be bound is closed to the principals it + # restricts (a deny-all row filter, else revoked), and the run fails. + _enforce_access(plugin, "post_apply", contract) + info(logger, "opentofu_apply_ok", provider=provider, **applied) return 0 +def _enforce_access( + plugin: Any, hook: str, contract: Mapping[str, Any], *, dry_run: bool = False +) -> None: + """Run a plugin's ``pre_apply`` / ``post_apply`` access-enforcement hook, if it has one. + + Optional plugin capability, discovered like ``plan_notices``: the Databricks plugin + binds Unity Catalog column masks and row filters, which the provider cannot manage + (``iac/providers/databricks_enforce.py``). Unlike the advisory hooks this one gates: a + refusal or failure is a typed error and the run exits non-zero. With ``dry_run`` the + plugin prints the statements and runs none. + """ + run = getattr(plugin, hook, None) + if not callable(run): + return + try: + if dry_run: + run(contract, say=cprint, dry_run=True) + else: + run(contract, say=cprint) + except UnsupportedBindingError as exc: + raise CLIError( + 1, + "opentofu_access_enforcement_failed", + {"kind": exc.kind, "error": str(exc), "remediation": list(exc.remediation)}, + ) from exc + + +def _enforce_after_failed_apply(plugin: Any, contract: Mapping[str, Any]) -> None: + """Best effort, bind-only ``post_apply`` for a ``tofu apply`` that failed part way. + + A failure here is printed, never raised: the error the run exits with is the apply's own + (``opentofu_apply_failed``). The plugin binds what exists, revokes ``SELECT`` where it + cannot, skips tables that do not exist and drops nothing (``after_failed_apply``). + """ + run = getattr(plugin, "post_apply", None) + if not callable(run): + return + cprint( + "\n tofu apply failed part way: binding the masks and row filter on the tables that " + "exist before the run stops (no mask or filter is dropped)." + ) + try: + run(contract, say=cprint, after_failed_apply=True) + except UnsupportedBindingError as exc: + cprint(f" access enforcement after the failed apply: {exc} {' '.join(exc.remediation)}") + except Exception as exc: # noqa: BLE001 — the apply's own error is what the run raises + cprint(f" access enforcement after the failed apply could not run: {type(exc).__name__}") + + def _applied_env(args) -> Optional[str]: """The overlay env this apply is for: ``--env``, else the bundle's own.""" env = getattr(args, "env", None) diff --git a/fluid_build/cli/_verify_databricks.py b/fluid_build/cli/_verify_databricks.py index 11fbce70f..051c555ff 100644 --- a/fluid_build/cli/_verify_databricks.py +++ b/fluid_build/cli/_verify_databricks.py @@ -37,7 +37,18 @@ Read from ``GET .../entity-tag-assignments/{type}/{name}/tags``, which pages. A dbt ``CREATE OR REPLACE`` may drop a table's tags (Databricks does not say), so this is the gate: the fix is ``fluid apply``. -4. **The jobs.** Each dbt build that writes the expose has exactly one job of +4. **Row filter and column masks.** The contract's ``columnRestrictions`` and ``rowFilters`` + are Unity Catalog SQL functions bound on the table (``databricks_access.py``), named by a + hash of their definition. The table's ``row_filter.function_name`` / ``input_column_names`` + and each column's ``mask.function_name``, from the GetTable already read, must be the + expected ones: a missing binding, or one to another function, is CRITICAL (the table is + readable without what the contract promises), and the fix is ``fluid apply``, which binds + them before it grants. Each bound function is read back too + (``GET /api/2.1/unity-catalog/functions/{name}``): its owner must be this verify's identity + or the table's owner, and its body, parameters and return type the expected ones, since a + content-hashed name is computable from the contract and could have been taken by someone + else's function. ``skip`` when the contract restricts nothing. +5. **The jobs.** Each dbt build that writes the expose has exactly one job of its name. None means ``fluid apply`` did not run (or the job was deleted); several mean an apply without its state created a duplicate. @@ -437,6 +448,167 @@ def _tags_dimension(client: Any, table: Any) -> Dict[str, Any]: } +def _enforcement_dimension( + table: Any, live_table: Mapping[str, Any], client: Any = None +) -> Dict[str, Any]: + """The row filter and column masks the contract asks for, against the table's live ones. + + Expected function names carry a hash of their definition, so a changed predicate or + rule is a different name and shows here as "different function". ``skip`` when the + contract declares no restriction: nothing was promised. A binding the contract does + not ask for is not checked. + + A bound function is read back too (``client`` given): its owner must be the identity + this verify runs as or the table's owner, and its body, parameters and return type the + expected ones. The hashed name is computable from the contract, so a function someone else + created under it first would pass a comparison of names. + """ + from fluid_build.iac.providers.databricks_access import same_function + + row_filter = getattr(table, "row_filter", None) + masks = tuple(getattr(table, "masks", ())) + if row_filter is None and not masks: + return { + "status": "skip", + "message": "the contract declares no column restriction or row filter to check", + "findings": [], + } + findings: List[Dict[str, Any]] = [] + if row_filter is not None: + function, columns = row_filter + live = live_table.get("row_filter") or {} + actual = str(live.get("function_name") or "") if isinstance(live, Mapping) else "" + inputs = [str(c).lower() for c in (live.get("input_column_names") or [])] + if not actual: + findings.append( + { + "kind": "row_filter", + "expected": function, + "actual": None, + "message": f"{table.full_name} has no row filter, expected {function}", + } + ) + elif not same_function(actual, function): + findings.append( + { + "kind": "row_filter", + "expected": function, + "actual": actual, + "message": ( + f"{table.full_name} has row filter {actual}, the contract's is {function}" + ), + } + ) + elif inputs != [c.lower() for c in columns]: + findings.append( + { + "kind": "row_filter", + "expected": function, + "actual": actual, + "message": ( + f"row filter {function} on {table.full_name} reads columns " + f"{', '.join(inputs) or 'none'}, expected {', '.join(columns)}" + ), + } + ) + live_columns = { + str(c.get("name") or "").lower(): c + for c in live_table.get("columns") or [] + if isinstance(c, Mapping) + } + for column, function in masks: + mask = (live_columns.get(column.lower()) or {}).get("mask") or {} + actual = str(mask.get("function_name") or "") if isinstance(mask, Mapping) else "" + if not actual: + findings.append( + { + "kind": "mask", + "column": column, + "expected": function, + "actual": None, + "message": f"{table.full_name}.{column} has no mask, expected {function}", + } + ) + elif not same_function(actual, function): + findings.append( + { + "kind": "mask", + "column": column, + "expected": function, + "actual": actual, + "message": ( + f"{table.full_name}.{column} has mask {actual}, the contract's is " + f"{function}" + ), + } + ) + if client is not None and getattr(table, "enforcement", None) is not None: + findings += _function_findings(table, live_table, client, findings) + total = len(masks) + (1 if row_filter is not None else 0) + if not findings: + return { + "status": "pass", + "message": f"all {total} row filter and column mask binding(s) are in place", + "findings": [], + } + return { + "status": "fail", + "message": ( + f"{len(findings)} of {total} row filter and column mask binding(s) missing or " + "different: the table is readable without what the contract promises" + ), + "findings": findings, + "fix": ( + "Run fluid apply: it binds the row filter and masks (before any grant it adds) " + "and revokes SELECT from the restricted principals of a table it cannot bind" + ), + } + + +def _function_findings( + table: Any, live_table: Mapping[str, Any], client: Any, named: Sequence[Mapping[str, Any]] +) -> List[Dict[str, Any]]: + """A finding for each expected, correctly named binding whose function is not the expected one.""" + from fluid_build.iac.providers.databricks_enforce import ( + deploy_identity, + function_problem, + read_function, + ) + + enforcement = table.enforcement + wrong = {f.get("expected") for f in named} + expected = [] + if enforcement.row_filter is not None: + expected.append(("row filter", enforcement.row_filter.function)) + expected += [(f"mask on {m.column}", m.function) for m in enforcement.masks] + expected = [(label, fn) for label, fn in expected if fn.full_name not in wrong] + if not expected: + return [] + owners = {str(live_table.get("owner") or "")} + try: + owners.add(deploy_identity(client)) + except Exception: # noqa: BLE001 — the table's owner alone is then the reference + pass + owners.discard("") + out: List[Dict[str, Any]] = [] + for label, fn in expected: + try: + info = read_function(client, fn.full_name) + except Exception as exc: # noqa: BLE001 — a function that cannot be read is not verified + raise _Unreadable(f"could not read function {fn.full_name}: {_failure(exc)}") from exc + problem = function_problem(info, fn, owners or {"unknown"}) + if problem is not None: + out.append( + { + "kind": "function", + "expected": fn.full_name, + "actual": fn.full_name, + "message": f"the {label} of {table.full_name}: {problem}", + } + ) + return out + + def _jobs_dimension(client: Any, expose_id: str, contract: Mapping[str, Any], jobs: Any) -> Any: """The dimension for the dbt jobs that write ``expose_id``; ``None`` when there are none.""" from fluid_build.util.contract import builds_writing_expose @@ -491,9 +663,11 @@ def _severity( missing_table: str = "", residency: Optional[Mapping[str, Any]] = None, tags: Optional[Mapping[str, Any]] = None, + enforcement: Optional[Mapping[str, Any]] = None, ) -> Dict[str, Any]: """The shared drift grading, raised to CRITICAL by a missing table, reader, job, - a workspace outside the declared region or a governance tag that is not there.""" + a workspace outside the declared region, a governance tag that is not there or a row + filter or column mask that is not bound.""" from fluid_build.cli.verify import assess_drift_severity severity = assess_drift_severity( @@ -520,6 +694,11 @@ def _severity( if (tags or {}).get("status") == "fail": reasons.append(tags["message"] + ": " + "; ".join(f["message"] for f in tags["findings"])) actions.append(tags["fix"]) + if (enforcement or {}).get("status") == "fail": + reasons.append( + enforcement["message"] + ": " + "; ".join(f["message"] for f in enforcement["findings"]) + ) + actions.append(enforcement["fix"]) if not reasons: return severity already = severity["level"] == "CRITICAL" @@ -621,6 +800,7 @@ def verify_databricks_expose( access = _access_dimension(client, table) residency = _residency_dimension(client, table) tags = _tags_dimension(client, table) + enforcement = _enforcement_dimension(table, live_table, client) job_dim = _jobs_dimension(client, expose_id, contract, jobs) except Exception as exc: # noqa: BLE001 — a check that could not run is an error return _error( @@ -632,6 +812,7 @@ def verify_databricks_expose( dimensions["catalog_access"] = access dimensions["residency"] = residency dimensions["governance_tags"] = tags + dimensions["row_filter_masks"] = enforcement if job_dim is not None: dimensions["jobs"] = job_dim @@ -644,6 +825,7 @@ def verify_databricks_expose( "catalog_access", "residency", "governance_tags", + "row_filter_masks", "jobs", ) if name in dimensions @@ -653,7 +835,9 @@ def verify_databricks_expose( "exists": True, "table_id": target, "target": target, - "severity": _severity(diff, access, job_dim, residency=residency, tags=tags), + "severity": _severity( + diff, access, job_dim, residency=residency, tags=tags, enforcement=enforcement + ), "dimensions": dimensions, "metadata": { "num_rows": None, diff --git a/fluid_build/cli/generate_iac.py b/fluid_build/cli/generate_iac.py index cfe27e497..f08cb4535 100644 --- a/fluid_build/cli/generate_iac.py +++ b/fluid_build/cli/generate_iac.py @@ -125,6 +125,12 @@ def run(args, logger: logging.Logger) -> int: provider = _resolve_provider(contract, getattr(args, "provider", "auto")) plugin = get_iac_plugin(provider) with contract_source(source_contract_file(contract_path)): + # A module written for review is applied with a plain `tofu apply`, which would + # skip what the plugin binds outside OpenTofu (Unity Catalog masks and row filters). + refuse = getattr(plugin, "standalone_module_refusal", None) + refusal = refuse(contract) if callable(refuse) else None + if refusal is not None: + raise refusal actions = native_actions(contract, logger, offline=True) resources = plugin.emit(contract, actions) data = plugin.emit_data(contract, actions) diff --git a/fluid_build/cli/verify.py b/fluid_build/cli/verify.py index 27f8d7b31..288e344e2 100644 --- a/fluid_build/cli/verify.py +++ b/fluid_build/cli/verify.py @@ -1193,6 +1193,7 @@ def _render_masking_dimension(masking: Dict[str, Any], ok: str, heading: str) -> ("catalog_access", "\n 🔍 Catalog access (USE CATALOG)"), ("residency", "\n 🔍 Residency (workspace region)"), ("governance_tags", "\n 🔍 Governance tags (Unity Catalog)"), + ("row_filter_masks", "\n 🔍 Row filter and column masks (Unity Catalog)"), ("jobs", "\n 🔍 Jobs"), ) diff --git a/fluid_build/iac/column_access.py b/fluid_build/iac/column_access.py index b74fecc2c..831ccd99d 100644 --- a/fluid_build/iac/column_access.py +++ b/fluid_build/iac/column_access.py @@ -104,9 +104,15 @@ def _where(exposure: Mapping[str, Any], index: int) -> str: return policy_where(exposure, index, "columnRestrictions") -def restrictions_for(exposure: Mapping[str, Any], index: int = 0) -> Tuple[Restriction, ...]: +def restrictions_for( + exposure: Mapping[str, Any], index: int = 0, *, rule_per_principal: bool = False +) -> Tuple[Restriction, ...]: """The expose's column restrictions, or ``()``; a malformed one is refused. + A column has one masking rule on GCP (its policy tag's data policy) and AWS (its view), + so two rules for one column are refused. ``rule_per_principal`` lifts that for a platform + whose mask is a function that tests each principal itself (Databricks). + Every column must be one the expose's schema declares: a restriction on a column the table does not have protects nothing, and on GCP it cannot be attached. """ @@ -182,7 +188,8 @@ def restrictions_for(exposure: Mapping[str, Any], index: int = 0) -> Tuple[Restr mask=mask, ) ) - _refuse_two_rules_for_a_column(out, where) + if not rule_per_principal: + _refuse_two_rules_for_a_column(out, where) return tuple(out) diff --git a/fluid_build/iac/providers/databricks.py b/fluid_build/iac/providers/databricks.py index dbebb0786..73185f8d5 100644 --- a/fluid_build/iac/providers/databricks.py +++ b/fluid_build/iac/providers/databricks.py @@ -52,6 +52,22 @@ existing assignment is imported best-effort before the first apply, and removing a label destroys only its assignment, which holds no data. Whether a tag survives a dbt ``CREATE OR REPLACE`` is undocumented, so ``fluid verify`` is the gate. +* ``policy.authz.columnRestrictions`` and ``policy.authz.rowFilters`` are enforced by Unity + Catalog: SQL functions bound on the table, with the semantics GCP and AWS give them + (``databricks_access.py``). ``deny`` is a mask that returns NULL (Unity Catalog has no + column grant, so the column stays listed), ``mask`` returns ``last_four`` / ``first_four`` / + NULL, and a row filter (one per table) is the union of its principals' predicates for the + members of those principals and every row for everyone else. The functions are created once + under names that carry a hash of their definition, since Unity Catalog refuses to replace one + a filter uses, and are bound with ``ALTER TABLE`` on the SQL warehouse through the SQL + Statement Execution API, outside OpenTofu: the provider has no function resource and no mask + or filter attribute (terraform-provider-databricks#4550). :meth:`DatabricksIacPlugin.pre_apply` + binds on the tables that exist before ``tofu apply`` runs, :meth:`~DatabricksIacPlugin.post_apply` + on the ones it created (revoking ``SELECT`` from the restricted principals of a table it + cannot bind) and drops what the contract no longer restricts (``databricks_enforce.py``). + A predicate that is not one boolean expression over the table's columns is + ``databricks-row-filter-invalid``; ``access: allow``, a principal with no SQL test and a text + mask on a non-string column are refused too. ``fluid verify`` checks the bindings. * A dbt build that runs on Databricks (``execution.runtime.platform: databricks``) becomes a serverless ``databricks_job`` with a ``dbt_task``, and a local project its ``databricks_workspace_file`` uploads @@ -111,7 +127,7 @@ import os import re from dataclasses import dataclass, field -from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple from ...providers._sql_safety import validate_ident, validate_sql_type_param_payload from ...util.contract import is_databricks_job_build @@ -129,7 +145,9 @@ ) from ..provider_match import canonical_cloud, is_cloud from ..versions import required_providers +from .databricks_access import TableEnforcement, compile_enforcement from .databricks_auth import tofu_env_overlay +from .databricks_enforce import Target, post_apply, pre_apply, preview, unprotected from .databricks_jobs import ( CompiledJobs, DbtJob, @@ -239,6 +257,8 @@ class _Table: exposure: Mapping[str, Any] #: The declared workspace region, lower-cased and validated; ``""`` if none. region: str = "" + #: The column masks and row filter Unity Catalog enforces on it, if the expose has any. + enforcement: Optional[TableEnforcement] = None @property def full_name(self) -> str: @@ -429,8 +449,6 @@ def _refuse_governance( for name, value in ( ("policy.privacy.masking", privacy.get("masking")), ("policy.privacy.rowLevelPolicy", privacy.get("rowLevelPolicy")), - ("policy.authz.rowFilters", authz.get("rowFilters")), - ("policy.authz.columnRestrictions", authz.get("columnRestrictions")), ("binding.governance", binding.get("governance")), ) if _set(value) @@ -442,8 +460,10 @@ def _refuse_governance( f"{where} declares {', '.join(unmapped)}, which the Databricks emitter does not " "map yet. Applied anyway, the table would be readable without the mask, filter " "or restriction the contract promises.", - "Remove them from this expose until Unity Catalog column masks, row filters " - "and ABAC policies are mapped, and enforce access with grants.", + "Remove them from this expose until they are mapped (landing-time masking and " + "ABAC policies are not), and enforce access with grants. policy.authz." + "columnRestrictions and policy.authz.rowFilters are mapped: they become Unity " + "Catalog column masks and a row filter.", ) ) if _set(binding.get("encryption")): @@ -666,6 +686,21 @@ def _resolve_table( partitions = _partitions(where, loc, columns, refusals, validating=validating) region = _region(where, loc, refusals) _refuse_governance(where, binding, exposure, refusals) + enforcement: Optional[TableEnforcement] = None + if len(refusals) == before: + try: + enforcement = compile_enforcement( + where=where, + index=index, + exposure=exposure, + binding=binding, + catalog=names["database"], + schema=names["schema"], + table=names["table"], + columns=columns, + ) + except UnsupportedBindingError as exc: + refusals.append(exc) if len(refusals) > before: return None return _Table( @@ -679,6 +714,7 @@ def _resolve_table( binding=binding, exposure=exposure, region=region, + enforcement=enforcement, ) @@ -1526,6 +1562,8 @@ def emit( if compiled.refusals: raise compiled.refusals[0] _enforce_sovereignty(contract, compiled.tables) + for job, _tables in _jobs_to_pause(compiled, contract): + _pause(compiled, job) return compiled.resources def emit_data( @@ -1605,9 +1643,94 @@ def suppressed_drift( hides, in the shape the engine reports (``_report_suppressed_drift``).""" return _dbt_column_drift(_compile(contract, os.environ), prior_resources) + def standalone_module_refusal( + self, contract: Mapping[str, Any] + ) -> Optional[UnsupportedBindingError]: + """Why this contract's module must not be applied on its own, or ``None``. + + Column masks and row filters are not in the module: they are bound on the SQL + warehouse around ``tofu apply`` (``pre_apply`` / ``post_apply``). A module written for + review and a plain ``tofu apply`` would create the table and grant SELECT with none of + them. ``fluid generate iac`` asks this before it writes one; ``emit`` stays pure, and + ``fluid apply``, ``plan`` and ``validate`` are unaffected. A contract that does not + compile is left to ``emit``, which raises its own refusal. + """ + compiled = _compile(contract, os.environ) + if compiled.refusals: + return None + guarded = [t for t in compiled.tables if t.enforcement is not None] + if not guarded: + return None + return _refusal( + "enforcement-outside-module", + f"{', '.join(t.where for t in guarded)} declare column restrictions or row filters, " + "which Unity Catalog enforces through functions bound on the SQL warehouse by " + "fluid apply, around tofu apply. They are not in the module generate iac writes, " + "and a plain tofu apply of it would create the table and grant SELECT with no mask " + "or row filter, exposing unmasked data to the principals the contract restricts.", + "Use fluid apply, which binds the masks and row filters before it grants and " + "revokes access when it cannot (fluid apply --mode dry-run shows the statements).", + "Or remove the column restrictions and row filters from the contract.", + ) + + @staticmethod + def _enforcement_targets(contract: Mapping[str, Any]) -> List[Target]: + compiled = _compile(contract, os.environ) + if compiled.refusals: + raise compiled.refusals[0] + return [Target(table.full_name, table.enforcement) for table in compiled.tables] + + def pre_apply( + self, + contract: Mapping[str, Any], + say: Optional[Callable[[str], None]] = None, + dry_run: bool = False, + client_factory: Optional[Callable[[], Any]] = None, + ) -> None: + """Before ``tofu apply``: bind the contract's column masks and row filter on the + tables that already exist (``databricks_enforce.pre_apply``), so a grant the apply + adds lands on a protected table. With ``dry_run`` it prints every statement either + hook would run, and runs none. Raises :class:`UnsupportedBindingError` to stop the + run before the apply.""" + targets = self._enforcement_targets(contract) + say = say or print + if dry_run: + preview(targets, say, client_factory) + else: + pre_apply(targets, say, client_factory) + + def post_apply( + self, + contract: Mapping[str, Any], + say: Optional[Callable[[str], None]] = None, + client_factory: Optional[Callable[[], Any]] = None, + after_failed_apply: bool = False, + ) -> None: + """After ``tofu apply``: bind on the tables it created, drop what the contract no + longer restricts, and revoke ``SELECT`` from the restricted principals of any table + it could not bind (``databricks_enforce.post_apply``). ``after_failed_apply`` is the + bind-only mode for an apply that failed part way: it drops nothing and skips a table + that was never created.""" + post_apply( + self._enforcement_targets(contract), + say or print, + client_factory, + after_failed_apply=after_failed_apply, + ) + def plan_notices(self, contract: Mapping[str, Any]) -> List[str]: - """What the apply cannot do itself and someone must: ``USE CATALOG``.""" - return _use_catalog_notices(_compile(contract, os.environ)) + """What the apply cannot do itself and someone must: ``USE CATALOG``; and the dbt jobs + it creates paused until the tables they write are protected.""" + compiled = _compile(contract, os.environ) + notices = _use_catalog_notices(compiled) + for job, tables in _jobs_to_pause(compiled, contract): + notices.append( + f"job {job.name!r} is created paused (pause_status PAUSED): it writes " + f"{', '.join(sorted(tables))}, whose masks and row filter are not bound yet, and " + "a run before that would fill an unprotected table. The next fluid apply, once " + "they are bound, starts its schedule; an explicit run-now works meanwhile." + ) + return notices def explain_plan_failure( self, @@ -1719,6 +1842,59 @@ def _column_details( return out +def _jobs_to_pause( + compiled: _Compiled, contract: Mapping[str, Any] +) -> List[Tuple[DbtJob, Dict[str, str]]]: + """``(job, {table: why})`` for each dbt job that writes a table that is not yet protected. + + The masks and row filters are bound around ``tofu apply``, not by it, so a job created in + the same apply as its table and the grant on it could run (continuous, or a frequent cron) + and fill the table before ``post_apply`` binds them. Such a job is created paused and + started by the next apply, once the live bindings match. A manual job has nothing to pause. + Read live, only for a contract with both a restricted table and a job; unreadable counts as + not protected. + """ + from ...util.contract import builds_writing_expose + + guarded = [t for t in compiled.tables if t.enforcement is not None] + if not guarded or not compiled.jobs: + return [] + builds = [b for b in contract.get("builds") or [] if isinstance(b, Mapping)] + writes = { + t.full_name: { + str(b.get("id")) for b in builds_writing_expose(t.exposure.get("exposeId"), builds) + } + for t in guarded + } + candidates = [ + job + for job in compiled.jobs + if job.trigger != "manual" and any(job.build_id in ids for ids in writes.values()) + ] + if not candidates: + return [] + open_tables = unprotected([Target(t.full_name, t.enforcement) for t in guarded]) + out: List[Tuple[DbtJob, Dict[str, str]]] = [] + for job in candidates: + why = { + name: reason + for name, reason in open_tables.items() + if job.build_id in writes.get(name, ()) + } + if why: + out.append((job, why)) + return out + + +def _pause(compiled: _Compiled, job: DbtJob) -> None: + """``pause_status: PAUSED`` on the job's schedule or continuous block.""" + key = job.address.split(".", 1)[1] + body = compiled.resources.get("databricks_job", {}).get(key) or {} + for block in ("schedule", "continuous"): + if isinstance(body.get(block), dict): + body[block]["pause_status"] = "PAUSED" + + def _job_actions(jobs: Sequence[DbtJob]) -> List[Dict[str, Any]]: """Plan actions for the dbt jobs and the project files they upload.""" actions: List[Dict[str, Any]] = [] @@ -1848,6 +2024,13 @@ class ManagedTable: #: ``databricks_entity_tag_assignment`` resources), sorted by key. tags: Tuple[Tuple[str, str], ...] = () schema_tags: Tuple[Tuple[str, str], ...] = () + #: The row filter Unity Catalog must have bound: ``(function full name, input columns)``. + row_filter: Optional[Tuple[str, Tuple[str, ...]]] = None + #: ``(column, function full name)`` per column mask it must have bound. + masks: Tuple[Tuple[str, str], ...] = () + #: The whole expectation, with each function's definition: what ``fluid verify`` checks + #: a bound function's owner, body, parameters and return type against. + enforcement: Optional[TableEnforcement] = None def managed_objects( @@ -1880,6 +2063,19 @@ def managed_objects( schema_tags=tuple( sorted(tagged.get(("schemas", f"{table.catalog}.{table.schema}"), ())) ), + row_filter=( + ( + table.enforcement.row_filter.function.full_name, + table.enforcement.row_filter.columns, + ) + if table.enforcement and table.enforcement.row_filter + else None + ), + masks=tuple( + (mask.column, mask.function.full_name) + for mask in (table.enforcement.masks if table.enforcement else ()) + ), + enforcement=table.enforcement, ) for table in compiled.tables ] diff --git a/fluid_build/iac/providers/databricks_access.py b/fluid_build/iac/providers/databricks_access.py new file mode 100644 index 000000000..45b40e000 --- /dev/null +++ b/fluid_build/iac/providers/databricks_access.py @@ -0,0 +1,635 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Column masks and row filters for a Databricks table: the SQL, and nothing that runs it. + +``exposes[].policy.authz.columnRestrictions`` and ``rowFilters`` are enforced by Unity +Catalog itself: a SQL function bound on the table, so a query by a restricted principal +is rewritten by the engine, whichever tool sends it. The semantics are the ones GCP and +AWS give the same fields (``iac/column_access.py``, ``iac/row_access.py``): + +* ``access: deny`` on a column becomes a mask that returns ``CAST(NULL AS )`` for + the principal: Unity Catalog has no column-level grant, so the column stays listed. +* ``access: mask`` returns the value masked, as BigQuery's data policy does: + ``last_four`` is ``XXXXX`` and the last four characters, ``first_four`` the first four + and ``XXXXX``, a value of four characters or fewer is its SHA-256 (hex) instead and a + NULL stays NULL (the AWS emitter's rule, ``aws.py::_athena_mask``); ``nullify`` is NULL. + The two text rules take a string column; ``nullify`` takes any. +* One mask per column (the Unity Catalog limit) combines every restriction on it; a + principal in a deny and a mask gets the deny. Everyone else reads the value. +* One row filter per table (the same limit). A reader no filter names reads every row; + a member of several filtered principals reads the union of their predicates. +* ``access: allow`` is refused: it would blind every reader the allow does not name, + the table's owner and its pipelines included. + +The functions are created once under names that carry a hash of their definition +(``fluid_rf_
_``, ``fluid_mask_
__``). Unity Catalog +refuses to replace a function a live filter or mask uses, so a changed definition is a new +function, bound in place of the old one; the same contract always names the same function, +which is how a second run finds nothing to do and ``fluid verify`` knows what to expect. + +A predicate is contract text that becomes a function body, so it is parsed (sqlglot, +Databricks dialect) as one boolean expression, checked node by node against an allowlist +and written back from the tree, never spliced. What the allowlist holds: comparisons, +``AND`` / ``OR`` / ``NOT``, ``IS [NOT] NULL``, ``IN``, ``BETWEEN``, ``LIKE`` / ``ILIKE``, +``+ - *``, literals, ``CAST``, columns of the table, and the scalar functions +:data:`ALLOWED_FUNCTIONS`: pure and deterministic over the row's own values. Anything that +reads session or other-table state (``current_user``, ``is_member``, a subquery), or that a +reader could use to learn something the row does not hold, is refused. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +from ...providers._sql_safety import reject_control_chars, validate_ident +from ..base import UnsupportedBindingError +from ..column_access import DENY, MASK, MASK_RULES, restrictions_for +from ..principals import DATABRICKS, logical_key, principal_map, resolve_principal +from ..row_access import row_filters_for + +#: Function name prefixes. A live filter or mask whose function carries one was made here, +#: so only those are ever dropped: a filter an admin set by hand is theirs. +ROW_FILTER_PREFIX = "fluid_rf_" +MASK_PREFIX = "fluid_mask_" + +#: The scalar functions a row filter may call, each pure and deterministic over its +#: arguments (sqlglot class names). +ALLOWED_FUNCTIONS = ("lower", "upper", "length", "trim", "coalesce", "abs") + +#: Mask rules, strictest first (nullify and first_four hide more than last_four; the deny above +#: them all hides everything). +MASK_STRICTNESS = ("nullify", "first_four", "last_four") + +_STRING_TYPES = frozenset({"string", "varchar", "char"}) +_MAX_NAME = 128 +_MASK_TEXT = "XXXXX" + +GROUP = "group" +USER = "user" +SERVICE_PRINCIPAL = "serviceAccount" + + +def _refusal(kind: str, message: str, *remediation: str) -> UnsupportedBindingError: + return UnsupportedBindingError(f"databricks-{kind}", message, tuple(remediation)) + + +# ── results ──────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class Function: + """A Unity Catalog SQL function to create, named by its definition.""" + + #: ``catalog.schema.name``, lower-case, unquoted. + full_name: str + #: The ``CREATE FUNCTION IF NOT EXISTS`` statement. + create: str + #: What the function must be, for :func:`fluid_build.iac.providers.databricks_enforce. + #: function_problem` to check a live one against: ``((name, SQL type), ...)``, the return + #: type and the body (the expression after ``RETURN``). + params: Tuple[Tuple[str, str], ...] = () + returns: str = "" + body: str = "" + + +@dataclass(frozen=True) +class RowFilterSpec: + function: Function + #: The columns the function takes, in parameter order: ``ON (...)``. + columns: Tuple[str, ...] + bind: str + + +@dataclass(frozen=True) +class MaskSpec: + column: str + function: Function + bind: str + + +@dataclass(frozen=True) +class TableEnforcement: + """What a table's contract says Unity Catalog must enforce on it.""" + + full_name: str + row_filter: Optional[RowFilterSpec] + masks: Tuple[MaskSpec, ...] + #: Every identity a restriction names: the ones a deny-all filter blocks, and the ones to + #: revoke ``SELECT`` from, if a binding cannot be made. + restricted: Tuple[str, ...] + #: The fixed-purpose row filter that blocks every restricted identity from every row, + #: whatever grant gives them the table: the zero-column function ``RETURN NOT ``. Bound when the real filter or a mask cannot be (a failed bind), and + #: as an interim filter while the real one waits for a column. ``None`` with no identity. + deny_all: Optional[RowFilterSpec] = None + + +# ── SQL helpers ──────────────────────────────────────────────────────── + + +def quote(name: str) -> str: + """A plain identifier, backtick-quoted (``validate_ident`` first: nothing else gets in).""" + return "`" + validate_ident(name) + "`" + + +def quote_principal(name: str) -> str: + """A principal name as a quoted identifier (a user's email is no plain identifier).""" + reject_control_chars(name, "principal") + return "`" + name.replace("`", "``") + "`" + + +def quote_table(full_name: str) -> str: + return ".".join(quote(part) for part in full_name.split(".")) + + +def revoke_statements(full_name: str, principals: Sequence[str]) -> List[str]: + """``REVOKE SELECT`` on the table from each principal, the fail-safe of a failed binding.""" + return [ + f"REVOKE SELECT ON TABLE {quote_table(full_name)} FROM {quote_principal(p)}" + for p in principals + ] + + +def revoke_all_statement(full_name: str, principal: str) -> str: + """``REVOKE ALL PRIVILEGES`` on the table, for a grantee holding ALL_PRIVILEGES (a revoke + of SELECT alone does not take that away).""" + return ( + f"REVOKE ALL PRIVILEGES ON TABLE {quote_table(full_name)} FROM {quote_principal(principal)}" + ) + + +def unbind_row_filter(full_name: str) -> str: + return f"ALTER TABLE {quote_table(full_name)} DROP ROW FILTER" + + +def unbind_mask(full_name: str, column: str) -> str: + return f"ALTER TABLE {quote_table(full_name)} ALTER COLUMN {quote(column)} DROP MASK" + + +def is_forge_function(function_name: str, prefix: str) -> bool: + """Whether a live function (full name, any quoting or case) was made by this module.""" + last = str(function_name or "").replace("`", "").rsplit(".", 1)[-1].lower() + return last.startswith(prefix) + + +def same_function(live: Any, expected: str) -> bool: + return str(live or "").replace("`", "").lower() == expected.lower() + + +def _digest(definition: str) -> str: + return hashlib.sha256(definition.encode("utf-8")).hexdigest()[:8] + + +def _function_name(prefix: str, stem: str, definition: str) -> str: + stem = stem[: _MAX_NAME - len(prefix) - 9] + return f"{prefix}{stem}_{_digest(definition)}" + + +def _function( + catalog: str, + schema: str, + prefix: str, + stem: str, + params: Sequence[Tuple[str, str]], + returns: str, + body: str, +) -> Function: + listed = ", ".join(f"{quote(name)} {sql_type}" for name, sql_type in params) + definition = f"({listed}) RETURNS {returns} RETURN {body}" + name = _function_name(prefix, stem, definition) + full = f"{catalog}.{schema}.{name}" + return Function( + full_name=full, + create=f"CREATE FUNCTION IF NOT EXISTS {quote_table(full)}{definition}", + params=tuple((n, t) for n, t in params), + returns=returns, + body=body, + ) + + +# ── principals ───────────────────────────────────────────────────────── + + +def _check_principal(principal: str, at: str) -> None: + """Refuse a principal no Unity Catalog identity can stand for: no prefix, or ``domain:``. + + The prefix is not used to choose how the identity is tested: ``binding.principals`` may + map a ``group:`` principal to a service principal's application ID or to a user's email, + and a test of the wrong kind would match no one while the grant to that identity still + applies, which fails open. Every identity is tested as a group and as a user + (:func:`_member_sql`). + """ + head, sep, rest = logical_key(principal).partition(":") + if sep and rest.strip() and head in (GROUP, USER, SERVICE_PRINCIPAL): + return + raise _refusal( + "principal-kind", + f"{at} names principal {principal!r}, which is not a group:, user: or serviceAccount: " + "principal, so there is no Unity Catalog identity to test for.", + "Write the principal as group:, user: or serviceAccount:, and map " + "it in binding.principals to the Databricks identity.", + ) + + +def string_literal(value: str) -> str: + """``value`` as a Databricks SQL string literal. + + Spark SQL escapes with a backslash and has no ``''``: ``'it''s'`` is the two literals + ``'it'`` and ``'s'``, which concatenate to ``its``, so the doubling + ``_sql_safety.quote_string_literal`` does would silently turn ``o'brien@x.com`` into a + name no one has, and the restriction would never match. Backslash first, then the quote; + a control character is refused. + """ + reject_control_chars(value, "string literal") + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" + + +def _member_sql(identity: str) -> str: + """True when the session user is ``identity``, whatever it is. + + A group, a user and a service principal are all tested: ``is_account_group_member`` for the + name as a group, ``current_user()`` for it as a user's email or a service principal's + application ID. A name that is both a group and a user is restricted either way, which errs + closed. + """ + reject_control_chars(identity, "principal") + literal = string_literal(identity) + return f"is_account_group_member({literal}) OR current_user() = {literal}" + + +class _Resolver: + """Logical principals to their Unity Catalog identities, through ``binding.principals``.""" + + def __init__(self, binding: Mapping[str, Any], where: str): + self._map = principal_map(binding) + self._where = where + + def __call__(self, principal: str, at: str) -> Tuple[str, ...]: + _check_principal(principal, at) + return resolve_principal(principal, self._map, platform=DATABRICKS, where=self._where) + + +def _any_member(identities: Sequence[str]) -> str: + terms = sorted({_member_sql(identity) for identity in identities}) + return "(" + " OR ".join(terms) + ")" + + +# ── predicates ───────────────────────────────────────────────────────── + + +def render_predicate( + where_text: str, + columns: Mapping[str, str], + at: str, + types: Optional[Mapping[str, str]] = None, +) -> Tuple[str, Tuple[str, ...]]: + """``(databricks SQL, columns read)`` for a predicate, or a ``row-filter-invalid`` refusal. + + ``columns`` is ``{lower-case name: table column name}`` and ``types`` ``{column name: + type}``, which a predicate that is a lone column must have as BOOLEAN. The SQL is written from the + parsed tree with every column back-quoted, then parsed again: it must give the same + text, or it is refused. + """ + import sqlglot + from sqlglot import expressions as exp + + def invalid(why: str) -> UnsupportedBindingError: + return _refusal( + "row-filter-invalid", + f"{at}.where {why}", + "Write one boolean expression over the table's columns, e.g. " + "consent_data_analytics = true AND (is_suppressed IS NULL OR is_suppressed = false).", + ) + + text = where_text.strip() + for token in (";", "--", "/*", "*/"): + if token in text: + raise invalid( + f"holds {token!r}: a statement separator or a comment. A row filter is one " + "boolean expression and nothing else." + ) + if any(ord(ch) < 32 or ord(ch) == 127 for ch in text): + raise invalid("holds a control character.") + try: + statements = sqlglot.parse(text, read="databricks") + except sqlglot.errors.SqlglotError as exc: + raise invalid(f"does not parse as Databricks SQL: {str(exc).splitlines()[0]}") from exc + if len(statements) != 1 or statements[0] is None: + raise invalid("is not exactly one expression.") + tree = statements[0] + + functions = { + getattr(exp, name.capitalize()) + for name in ALLOWED_FUNCTIONS + if hasattr(exp, name.capitalize()) + } + allowed = { + exp.And, exp.Or, exp.Not, exp.Paren, exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, + exp.NullSafeEQ, exp.NullSafeNEQ, exp.Is, exp.Null, exp.Boolean, exp.Literal, exp.Column, + exp.Identifier, exp.In, exp.Between, exp.Like, exp.ILike, exp.Neg, exp.Add, exp.Sub, + exp.Mul, exp.Cast, exp.DataType, exp.DataTypeParam, *functions, + } # fmt: skip + for node in tree.walk(): + if type(node) not in allowed: + raise invalid( + f"uses {type(node).__name__} ({node.sql(dialect='databricks')[:40]!r}), which a " + "row filter may not: a subquery, a statement, a session function such as " + "current_user() or is_member(), or a function outside " + f"{', '.join(ALLOWED_FUNCTIONS)}." + ) + inner = tree + while isinstance(inner, exp.Paren): + inner = inner.this + boolean = ( + exp.And, exp.Or, exp.Not, exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, + exp.NullSafeEQ, exp.NullSafeNEQ, exp.Is, exp.In, exp.Between, exp.Like, exp.ILike, + exp.Boolean, exp.Column, + ) # fmt: skip + if not isinstance(inner, boolean): + raise invalid("is not a boolean expression.") + if isinstance(inner, exp.Column) and types is not None: + canonical = columns.get(inner.name.lower()) + if canonical is not None and str(types.get(canonical, "")).lower() != "boolean": + raise invalid(f"is the column {inner.name!r}, which is not BOOLEAN.") + + used: Dict[str, None] = {} + + def requote(node: Any) -> Any: + if not isinstance(node, exp.Column): + return node + if node.table or node.db or node.args.get("catalog"): + raise invalid(f"qualifies column {node.name!r}; name the table's columns alone.") + canonical = columns.get(node.name.lower()) + if canonical is None: + raise invalid(f"reads {node.name!r}, which the table's schema does not declare.") + used.setdefault(canonical, None) + return exp.column(canonical, quoted=True) + + tree = tree.transform(requote) + if not used: + raise invalid("reads no column, so it would keep or hide every row for everyone.") + rendered = tree.sql(dialect="databricks") + again = sqlglot.parse(rendered, read="databricks") + if len(again) != 1 or again[0] is None or again[0].sql(dialect="databricks") != rendered: + raise invalid("does not render back to the same expression, so it is refused.") + return rendered, tuple(used) + + +# ── the table ────────────────────────────────────────────────────────── + + +def _mask_value(rule: str, column_type: str) -> str: + """The masked value of ``val``, for a rule (see the module docstring).""" + if rule == "nullify": + return f"CAST(NULL AS {column_type.upper()})" + shown = ( + f"concat('{_MASK_TEXT}', right(`val`, 4))" + if rule == "last_four" + else (f"concat(left(`val`, 4), '{_MASK_TEXT}')") + ) + return ( + "CASE WHEN `val` IS NULL THEN CAST(NULL AS STRING) " + f"WHEN length(`val`) <= 4 THEN sha2(`val`, 256) ELSE {shown} END" + ) + + +def compile_enforcement( + *, + where: str, + index: int, + exposure: Mapping[str, Any], + binding: Mapping[str, Any], + catalog: str, + schema: str, + table: str, + columns: Sequence[Mapping[str, Any]], +) -> Optional[TableEnforcement]: + """The expose's mask and row-filter enforcement, ``None`` if it declares none. + + Raises :class:`UnsupportedBindingError` (``databricks-...``) for what Unity Catalog + cannot express; it never drops a restriction. + """ + full_name = f"{catalog}.{schema}.{table}" + types = {str(c["name"]): str(c["type"]) for c in columns} + lower = {name.lower(): name for name in types} + resolve = _Resolver(binding, where) + try: + restrictions = restrictions_for(exposure, index, rule_per_principal=True) + except UnsupportedBindingError as exc: + raise _refusal("column-restriction-invalid", f"{where}: {exc}", *exc.remediation) from exc + try: + filters = row_filters_for(exposure, index) + except UnsupportedBindingError as exc: + raise _refusal("row-filter-invalid", f"{where}: {exc}", *exc.remediation) from exc + except Exception as exc: # noqa: BLE001 — the shared reader's parser can raise its own + # tokenizer errors on text it cannot read (a backslash-escaped quote, say): refused. + raise _refusal( + "row-filter-invalid", + f"{where}: a row filter predicate could not be read as SQL ({type(exc).__name__}).", + "Write each predicate as one boolean expression over the table's columns.", + ) from exc + if not restrictions and not filters: + return None + + restricted: Dict[str, None] = {} + masks = _compile_masks( + where, catalog, schema, table, full_name, types, lower, restrictions, resolve, restricted + ) + row_filter = _compile_row_filter( + where, catalog, schema, table, full_name, types, lower, filters, resolve, restricted + ) + return TableEnforcement( + full_name=full_name, + row_filter=row_filter, + masks=masks, + restricted=tuple(sorted(restricted)), + deny_all=_deny_all(catalog, schema, table, full_name, tuple(sorted(restricted))), + ) + + +def _deny_all( + catalog: str, schema: str, table: str, full_name: str, restricted: Sequence[str] +) -> Optional[RowFilterSpec]: + """The zero-column row filter that hides every row from the restricted identities.""" + if not restricted: + return None + function = _function( + catalog, + schema, + ROW_FILTER_PREFIX, + f"{table}_deny_all", + [], + "BOOLEAN", + f"NOT {_any_member(restricted)}", + ) + return RowFilterSpec( + function=function, + columns=(), + bind=f"ALTER TABLE {quote_table(full_name)} SET ROW FILTER " + f"{quote_table(function.full_name)} ON ()", + ) + + +def _compile_masks( + where: str, + catalog: str, + schema: str, + table: str, + full_name: str, + types: Mapping[str, str], + lower: Mapping[str, str], + restrictions: Sequence[Any], + resolve: _Resolver, + restricted: Dict[str, None], +) -> Tuple[MaskSpec, ...]: + denied: Dict[str, List[str]] = {} + # column -> mask rule -> identities that read it masked by that rule: a rule belongs to + # the principal that names it, not to the column. + masked: Dict[str, Dict[str, List[str]]] = {} + for i, restriction in enumerate(restrictions): + at = f"{where}.policy.authz.columnRestrictions[{i}]" + if restriction.access not in (DENY, MASK) or ( + restriction.access == MASK and restriction.mask not in MASK_RULES + ): + raise _refusal( + "column-allow-unmapped", + f"{at} is access: {restriction.access}. An allow list would hide the column " + "from every reader it does not name, the table's owner and its pipelines " + "included, and Unity Catalog has no column grant to express it.", + "Use access: deny for the principals that must not read the columns, or " + "access: mask to let them read them masked.", + ) + members = resolve(restriction.principal, at) + for identity in members: + restricted.setdefault(identity, None) + for column in restriction.columns: + canonical = lower[column.lower()] + if restriction.access == DENY: + denied.setdefault(canonical, []).extend(members) + else: + masked.setdefault(canonical, {}).setdefault(restriction.mask, []).extend(members) + out: List[MaskSpec] = [] + for column in types: # the table's column order + deny_members = denied.get(column, []) + by_rule = {rule: members for rule, members in masked.get(column, {}).items() if members} + if not (deny_members or by_rule): + continue + column_type = types[column] + for rule in by_rule: + if rule != "nullify" and column_type.lower() not in _STRING_TYPES: + raise _refusal( + "mask-type", + f"{where} masks {column} ({column_type}) by {rule}, which masks text.", + "Mask a string column, or use mask: nullify.", + ) + # Strictest first, so a member of several groups gets the strictest rule that matches: + # the deny (NULL), then nullify (NULL), then first_four, then last_four. + branches: List[str] = [] + if deny_members: + branches.append( + f"WHEN {_any_member(deny_members)} THEN CAST(NULL AS {column_type.upper()})" + ) + for rule in MASK_STRICTNESS: + if rule in by_rule: + branches.append( + f"WHEN {_any_member(by_rule[rule])} THEN {_mask_value(rule, column_type)}" + ) + body = "CASE " + " ".join(branches) + " ELSE `val` END" + function = _function( + catalog, schema, MASK_PREFIX, f"{table}_{column}", [("val", column_type.upper())], + column_type.upper(), body, + ) # fmt: skip + out.append( + MaskSpec( + column=column, + function=function, + bind=( + f"ALTER TABLE {quote_table(full_name)} ALTER COLUMN {quote(column)} " + f"SET MASK {quote_table(function.full_name)}" + ), + ) + ) + return tuple(out) + + +def _compile_row_filter( + where: str, + catalog: str, + schema: str, + table: str, + full_name: str, + types: Mapping[str, str], + lower: Mapping[str, str], + filters: Sequence[Any], + resolve: _Resolver, + restricted: Dict[str, None], +) -> Optional[RowFilterSpec]: + if not filters: + return None + parts: List[Tuple[str, str]] = [] # (membership SQL, predicate SQL) + used: Dict[str, None] = {} + all_members: List[str] = [] + for i, entry in enumerate(filters): + at = f"{where}.policy.authz.rowFilters[{i}]" + members = resolve(entry.principal, at) + if not members: + continue # mapped to no identity on Databricks: nothing to restrict there + sql, columns = render_predicate(entry.where, lower, at, types) + for column in columns: + used.setdefault(column, None) + for identity in members: + restricted.setdefault(identity, None) + all_members.extend(members) + parts.append((_any_member(members), sql)) + if not parts: + return None + ordered = tuple(c for c in types if c in used) # the table's column order + body = " OR ".join( + [f"NOT {_any_member(all_members)}"] + + [f"({member} AND ({predicate}))" for member, predicate in sorted(parts)] + ) + params = [(c, types[c].upper()) for c in ordered] + function = _function(catalog, schema, ROW_FILTER_PREFIX, table, params, "BOOLEAN", body) + return RowFilterSpec( + function=function, + columns=ordered, + bind=( + f"ALTER TABLE {quote_table(full_name)} SET ROW FILTER " + f"{quote_table(function.full_name)} ON ({', '.join(quote(c) for c in ordered)})" + ), + ) + + +__all__ = [ + "ALLOWED_FUNCTIONS", + "MASK_PREFIX", + "ROW_FILTER_PREFIX", + "Function", + "MaskSpec", + "RowFilterSpec", + "TableEnforcement", + "compile_enforcement", + "is_forge_function", + "quote_table", + "render_predicate", + "revoke_all_statement", + "revoke_statements", + "string_literal", + "same_function", + "unbind_mask", + "unbind_row_filter", +] diff --git a/fluid_build/iac/providers/databricks_enforce.py b/fluid_build/iac/providers/databricks_enforce.py new file mode 100644 index 000000000..83fbc8f23 --- /dev/null +++ b/fluid_build/iac/providers/databricks_enforce.py @@ -0,0 +1,1038 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run a table's column masks and row filter on the SQL warehouse: read, diff, bind, fail safe. + +The SQL itself comes from :mod:`databricks_access`. The ``databricks/databricks`` provider +(1.140.0) has no function resource and no mask or filter attribute on +``databricks_sql_table`` (terraform-provider-databricks#4550), so these statements run on the +warehouse ``DATABRICKS_WAREHOUSE_ID`` names, through the SQL Statement Execution API, outside +OpenTofu, in two hooks ``fluid apply`` calls around ``tofu apply``: + +* :func:`pre_apply` binds what the contract asks on every table that ALREADY exists, before + ``tofu apply`` runs, so a grant the apply adds lands on a table that is already protected. + A statement that needs a column the apply has yet to add waits for :func:`post_apply`. +* :func:`post_apply` binds whatever is still missing (the tables the apply just created, empty + at that moment), and drops what the contract no longer asks for: a mask or filter whose + function carries this module's name prefix. A filter an admin set by hand is left alone. + Dropping widens access, so it waits for the apply to succeed and is printed as such. + +Both read the live bindings first (``GET /api/2.1/unity-catalog/tables/{name}``: +``row_filter.function_name`` / ``input_column_names`` and ``columns[].mask.function_name``, +databricks-sdk-go v0.190.0 ``catalog/model.go``) and run only what differs, so a second run +executes nothing. + +Fail closed. ``pre_apply`` stops the run before ``tofu apply`` on any failure. When +``post_apply`` cannot bind a table it revokes ``SELECT`` on it from every principal a +restriction names, so a later dbt run cannot expose unmasked rows to them, says exactly what +it revoked and why, and the run exits non-zero; ``fluid apply`` again, once the cause is fixed, +binds first and grants again after. + +The client is ``databricks.sdk.WorkspaceClient`` built from ``sdk_config`` (the credential +policy the provider and the doctor share), used through ``api_client.do``. The SDK is imported +lazily: it is the ``[databricks]`` extra. No message carries a credential. +""" + +from __future__ import annotations + +import os +import time +from dataclasses import dataclass, field +from typing import Any, Callable, Collection, Dict, List, Mapping, Optional, Sequence, Tuple +from urllib.parse import quote + +from ..base import UnsupportedBindingError +from .databricks_access import ( + MASK_PREFIX, + ROW_FILTER_PREFIX, + Function, + TableEnforcement, + is_forge_function, + quote_principal, + quote_table, + revoke_all_statement, + revoke_statements, + same_function, + unbind_mask, + unbind_row_filter, +) +from .databricks_access import ( + quote as quote_ident, +) + +#: The SQL warehouse every statement runs on. +WAREHOUSE_ENV = "DATABRICKS_WAREHOUSE_ID" +DATABRICKS_EXTRA_INSTALL = "pip install 'data-product-forge[databricks]'" + +#: Seconds one statement may take, and how long to wait between polls of it. +STATEMENT_TIMEOUT_SECONDS = 600 +POLL_SECONDS = 1.0 +_MAX_MESSAGE = 400 + +Say = Callable[[str], None] +ClientFactory = Callable[[], Any] + + +class EnforcementError(UnsupportedBindingError): + """A mask or row filter could not be put in place (``databricks-enforcement-failed``).""" + + def __init__(self, message: str, remediation: Sequence[str] = ()): + super().__init__("databricks-enforcement-failed", message, tuple(remediation)) + + +class ForeignBinding(EnforcementError): + """The table has a row filter or mask this module did not create, where the contract asks + for its own: setting ours would silently replace it.""" + + +class StatementFailed(Exception): + """One SQL statement did not succeed; the message is the server's, scrubbed.""" + + +class FunctionMismatch(StatementFailed): + """A function under a content-hashed name is not the one this module would create.""" + + +@dataclass(frozen=True) +class Step: + """One statement, and what it is for.""" + + statement: str + what: str + #: Lower-case columns of the table the statement refers to; it waits until they exist. + needs: Tuple[str, ...] = () + #: Drops a mask or filter: it widens access. + widens: bool = False + #: A step with no statement: read this function back and check it is the expected one + #: (:func:`function_problem`) before anything binds it. + function: Optional[Function] = None + + +@dataclass +class TablePlan: + full_name: str + exists: bool + bind: List[Step] = field(default_factory=list) + drop: List[Step] = field(default_factory=list) + + +# ── client and statements ────────────────────────────────────────────── + + +def default_client() -> Any: + """``WorkspaceClient(**sdk_config(os.environ))``; the SDK imported only here. + + ``sdk_config`` first: it needs no SDK, and a credential problem should be reported as + that, not as a missing extra. + """ + from .databricks_auth import sdk_config + + config = sdk_config(os.environ) + from databricks.sdk import WorkspaceClient + + return WorkspaceClient(**config) + + +def _scrub(text: Any) -> str: + """One short line with every credential value in the environment removed.""" + from ...observability.secret_redactor import redact_secret_text + from .databricks_auth import credential_values + + line = " ".join( + redact_secret_text(str(text), extra_secrets=credential_values(os.environ)).split() + ) + return line if len(line) <= _MAX_MESSAGE else line[: _MAX_MESSAGE - 3] + "..." + + +def _is_not_found(exc: BaseException) -> bool: + for attr in ("status_code", "status", "http_status"): + if getattr(exc, attr, None) == 404: + return True + if getattr(getattr(exc, "response", None), "status_code", None) == 404: + return True + if type(exc).__name__ in {"NotFound", "ResourceDoesNotExist"}: + return True + return str(getattr(exc, "error_code", "") or "") in { + "TABLE_DOES_NOT_EXIST", + "RESOURCE_DOES_NOT_EXIST", + "NOT_FOUND", + } + + +def read_table(client: Any, full_name: str) -> Optional[Dict[str, Any]]: + """The live table, or ``None`` when it does not exist.""" + try: + out = client.api_client.do( + "GET", f"/api/2.1/unity-catalog/tables/{quote(full_name, safe='.')}" + ) + except Exception as exc: # noqa: BLE001 — a 404 is an answer, anything else is raised on + if _is_not_found(exc): + return None + raise + return out if isinstance(out, dict) else {} + + +def read_function(client: Any, full_name: str) -> Optional[Dict[str, Any]]: + """``GET /api/2.1/unity-catalog/functions/{full_name}`` (FunctionInfo), or ``None`` if absent.""" + try: + out = client.api_client.do( + "GET", f"/api/2.1/unity-catalog/functions/{quote(full_name, safe='.')}" + ) + except Exception as exc: # noqa: BLE001 — a 404 is an answer, anything else is raised on + if _is_not_found(exc): + return None + raise + return out if isinstance(out, dict) else {} + + +def deploy_identity(client: Any) -> str: + """The identity this run authenticates as: ``userName`` of ``GET /api/2.0/preview/scim/v2/Me``. + + That is who ``CREATE FUNCTION`` makes the owner (a user's email, a service principal's + application ID), and the credential the provider and this module share, so it is the one + owner a function under a hashed name may have. + """ + me = client.api_client.do("GET", "/api/2.0/preview/scim/v2/Me") + name = str(me.get("userName") or "") if isinstance(me, Mapping) else "" + if not name: + raise StatementFailed("the workspace did not say which identity this run is") + return name + + +def _squash(text: Any) -> str: + return " ".join(str(text or "").split()) + + +def _type(text: Any) -> str: + return "".join(str(text or "").split()).lower() + + +def function_problem( + info: Optional[Mapping[str, Any]], expected: Function, owners: Collection[str] +) -> Optional[str]: + """Why the live function is not ``expected``, or ``None`` when it is. + + A content-hashed name can be computed from the contract, so anyone who can create a + function in the schema can create one under it first, and ``CREATE FUNCTION IF NOT + EXISTS`` would keep theirs. Owner, body, parameters and return type must all be the + expected ones, whitespace-normalised (FunctionInfo: ``owner``, ``routine_definition``, + ``input_params.parameters[]``, ``full_data_type``; databricks-sdk-go v0.190.0). + """ + if info is None: + return f"{expected.full_name} does not exist" + problems: List[str] = [] + owner = str(info.get("owner") or "") + if owner.lower() not in {o.lower() for o in owners}: + problems.append(f"its owner is {owner or 'unknown'}, not {' or '.join(sorted(owners))}") + body_language = str(info.get("routine_body") or "SQL").upper() + if body_language != "SQL": + problems.append(f"it is a {body_language} function, not SQL") + if _squash(info.get("routine_definition")) != _squash(expected.body): + problems.append( + f"its body is {_squash(info.get('routine_definition'))!r}, expected " + f"{_squash(expected.body)!r}" + ) + live_params = [ + (str(p.get("name") or "").lower(), _type(p.get("type_text") or p.get("type_name"))) + for p in ((info.get("input_params") or {}).get("parameters") or []) + if isinstance(p, Mapping) + ] + want_params = [(n.lower(), _type(t)) for n, t in expected.params] + if live_params != want_params: + problems.append(f"its parameters are {live_params}, expected {want_params}") + live_returns = _type(info.get("full_data_type") or info.get("data_type")) + if live_returns != _type(expected.returns): + problems.append( + f"it returns {live_returns or 'nothing'}, expected {_type(expected.returns)}" + ) + if not problems: + return None + return f"{expected.full_name} is not the function this module creates: " + "; ".join(problems) + + +class SqlRunner: + """Runs statements on a warehouse through the SQL Statement Execution API.""" + + def __init__( + self, + client: Any, + warehouse_id: str, + *, + timeout: float = STATEMENT_TIMEOUT_SECONDS, + poll: float = POLL_SECONDS, + sleep: Callable[[float], None] = time.sleep, + ): + self._client = client + self._warehouse = warehouse_id + self._timeout = timeout + self._poll = poll + self._sleep = sleep + + def run(self, statement: str) -> None: + """Run one statement to ``SUCCEEDED``; :class:`StatementFailed` otherwise.""" + do = self._client.api_client.do + try: + doc = do( + "POST", + "/api/2.0/sql/statements/", + body={ + "warehouse_id": self._warehouse, + "statement": statement, + "wait_timeout": "30s", + "on_wait_timeout": "CONTINUE", + }, + ) + deadline = time.monotonic() + self._timeout + while self._state(doc) in ("PENDING", "RUNNING"): + if time.monotonic() > deadline: + self._cancel(doc) + raise StatementFailed(f"it did not finish in {self._timeout:.0f}s") + self._sleep(self._poll) + doc = do("GET", f"/api/2.0/sql/statements/{quote(str(doc.get('statement_id')))}") + except StatementFailed: + raise + except Exception as exc: # noqa: BLE001 — every transport failure is the statement's + raise StatementFailed(f"{type(exc).__name__}: {_scrub(exc)}") from exc + state = self._state(doc) + if state != "SUCCEEDED": + error = (doc.get("status") or {}).get("error") or {} + raise StatementFailed( + f"{state}: {_scrub(error.get('message') or 'no message from the server')}" + ) + + @staticmethod + def _state(doc: Any) -> str: + status = doc.get("status") if isinstance(doc, dict) else None + return str((status or {}).get("state") or "UNKNOWN").upper() + + def _cancel(self, doc: Mapping[str, Any]) -> None: + try: + self._client.api_client.do( + "POST", f"/api/2.0/sql/statements/{quote(str(doc.get('statement_id')))}/cancel" + ) + except Exception: # noqa: BLE001, S110 — best effort; the timeout is what is reported + pass + + +# ── the diff ─────────────────────────────────────────────────────────── + + +def _live_columns(live: Mapping[str, Any]) -> Dict[str, Mapping[str, Any]]: + return { + str(c["name"]).lower(): c + for c in live.get("columns") or [] + if isinstance(c, Mapping) and c.get("name") + } + + +def _verify_step(function: Function, needs: Tuple[str, ...]) -> Step: + return Step( + "", + f"check that {function.full_name} is the function this module creates (owner, body, " + "parameters, return type)", + needs, + function=function, + ) + + +def _foreign(table: str, kind: str, live: str, column: Optional[str]) -> ForeignBinding: + where = f"{table}.{column}" if column else table + drop = ( + f"ALTER TABLE {quote_table(table)} ALTER COLUMN {quote_ident(column)} DROP MASK" + if column + else unbind_row_filter(table) + ) + contract_field = "policy.authz.columnRestrictions" if column else "policy.authz.rowFilters" + return ForeignBinding( + f"{where} has a {kind}, {live}, that this module did not create (its name has no " + "fluid_ prefix), and the contract asks for its own: binding it would replace " + "that one silently. Nothing was changed.", + ( + f"Remove it ({drop}) so the contract's {kind} takes its place, or move its rule into " + f"the contract ({contract_field}) and remove it, then re-run fluid apply.", + ), + ) + + +def plan_table( + full_name: str, enforcement: Optional[TableEnforcement], live: Optional[Mapping[str, Any]] +) -> TablePlan: + """What differs between what the table must enforce and what it enforces now. + + ``live`` is ``None`` for a table that does not exist yet: everything is to bind after + the apply creates it. Without ``enforcement`` the only work is dropping the mask or filter + an earlier run bound that the contract no longer asks for. + """ + plan = TablePlan(full_name=full_name, exists=live is not None) + live_columns = _live_columns(live or {}) + row_filter = (live or {}).get("row_filter") or {} + live_filter = str(row_filter.get("function_name") or "") if row_filter else "" + if enforcement is not None and enforcement.row_filter is not None: + spec = enforcement.row_filter + live_inputs = [str(c).lower() for c in row_filter.get("input_column_names") or []] + if live is None or not ( + same_function(live_filter, spec.function.full_name) + and live_inputs == [c.lower() for c in spec.columns] + ): + if live_filter and not is_forge_function(live_filter, ROW_FILTER_PREFIX): + raise _foreign(full_name, "row filter", live_filter, None) + needs = tuple(c.lower() for c in spec.columns) + plan.bind.append( + Step( + spec.function.create, + f"create row filter function {spec.function.full_name}", + needs, + ) + ) + plan.bind.append(_verify_step(spec.function, needs)) + plan.bind.append( + Step(spec.bind, f"bind row filter {spec.function.full_name} on {full_name}", needs) + ) + elif live_filter and is_forge_function(live_filter, ROW_FILTER_PREFIX): + plan.drop.append( + Step( + unbind_row_filter(full_name), + f"drop row filter {live_filter} from {full_name}: the contract has no row filter", + widens=True, + ) + ) + wanted = {m.column.lower() for m in (enforcement.masks if enforcement else ())} + for mask in enforcement.masks if enforcement else (): + live_mask = (live_columns.get(mask.column.lower()) or {}).get("mask") or {} + if live is None or not same_function( + live_mask.get("function_name"), mask.function.full_name + ): + live_function = str(live_mask.get("function_name") or "") + if live_function and not is_forge_function(live_function, MASK_PREFIX): + raise _foreign(full_name, "column mask", live_function, mask.column) + needs = (mask.column.lower(),) + plan.bind.append( + Step( + mask.function.create, + f"create mask function {mask.function.full_name}", + needs, + ) + ) + plan.bind.append(_verify_step(mask.function, needs)) + plan.bind.append( + Step( + mask.bind, + f"bind mask {mask.function.full_name} on {full_name}.{mask.column}", + needs, + ) + ) + for name, column in live_columns.items(): + function = str((column.get("mask") or {}).get("function_name") or "") + if function and name not in wanted and is_forge_function(function, MASK_PREFIX): + plan.drop.append( + Step( + unbind_mask(full_name, str(column["name"])), + f"drop mask {function} from {full_name}.{column['name']}: the contract " + "restricts no such column", + widens=True, + ) + ) + return plan + + +def interim_steps( + enforcement: Optional[TableEnforcement], + plan: TablePlan, + live: Optional[Mapping[str, Any]], + full_name: str, +) -> List[Step]: + """The deny-all row filter to bind while the real one waits for a column the apply adds. + + The table exists and holds rows, so until ``post_apply`` binds the real filter the + restricted identities would read every row. The interim filter is the fail-safe's own + (:func:`deny_all_steps`); ``post_apply`` replaces it. None when the real filter can bind now, + or the deny-all one is already bound. + """ + if live is None or enforcement is None: + return [] # a table that does not exist has no rows to hide yet + if enforcement.row_filter is None or enforcement.deny_all is None: + return [] + real = enforcement.row_filter + waiting = [s for s in plan.bind if s.statement == real.bind and not _ready(s, live)] + if not waiting: + return [] + current = ((live or {}).get("row_filter") or {}).get("function_name") + if same_function(current, enforcement.deny_all.function.full_name): + return [] + return deny_all_steps(enforcement, full_name) + + +def _ready(step: Step, live: Optional[Mapping[str, Any]]) -> bool: + columns = _live_columns(live or {}) + return live is not None and all(c in columns for c in step.needs) + + +class Session: + """The client, the warehouse runner and, read once, the identity this run is.""" + + def __init__(self, client: Any, runner: SqlRunner): + self.client = client + self.runner = runner + self._identity: Optional[str] = None + + @property + def identity(self) -> str: + if self._identity is None: + self._identity = deploy_identity(self.client) + return self._identity + + def run(self, step: Step) -> None: + """Run a step: a statement on the warehouse, or the check of a function.""" + if step.function is None: + self.runner.run(step.statement) + return + try: + info = read_function(self.client, step.function.full_name) + problem = function_problem(info, step.function, {self.identity}) + except StatementFailed: + raise + except Exception as exc: # noqa: BLE001 + raise FunctionMismatch( + f"could not read {step.function.full_name}: {type(exc).__name__}: {_scrub(exc)}" + ) from exc + if problem is not None: + raise FunctionMismatch( + f"{problem}. It was not bound: a function someone else made under this name " + "would otherwise decide who reads the table" + ) + + +# ── the hooks ────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class Target: + full_name: str + enforcement: Optional[TableEnforcement] + + +def _client_for( + targets: Sequence[Target], factory: Optional[ClientFactory], say: Say +) -> Optional[Session]: + """A client and runner, or ``None`` (with a note) when only removals could be checked. + + A contract with a restriction needs both, and fails without them; one without can only + be dropping what an earlier apply bound, which is access narrowing left in place, so + an unreachable workspace is a note, not an error. + """ + required = any(t.enforcement is not None for t in targets) + # Without the SDK a contract that restricts nothing has no apply-time use for it, and an + # earlier restriction left in place only narrows access: silent. Any other failure to get a + # client is a note, since a removal could not be checked. + try: + client = (factory or default_client)() + warehouse = str(os.environ.get(WAREHOUSE_ENV) or "").strip() + if not warehouse: + raise EnforcementError( + f"{WAREHOUSE_ENV} is not set, and the masks and row filters run on that warehouse.", + (f"Set {WAREHOUSE_ENV} to the id of a SQL warehouse in the workspace.",), + ) + return Session(client, SqlRunner(client, warehouse)) + except UnsupportedBindingError: + if required: + raise + except ImportError: + if required: + raise EnforcementError( + "databricks-sdk is not installed, and the contract's column restrictions and " + "row filters are applied through it.", + (f"Install the extra: {DATABRICKS_EXTRA_INSTALL}",), + ) from None + return None # no restriction was ever asked for: there is nothing to say + except Exception as exc: # noqa: BLE001 — a bad credential must not become a traceback + if required: + raise EnforcementError( + f"could not create the Databricks client for the masks and row filters: " + f"{type(exc).__name__}: {_scrub(exc)}" + ) from exc + say( + " note: masks and row filters an earlier apply bound were not checked for removal " + "(no Databricks client); any left in place keep restricting access." + ) + return None + + +def _read(client: Any, target: Target) -> Optional[Dict[str, Any]]: + try: + return read_table(client, target.full_name) + except Exception as exc: # noqa: BLE001 + raise EnforcementError( + f"could not read {target.full_name} from Unity Catalog to see which masks and row " + f"filters it has: {type(exc).__name__}: {_scrub(exc)}", + ("Check DATABRICKS_HOST and the credential, then re-run fluid apply.",), + ) from exc + + +def preview( + targets: Sequence[Target], say: Say, client_factory: Optional[ClientFactory] = None +) -> None: + """Print the statements an apply would run, in the order it runs them; run none.""" + if not targets: + return + pair = _client_for(targets, client_factory, say) + if pair is None: + return + client = pair.client + lines: List[Tuple[TablePlan, List[Tuple[str, Step]]]] = [] + for target in targets: + live = _read(client, target) + plan = plan_table(target.full_name, target.enforcement, live) + steps = [ + ("before apply", s) + for s in interim_steps(target.enforcement, plan, live, target.full_name) + ] + steps += [("before apply" if _ready(s, live) else "after apply", s) for s in plan.bind] + steps += [("after apply", s) for s in plan.drop] + if steps: + lines.append((plan, steps)) + if not lines: + return + say("\n Unity Catalog masks and row filters (run on the SQL warehouse, not by OpenTofu):") + for plan, steps in lines: + say(f" {plan.full_name}" + ("" if plan.exists else " (not created yet)")) + for when, step in steps: + note = " [widens access]" if step.widens else "" + say(f" {when}{note}: {step.what}") + if step.statement: + say(f" {step.statement}") + say(" dry-run: none of these statements ran.") + + +def pre_apply( + targets: Sequence[Target], say: Say, client_factory: Optional[ClientFactory] = None +) -> None: + """Bind what the contract asks on the tables that already exist, before ``tofu apply``.""" + wanted = [t for t in targets if t.enforcement is not None] + if not wanted: + return + pair = _client_for(wanted, client_factory, say) + if pair is None: # unreachable: a restriction was declared, so _client_for raised + raise EnforcementError("no Databricks client for the masks and row filters.") + client, runner = pair.client, pair + # Read and plan every table first: a refusal (a binding this module did not create) must + # come before any change, on any table. + work: List[Tuple[Target, List[Step]]] = [] + for target in wanted: + live = _read(client, target) + if live is None: + continue # created by this apply: post_apply binds it, empty + plan = plan_table(target.full_name, target.enforcement, live) + steps = [s for s in plan.bind if _ready(s, live)] + interim = interim_steps(target.enforcement, plan, live, target.full_name) + if interim: + steps = [*interim, *steps] + if steps: + work.append((target, steps)) + for target, steps in work: + say(f"\n {target.full_name}: binding masks and row filter before the apply") + for step in steps: + try: + runner.run(step) + except StatementFailed as exc: + raise EnforcementError( + f"{target.full_name}: {step.what} failed before the apply, so the apply did " + f"not run and nothing was changed after it: {exc}. The statement was: " + f"{step.statement}", + ( + "Fix the cause (the warehouse must be able to run it, and this identity " + "needs CREATE FUNCTION on the schema and to own or MANAGE the table), " + "then re-run fluid apply.", + ), + ) from exc + say(f" done: {step.what}") + + +def post_apply( + targets: Sequence[Target], + say: Say, + client_factory: Optional[ClientFactory] = None, + *, + after_failed_apply: bool = False, +) -> None: + """Bind what is still missing, drop what the contract no longer asks, fail safe. + + ``after_failed_apply`` is the bind-only mode for a ``tofu apply`` that failed part way: a + table it created, its grants and the job that fills it can exist with no mask or filter, and + a scheduled run would expose unmasked rows. It binds what exists, revokes SELECT where it + cannot bind, skips a table that does not exist (expected after a failed apply) and drops + nothing: dropping widens access and waits for an apply that succeeded. + """ + if after_failed_apply: + targets = [t for t in targets if t.enforcement is not None] + if not targets: + return + pair = _client_for(targets, client_factory, say) + if pair is None: + return + client, runner = pair.client, pair + failed: List[Tuple[Target, str, FailSafe]] = [] + for target in targets: + try: + live = _read(client, target) + except EnforcementError as exc: + # A transient 429 or 503 on one table must not leave it granted and unbound, nor + # stop the others from being tried: it is a failed bind for that table. + if target.enforcement is None: + say( + f" note: {target.full_name} could not be read, so its earlier masks and " + "row filter were not checked for removal; any left in place keep restricting." + ) + continue + say(f" FAILED: {exc}") + failed.append((target, str(exc), _fail_safe(runner, target, say))) + continue + if live is None: + if after_failed_apply: + say(f" {target.full_name}: not created by the failed apply; nothing to bind.") + elif target.enforcement is not None: + why = "the table does not exist after the apply" + failed.append((target, why, _fail_safe(runner, target, say))) + continue + try: + plan = plan_table(target.full_name, target.enforcement, live) + except ForeignBinding as exc: + say(f" FAILED: {exc}") + failed.append((target, str(exc), _fail_safe(runner, target, say, live=live))) + continue + if after_failed_apply: + plan.drop = [] + if plan.bind or plan.drop: + say(f"\n {target.full_name}: masks and row filter") + for step in [*plan.bind, *plan.drop]: + if step.widens: + say(f" access widened (no data lost, not gated): {step.what}") + try: + runner.run(step) + except StatementFailed as exc: + reason = f"{step.what} failed: {exc}. The statement was: {step.statement}" + say(f" FAILED: {reason}") + if target.enforcement is not None: + failed.append((target, reason, _fail_safe(runner, target, say, live=live))) + break + say(f" done: {step.what}") + if failed: + names = ", ".join(sorted({t.full_name for t, _, _ in failed})) + raise EnforcementError( + ( + "the apply failed part way, and " + if after_failed_apply + else "the apply succeeded, but " + ) + + f"the masks or row filter of {names} could not be put in place. " + + " ".join(f"{t.full_name}: {why}. {_describe(t, fs)}" for t, why, fs in failed), + ( + "Fix the cause, then re-run fluid apply: it binds the masks and row filter " + "first (replacing a deny-all filter) and grants again afterwards.", + ), + ) + + +@dataclass +class FailSafe: + """What the fail-safe did for one table that could not be bound, and what it could not close.""" + + #: The deny-all row filter is bound: the restricted identities read no rows, by any grant path. + locked: bool = False + #: The contract restricts no identity on Databricks: nothing to bind or revoke. + nothing_to_restrict: bool = False + #: Why the deny-all filter was not bound (it failed, or it would replace a filter not made here). + deny_all_problem: str = "" + revoked: List[str] = field(default_factory=list) + unrevoked: List[str] = field(default_factory=list) + #: Grant paths a table-level REVOKE cannot close, each with what to run. + still_open: List[str] = field(default_factory=list) + #: Whether the inherited grants could be listed at all. + checked: bool = False + #: Whether the table's grants could be listed (else only the restricted identities were revoked). + listed: bool = True + + +_READ = frozenset({"SELECT", "ALL_PRIVILEGES"}) + + +def deny_all_steps(enforcement: TableEnforcement, full_name: str) -> List[Step]: + """Create, check and bind the zero-column deny-all row filter for the restricted identities.""" + spec = enforcement.deny_all + assert spec is not None + return [ + Step( + spec.function.create, f"create deny-all row filter function {spec.function.full_name}" + ), + _verify_step(spec.function, ()), + Step(spec.bind, f"bind deny-all row filter {spec.function.full_name} on {full_name}"), + ] + + +def _list_pages(client: Any, path: str) -> List[Dict[str, Any]]: + """Every privilege assignment the permissions API lists for a securable, following pages.""" + out: List[Dict[str, Any]] = [] + token: Optional[str] = None + for _ in range(50): + page = client.api_client.do("GET", path, query={"page_token": token} if token else None) + out += [a for a in (page.get("privilege_assignments") or []) if isinstance(a, Mapping)] + token = page.get("next_page_token") or None + if not token: + return out + raise StatementFailed("the permission list never ended") + + +def _privileges(entry: Mapping[str, Any]) -> Dict[str, Mapping[str, Any]]: + out: Dict[str, Mapping[str, Any]] = {} + for priv in entry.get("privileges") or []: + detail = priv if isinstance(priv, Mapping) else {"privilege": priv} + if detail.get("privilege"): + out[str(detail["privilege"]).upper()] = detail + return out + + +def unprotected( + targets: Sequence[Target], client_factory: Optional[ClientFactory] = None +) -> Dict[str, str]: + """``{table: why}`` for each table with restrictions that is not yet protected as the + contract expects, read live; every such table when the live state cannot be read. + + Protected means the table exists, its row filter and masks are the expected functions bound + on the expected columns, and each of those functions is the one this module creates (owner, + body, parameters, return type). A dbt job that writes a table that is not yet protected is + created paused (``DatabricksIacPlugin.emit``), so it cannot fill it first. Fails closed: no + SDK, no credential or a failed read all count as not protected. + """ + wanted = [t for t in targets if t.enforcement is not None] + if not wanted: + return {} + quiet: List[str] = [] + try: + session = _client_for(wanted, client_factory, quiet.append) + except Exception as exc: # noqa: BLE001 — unknown is unprotected + return { + t.full_name: f"the live state could not be read ({type(exc).__name__})" for t in wanted + } + if session is None: + return {t.full_name: "the live state could not be read" for t in wanted} + out: Dict[str, str] = {} + for target in wanted: + enforcement = target.enforcement + assert enforcement is not None + try: + live = read_table(session.client, target.full_name) + if live is None: + out[target.full_name] = "the table does not exist yet" + continue + plan = plan_table(target.full_name, enforcement, live) + if plan.bind: + out[target.full_name] = "its masks or row filter are not bound as the contract says" + continue + functions = [m.function for m in enforcement.masks] + if enforcement.row_filter is not None: + functions.append(enforcement.row_filter.function) + for fn in functions: + problem = function_problem( + read_function(session.client, fn.full_name), fn, {session.identity} + ) + if problem is not None: + out[target.full_name] = problem + break + except Exception as exc: # noqa: BLE001 — a failed read is not protection + out[target.full_name] = f"it could not be checked ({type(exc).__name__})" + return out + + +def _fail_safe( + session: Session, target: Target, say: Say, *, live: Optional[Mapping[str, Any]] = None +) -> FailSafe: + """Close the table to the restricted identities after a failed bind, and say exactly how. + + First a deny-all row filter bound for them (:func:`deny_all_steps`): it blocks them whatever + grant path gives them the table, a group they are in, a schema or a catalog grant. If that + cannot be bound (it failed, or the table has a row filter this module did not create, which + it would replace), SELECT is revoked on the table from every grantee but its owner, listed + through the permissions API. A table-level REVOKE cannot close a grant on the schema or the + catalog: those are listed, with what to run. + """ + enforcement = target.enforcement + assert enforcement is not None + result = FailSafe() + table = target.full_name + if not enforcement.restricted or enforcement.deny_all is None: + result.nothing_to_restrict = True + say(f" fail-safe: {table} restricts no identity on Databricks; nothing to close.") + return result + if live is None: + try: + live = read_table(session.client, table) + except Exception: # noqa: BLE001 — unreadable: the filter that is there is unknown + live = None + current = ((live or {}).get("row_filter") or {}).get("function_name") or "" + if live is None: + result.deny_all_problem = "the table could not be read, so a filter on it is unknown" + elif current and not is_forge_function(current, ROW_FILTER_PREFIX): + result.deny_all_problem = ( + f"it would replace {current}, a row filter this module did not create" + ) + else: + say( + f" fail-safe: binding a deny-all row filter on {table} for " + f"{', '.join(enforcement.restricted)}:" + ) + try: + for step in deny_all_steps(enforcement, table): + session.run(step) + except StatementFailed as exc: + result.deny_all_problem = f"it failed: {exc}" + say(f" deny-all row filter {result.deny_all_problem}") + else: + result.locked = True + say(f" done: {enforcement.deny_all.function.full_name} is bound on {table}") + return result + say(f" fail-safe: no deny-all row filter ({result.deny_all_problem}); locking {table}.") + _lock(session, table, live, enforcement, result, say) + return result + + +def _lock( + session: Session, + table: str, + live: Optional[Mapping[str, Any]], + enforcement: TableEnforcement, + result: FailSafe, + say: Say, +) -> None: + """Revoke SELECT (and ALL PRIVILEGES) on the table from every grantee but its owner.""" + owner = str((live or {}).get("owner") or "") + grantees: Dict[str, set] = {} + try: + for entry in _list_pages( + session.client, f"/api/2.1/unity-catalog/permissions/table/{quote(table, safe='.')}" + ): + held = set(_privileges(entry)) & _READ + if entry.get("principal") and held: + grantees[str(entry["principal"])] = held + except Exception as exc: # noqa: BLE001 + result.listed = False + say( + f" could not list the grants on {table} ({type(exc).__name__}): revoking only from " + f"{', '.join(enforcement.restricted)}" + ) + grantees = {p: {"SELECT"} for p in enforcement.restricted} + for principal, held in sorted(grantees.items()): + if owner and principal.lower() == owner.lower(): + continue + statements = [] + if "SELECT" in held: + statements += revoke_statements(table, [principal]) + if "ALL_PRIVILEGES" in held: + statements.append(revoke_all_statement(table, principal)) + for statement in statements: + try: + session.runner.run(statement) + except StatementFailed as exc: + result.unrevoked.append(statement) + say(f" NOT REVOKED: {statement} failed ({exc})") + else: + result.revoked.append(statement) + say(f" revoked: {statement}") + _open_paths(session, table, owner, result) + + +def _open_paths(session: Session, table: str, owner: str, result: FailSafe) -> None: + """What still lets someone read the table: a grant inherited from its schema or catalog.""" + try: + entries = _list_pages( + session.client, + f"/api/2.1/unity-catalog/effective-permissions/table/{quote(table, safe='.')}", + ) + except Exception: # noqa: BLE001 + result.checked = False + return + result.checked = True + for entry in entries: + principal = str(entry.get("principal") or "") + if not principal or (owner and principal.lower() == owner.lower()): + continue + for priv, detail in _privileges(entry).items(): + source = str(detail.get("inherited_from_name") or "") + if priv not in _READ or not source: + continue + kind = str(detail.get("inherited_from_type") or "SCHEMA").upper() + target_sql = ".".join(quote_ident(part) for part in source.split(".")) + verb = "ALL PRIVILEGES" if priv == "ALL_PRIVILEGES" else priv + result.still_open.append( + f"{principal} holds {priv} on {kind} {source}, which a table-level REVOKE cannot " + f"close; run: REVOKE {verb} ON {kind} {target_sql} FROM {quote_principal(principal)}" + ) + + +def _describe(target: Target, fs: FailSafe) -> str: + """The fail-safe, in words: exactly what was done, and every path still open.""" + if fs.nothing_to_restrict: + return "The contract restricts no identity on Databricks, so nothing was bound or revoked." + if fs.locked: + assert target.enforcement is not None and target.enforcement.deny_all is not None + return ( + f"A deny-all row filter ({target.enforcement.deny_all.function.full_name}) is bound: " + f"{', '.join(target.enforcement.restricted)} read no rows of it, whatever grant gives " + "them the table. SELECT was not revoked from anyone." + ) + done = f"The deny-all row filter could not be bound ({fs.deny_all_problem}). " + ( + "SELECT (and ALL PRIVILEGES where held) was revoked on the table from " + + ", ".join(sorted({s.rsplit(" FROM ", 1)[1] for s in fs.revoked})) + + " (every grantee but its owner). " + if fs.revoked + else "Nothing was revoked. " + ) + if not fs.listed: + done += "Its grants could not be listed, so only the restricted identities were revoked. " + if fs.unrevoked: + done += ( + "These could NOT be revoked, run them yourself, now: " + "; ".join(fs.unrevoked) + ". " + ) + if fs.still_open: + done += "Still open: " + " | ".join(fs.still_open) + ". " + elif fs.checked: + done += "No grant inherited from the schema or the catalog was found open. " + else: + done += "Grants inherited from the schema or the catalog could not be checked: check them. " + return done.rstrip() + + +__all__ = [ + "DATABRICKS_EXTRA_INSTALL", + "WAREHOUSE_ENV", + "EnforcementError", + "ForeignBinding", + "FailSafe", + "FunctionMismatch", + "Session", + "SqlRunner", + "StatementFailed", + "Step", + "Target", + "TablePlan", + "deny_all_steps", + "interim_steps", + "deploy_identity", + "function_problem", + "plan_table", + "post_apply", + "pre_apply", + "preview", + "read_function", + "read_table", + "unprotected", +] diff --git a/tests/cli/test_verify_databricks.py b/tests/cli/test_verify_databricks.py index 508cea1c2..efd876f4e 100644 --- a/tests/cli/test_verify_databricks.py +++ b/tests/cli/test_verify_databricks.py @@ -37,6 +37,7 @@ pytestmark = pytest.mark.unit +DEPLOYER = "deployer@example.test" FAKE_TOKEN = "fake-workspace-token" # pragma: allowlist secret CONTRACT: Dict[str, Any] = { "fluidVersion": "0.7.6", @@ -124,6 +125,13 @@ def __init__(self) -> None: # (entity type, full name) -> {tag key: value}, served paginated by ``tag_page_size`` self.tags: Dict[Any, Dict[str, str]] = {} self.tag_page_size = 100 + # full name -> the table's ``row_filter`` as GetTable serves it + self.row_filters: Dict[str, Dict[str, Any]] = {} + # function full name -> FunctionInfo as GET .../functions/{name} serves it + self.functions: Dict[str, Dict[str, Any]] = {} + self.identity = DEPLOYER # ``userName`` of GET .../scim/v2/Me + self.table_owner = "table-owner@example.test" + self.me_fails = False self.empty_tag_page_first = False self.stuck_tag_token = False self.api_client = StubApiClient(self) @@ -142,7 +150,23 @@ def answer(self, method: str, path: str, query: Dict[str, Any]) -> Dict[str, Any full = path.rsplit("/", 1)[1] if full not in self.tables: raise StubHttpError(404, f"Table '{full}' does not exist.") - return {"full_name": full, "columns": self.tables[full]} + table: Dict[str, Any] = { + "full_name": full, + "columns": self.tables[full], + "owner": self.table_owner, + } + if full in self.row_filters: + table["row_filter"] = self.row_filters[full] + return table + if path == "/api/2.0/preview/scim/v2/Me": + if self.me_fails: + raise StubHttpError(500, "scim down") + return {"userName": self.identity} + if path.startswith("/api/2.1/unity-catalog/functions/"): + full = path.rsplit("/", 1)[1] + if full not in self.functions: + raise StubHttpError(404, f"Function '{full}' does not exist.") + return self.functions[full] if path.startswith("/api/2.1/unity-catalog/effective-permissions/catalog/"): principal = query["principal"] privileges = self.catalog_privileges.get(principal, []) @@ -715,6 +739,300 @@ def test_fluid_verify_shows_the_tags_skipped_without_labels( assert "Governance tags (Unity Catalog) ⚪ SKIPPED" in out and code == 0 +# ── Row filter and column masks ───────────────────────────────────────── + + +def restricted_contract(**overrides: Any) -> Dict[str, Any]: + """``orders`` with ``amount`` denied and ``region`` masked to ``analysts``, who also see + only ``eu`` rows (``bi`` is not restricted).""" + document = copy.deepcopy(CONTRACT) + authz = document["exposes"][0]["policy"]["authz"] + authz["columnRestrictions"] = [ + {"principal": "group:analysts", "columns": ["amount"], "access": "deny"}, + { + "principal": "group:analysts", + "columns": ["region"], + "access": "mask", + "mask": "last_four", + }, + ] + authz["rowFilters"] = [ + {"principal": "group:analysts", "where": overrides.get("where", "region = 'eu'")} + ] + return document + + +def expected_enforcement(contract: Dict[str, Any]): + from fluid_build.iac.providers.databricks import managed_objects + + (table,), _ = managed_objects(contract, {}) + return table + + +def function_info(fn: Any, owner: str = DEPLOYER, **changes: Any) -> Dict[str, Any]: + """FunctionInfo for an expected ``Function``, as Unity Catalog serves it.""" + info: Dict[str, Any] = { + "full_name": fn.full_name, + "owner": owner, + "routine_body": "SQL", + "routine_definition": fn.body, + "full_data_type": fn.returns.lower(), + "input_params": { + "parameters": [ + {"name": n, "position": i, "type_text": t.lower()} + for i, (n, t) in enumerate(fn.params) + ] + }, + } + info.update(changes) + return info + + +def bind_all(workspace: StubWorkspace, contract: Dict[str, Any]) -> None: + """What ``fluid apply`` leaves on the table: the contract's own filter and masks, and + their functions, made by the identity that deployed.""" + table = expected_enforcement(contract) + enforcement = table.enforcement + for fn in [enforcement.row_filter.function, *(m.function for m in enforcement.masks)]: + workspace.functions[fn.full_name] = function_info(fn) + function, columns = table.row_filter + workspace.row_filters[TABLE] = {"function_name": function, "input_column_names": list(columns)} + for column, mask in table.masks: + next(c for c in workspace.tables[TABLE] if c["name"] == column)["mask"] = { + "function_name": mask + } + + +def test_the_expected_names_carry_a_hash_of_the_definition(): + table = expected_enforcement(restricted_contract()) + assert table.row_filter[0].startswith("main.sales.fluid_rf_orders_") + assert [c for c, _f in table.masks] == ["amount", "region"] + other = expected_enforcement(restricted_contract(where="region = 'uk'")) + assert other.row_filter[0] != table.row_filter[0] + assert other.masks == table.masks + + +def test_a_table_with_every_expected_binding_passes(workspace): + contract = restricted_contract() + bind_all(workspace, contract) + + result = verify(workspace, contract) + + assert result["status"] == "match" and result["severity"]["level"] == "SUCCESS" + dimension = result["dimensions"]["row_filter_masks"] + assert dimension["status"] == "pass" and "all 3 " in dimension["message"] + + +def test_a_missing_row_filter_fails_with_the_fix_and_is_critical(workspace): + contract = restricted_contract() + bind_all(workspace, contract) + del workspace.row_filters[TABLE] + + result = verify(workspace, contract) + + assert result["status"] == "mismatch" and result["severity"]["level"] == "CRITICAL" + dimension = result["dimensions"]["row_filter_masks"] + assert dimension["status"] == "fail" + (finding,) = dimension["findings"] + assert finding["kind"] == "row_filter" and finding["actual"] is None + assert "has no row filter" in finding["message"] + assert "fluid apply" in dimension["fix"] + assert "has no row filter" in result["severity"]["reason"] + assert dimension["fix"] in " ".join(result["severity"]["actions"]) + + +def test_a_row_filter_bound_to_another_function_fails_and_names_both(workspace): + contract = restricted_contract() + bind_all(workspace, restricted_contract(where="region = 'uk'")) # the old predicate's + + result = verify(workspace, contract) + + (finding,) = [ + f for f in result["dimensions"]["row_filter_masks"]["findings"] if f["kind"] == "row_filter" + ] + assert finding["actual"] != finding["expected"] + assert finding["actual"] in finding["message"] and finding["expected"] in finding["message"] + assert result["severity"]["level"] == "CRITICAL" + + +def test_a_row_filter_reading_other_columns_fails(workspace): + contract = restricted_contract() + bind_all(workspace, contract) + workspace.row_filters[TABLE]["input_column_names"] = ["id"] + + dimension = verify(workspace, contract)["dimensions"]["row_filter_masks"] + + assert ( + dimension["status"] == "fail" and "reads columns id" in dimension["findings"][0]["message"] + ) + + +def test_a_missing_mask_and_a_mask_of_another_function_fail(workspace): + contract = restricted_contract() + bind_all(workspace, contract) + columns = {c["name"]: c for c in workspace.tables[TABLE]} + del columns["amount"]["mask"] + columns["region"]["mask"] = {"function_name": "main.sales.fluid_mask_orders_region_deadbeef"} + + dimension = verify(workspace, contract)["dimensions"]["row_filter_masks"] + + assert dimension["status"] == "fail" and len(dimension["findings"]) == 2 + by_column = {f["column"]: f for f in dimension["findings"]} + assert by_column["amount"]["actual"] is None and "has no mask" in by_column["amount"]["message"] + assert by_column["region"]["actual"].endswith("deadbeef") + + +def test_function_names_compare_without_case_or_backticks(workspace): + contract = restricted_contract() + bind_all(workspace, contract) + workspace.row_filters[TABLE]["function_name"] = ( + "`" + workspace.row_filters[TABLE]["function_name"].upper() + "`" + ) + + assert verify(workspace, contract)["dimensions"]["row_filter_masks"]["status"] == "pass" + + +def test_a_contract_without_restrictions_skips_the_dimension(workspace): + result = verify(workspace) + + assert result["dimensions"]["row_filter_masks"]["status"] == "skip" + assert result["status"] == "match" + + +def _first_function(contract: Dict[str, Any]) -> Any: + return expected_enforcement(contract).enforcement.masks[0].function + + +def test_a_function_someone_else_made_under_the_hashed_name_fails_verify_though_the_name_matches( + workspace, +): + """The name is computable from the contract: ``RETURN val`` under it, bound, passes every + check of names and shows the column unmasked.""" + contract = restricted_contract() + bind_all(workspace, contract) + fn = _first_function(contract) + workspace.functions[fn.full_name] = function_info( + fn, owner="attacker@example.test", routine_definition="`val`" + ) + + result = verify(workspace, contract) + + dimension = result["dimensions"]["row_filter_masks"] + assert dimension["status"] == "fail" and result["severity"]["level"] == "CRITICAL" + (finding,) = dimension["findings"] + assert finding["kind"] == "function" + assert "attacker@example.test" in finding["message"] + assert "its body is '`val`'" in finding["message"] + + +def test_a_function_with_the_right_body_but_another_owner_fails(workspace): + contract = restricted_contract() + bind_all(workspace, contract) + fn = expected_enforcement(contract).enforcement.row_filter.function + workspace.functions[fn.full_name] = function_info(fn, owner="someone@example.test") + + dimension = verify(workspace, contract)["dimensions"]["row_filter_masks"] + + assert dimension["status"] == "fail" + assert "owner is someone@example.test" in dimension["findings"][0]["message"] + + +def test_a_function_owned_by_the_table_owner_passes(workspace): + """Verify may run as someone other than the deployer: the table's owner is its reference too.""" + contract = restricted_contract() + bind_all(workspace, contract) + workspace.identity = "reader@example.test" + for name, info in list(workspace.functions.items()): + workspace.functions[name] = dict(info, owner=workspace.table_owner) + + assert verify(workspace, contract)["dimensions"]["row_filter_masks"]["status"] == "pass" + + +@pytest.mark.parametrize( + "change", + [ + {"full_data_type": "int"}, + {"routine_body": "EXTERNAL"}, + {"input_params": {"parameters": []}}, + {"routine_definition": "`val` IS NOT NULL"}, + ], + ids=["returns", "language", "parameters", "body"], +) +def test_a_function_that_differs_in_return_type_language_parameters_or_body_fails( + workspace, change +): + contract = restricted_contract() + bind_all(workspace, contract) + fn = _first_function(contract) + workspace.functions[fn.full_name] = function_info(fn, **change) + + assert verify(workspace, contract)["dimensions"]["row_filter_masks"]["status"] == "fail" + + +def test_a_body_that_differs_only_in_whitespace_passes(workspace): + contract = restricted_contract() + bind_all(workspace, contract) + fn = _first_function(contract) + spaced = fn.body.replace(" ", " \n ") + workspace.functions[fn.full_name] = function_info(fn, routine_definition=spaced) + + assert verify(workspace, contract)["dimensions"]["row_filter_masks"]["status"] == "pass" + + +def test_a_bound_function_that_does_not_exist_fails(workspace): + contract = restricted_contract() + bind_all(workspace, contract) + fn = _first_function(contract) + del workspace.functions[fn.full_name] + + dimension = verify(workspace, contract)["dimensions"]["row_filter_masks"] + + assert dimension["status"] == "fail" and "does not exist" in dimension["findings"][0]["message"] + + +def test_verify_still_checks_the_function_when_the_identity_cannot_be_read(workspace): + contract = restricted_contract() + bind_all(workspace, contract) + workspace.me_fails = True + for name, info in list(workspace.functions.items()): + workspace.functions[name] = dict(info, owner=workspace.table_owner) + assert verify(workspace, contract)["dimensions"]["row_filter_masks"]["status"] == "pass" + fn = _first_function(contract) + workspace.functions[fn.full_name] = function_info(fn, owner="attacker@example.test") + assert verify(workspace, contract)["dimensions"]["row_filter_masks"]["status"] == "fail" + + +def test_a_binding_the_contract_does_not_ask_for_is_not_a_finding(workspace): + workspace.row_filters[TABLE] = { + "function_name": "main.sales.admin_rows", + "input_column_names": [], + } + + assert verify(workspace)["status"] == "match" + + +def test_fluid_verify_prints_the_missing_binding_and_strict_fails( + tmp_path, monkeypatch, capfd, workspace +): + code, out = _run_verify( + tmp_path, monkeypatch, capfd, workspace, "--strict", contract=restricted_contract() + ) + + assert "Row filter and column masks (Unity Catalog)" in out + assert "main.sales.orders has no row filter" in out + assert "main.sales.orders.amount has no mask" in out + assert "Run fluid apply" in out + assert code != 0 + + +def test_fluid_verify_shows_the_dimension_skipped_without_restrictions( + tmp_path, monkeypatch, capfd, workspace +): + code, out = _run_verify(tmp_path, monkeypatch, capfd, workspace, "--strict") + + assert "Row filter and column masks (Unity Catalog) ⚪ SKIPPED" in out and code == 0 + + # ── The real SDK, against the stand-in workspace ──────────────────────── diff --git a/tests/iac/_databricks_governed.py b/tests/iac/_databricks_governed.py new file mode 100644 index 000000000..7c5abe273 --- /dev/null +++ b/tests/iac/_databricks_governed.py @@ -0,0 +1,81 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Customer-Profile-360-shaped Databricks contract with column restrictions and a row filter, +shared by the SQL, enforcement and stand-in tests.""" + +from __future__ import annotations + +import copy +from typing import Any, Dict, Iterable, List, Optional, Tuple + +ANALYSTS = "group:analysts@northwind.example" +STEWARDS = "group:stewards@northwind.example" +ANALYSTS_GROUP = "fluid-demo-lab-analysts" +STEWARDS_GROUP = "fluid-demo-lab-stewards" + +COLUMNS: List[Tuple[str, str]] = [ + ("customer_id", "string"), + ("msisdn", "string"), + ("email", "string"), + ("score", "double"), + ("region", "string"), + ("consent_data_analytics", "boolean"), + ("is_suppressed", "boolean"), +] +DENIED = ["customer_id", "email", "score"] +CONSENT = "consent_data_analytics = true AND (is_suppressed IS NULL OR is_suppressed = false)" + +DENY = {"principal": ANALYSTS, "columns": DENIED, "access": "deny"} +MASK = {"principal": ANALYSTS, "columns": ["msisdn"], "access": "mask", "mask": "last_four"} +FILTER = {"principal": ANALYSTS, "where": CONSENT} + + +def contract( + restrictions: Optional[Iterable[Dict[str, Any]]] = (DENY, MASK), + filters: Optional[Iterable[Dict[str, Any]]] = (FILTER,), + *, + principals: Optional[Dict[str, Any]] = None, + columns: Iterable[Tuple[str, str]] = COLUMNS, +) -> Dict[str, Any]: + authz: Dict[str, Any] = {"readers": [ANALYSTS, STEWARDS]} + if restrictions: + authz["columnRestrictions"] = [copy.deepcopy(r) for r in restrictions] + if filters: + authz["rowFilters"] = [copy.deepcopy(f) for f in filters] + binding: Dict[str, Any] = { + "platform": "databricks", + "format": "delta_table", + "location": {"database": "main", "schema": "cp360", "table": "profile"}, + "principals": ( + {ANALYSTS: ANALYSTS_GROUP, STEWARDS: STEWARDS_GROUP} + if principals is None + else principals + ), + } + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "cp360.profile", + "name": "Customer Profile 360", + "exposes": [ + { + "exposeId": "profile", + "kind": "table", + "binding": binding, + "policy": {"authz": authz}, + "contract": {"schema": [{"name": n, "type": t} for n, t in columns]}, + } + ], + } diff --git a/tests/iac/_fake_databricks.py b/tests/iac/_fake_databricks.py index 853752f90..661c36440 100644 --- a/tests/iac/_fake_databricks.py +++ b/tests/iac/_fake_databricks.py @@ -21,7 +21,15 @@ update, delete and the paginated list ``fluid verify`` reads; ``tags``, ``tag_page_size``). A small SQL interpreter understands exactly the statements ``databricks_sql_table`` sends (CREATE [OR REPLACE] TABLE, ALTER TABLE ADD / DROP / RENAME / ALTER COLUMN, SET / UNSET TBLPROPERTIES, -CLUSTER BY, COMMENT ON TABLE, DROP TABLE). +CLUSTER BY, COMMENT ON TABLE, DROP TABLE), and the statements forge-cli sends for a contract's +column restrictions and row filters: CREATE FUNCTION IF NOT EXISTS (a SQL function is kept as +sent in ``functions``; CREATE OR REPLACE of one a filter or mask uses is refused, as Unity Catalog +refuses it), ALTER TABLE SET / DROP ROW FILTER, ALTER COLUMN SET / DROP MASK, and REVOKE SELECT. +The bindings are served on the table (``row_filter``, ``columns[].mask``) as Unity Catalog serves +them, and a CREATE OR REPLACE TABLE keeps them (Databricks docs: REPLACE preserves row filters +and column masks). ``fail_on(substring, message)`` makes every statement containing it FAIL, +for the fail-safe tests; ``fail_route(method, path_part)`` answers matching requests with an +error, to fail a ``tofu apply`` part way; ``timeline`` orders statements against permission changes. Any other statement FAILS (state ``FAILED``) and stays visible in ``sql_log``, so a new statement shape shows up as an error, never as a silent no-op. Every other endpoint answers a Databricks-shaped 404 and is recorded in ``unknown``. @@ -62,7 +70,9 @@ from urllib.parse import parse_qs, unquote, urlparse NOT_FOUND = "NOT_FOUND" -_USER = "fake-user@example.test" +#: The identity a run authenticates as (``GET .../scim/v2/Me``), and so a function's owner. +DEPLOY_USER = "fake-user@example.test" +_USER = DEPLOY_USER _ORG_ID = "1234567890123456" _IDENT = r"`?([^`.\s]+)`?" @@ -248,6 +258,13 @@ def _new_table(cols: List[Dict[str, Any]], tail: Dict[str, Any], marker: int) -> } +def _coldef_param(txt: str) -> Dict[str, str]: + m = re.match(r"\s*`?([^`\s]+)`?\s+(.+?)\s*$", txt, re.S) + if not m: + raise ValueError(f"cannot parse function parameter: {txt!r}") + return {"name": m.group(1), "type": m.group(2).strip().lower()} + + def _execute(fake: "FakeDatabricks", sql: str) -> Tuple[bool, str]: """Run one statement against ``fake``; return (ok, error message).""" s = sql.strip().rstrip(";").strip() @@ -270,9 +287,70 @@ def _execute(fake: "FakeDatabricks", sql: str) -> Tuple[bool, str]: tail = _parse_create_tail(s[close + 1 :]) except ValueError as exc: return False, f"fake_databricks: {exc}" - if full in fake.tables and replace: + old = fake.tables.get(full) + if old is not None and replace: fake.replaced.append(full) # evidence: CREATE OR REPLACE hit an existing table - fake.tables[full] = _new_table(cols, tail, fake._next_marker()) + new = _new_table(cols, tail, fake._next_marker()) + if old is not None and replace: + # REPLACE preserves row filters and column masks (Databricks docs). + if old.get("row_filter"): + new["row_filter"] = old["row_filter"] + kept = {c["name"]: c["mask"] for c in old["columns"] if c.get("mask")} + for col in new["columns"]: + if col["name"] in kept: + col["mask"] = kept[col["name"]] + fake.tables[full] = new + return True, "" + + m = re.match( + rf"CREATE\s+(OR\s+REPLACE\s+)?FUNCTION\s+(IF\s+NOT\s+EXISTS\s+)?{_FULL}\s*\(", + s, + re.S | re.I, + ) + if m: + replace, ine, cat, sch, name = m.groups() + full = f"{cat}.{sch}.{name}" + if f"{cat}.{sch}" not in fake.schemas: + return False, f"[SCHEMA_NOT_FOUND] The schema `{cat}`.`{sch}` cannot be found." + close = _balanced(s, m.end() - 1) + returns = re.match(r"\s*RETURNS\s+(.+?)\s+RETURN\s+(.+)$", s[close + 1 :], re.S | re.I) + if close < 0 or not returns: + return False, "[PARSE_SYNTAX_ERROR] CREATE FUNCTION needs RETURNS ... RETURN ..." + if full in fake.functions: + if replace: + if fake.function_in_use(full): + return ( + False, + f"[FUNCTION_IN_USE] {full} is used by a row filter or a column mask", + ) + elif ine: + return True, "" + else: + return False, f"[ROUTINE_ALREADY_EXISTS] {full}" + params = [_coldef_param(p) for p in _split_top(s[m.end() : close])] + fake.functions[full] = { + "params": params, + "returns": returns.group(1).strip(), + "body": returns.group(2).strip(), + "sql": s, + "owner": _USER, # the identity the run authenticates as (GET .../scim/v2/Me) + } + return True, "" + + m = re.match( + rf"REVOKE\s+(ALL\s+PRIVILEGES|\w+)\s+ON\s+TABLE\s+{_FULL}\s+FROM\s+`((?:[^`]|``)+)`\s*$", + s, + re.I, + ) + if m: + priv, cat, sch, name, who = m.groups() + full = f"{cat}.{sch}.{name}" + if full not in fake.tables: + return False, f"[TABLE_OR_VIEW_NOT_FOUND] {full}" + held = fake.grants.setdefault(("table", full), {}) + who = who.replace("``", "`") + gone = "ALL_PRIVILEGES" if priv.upper().startswith("ALL") else priv.upper() + held[who] = [p for p in held.get(who, []) if p != gone] return True, "" m = re.match(rf"DROP\s+TABLE\s+(IF\s+EXISTS\s+)?{_FULL}\s*$", s, re.I) @@ -307,6 +385,64 @@ def _execute(fake: "FakeDatabricks", sql: str) -> Tuple[bool, str]: def find(col: str) -> int: return next((i for i, c in enumerate(cols) if c["name"] == col), -1) + fn = rf"{_IDENT}\.{_IDENT}\.{_IDENT}" + m = re.match(rf"SET\s+ROW\s+FILTER\s+{fn}\s+ON\s*\((.*)\)\s*$", action, re.S | re.I) + if m: + fcat, fsch, fname, inner = m.groups() + func = fake.functions.get(f"{fcat}.{fsch}.{fname}") + if func is None: + return False, f"[ROUTINE_NOT_FOUND] {fcat}.{fsch}.{fname}" + names = _names(inner) if inner.strip() else [] + missing = [c for c in names if find(c) < 0] + if missing: + return False, f"[UNRESOLVED_COLUMN] {missing[0]}" + if len(names) != len(func["params"]) or func["returns"].lower() != "boolean": + return ( + False, + "[INVALID_ROW_FILTER] the function must return BOOLEAN and take one argument per column", + ) + for col, param in zip(names, func["params"], strict=True): + if cols[find(col)]["type"].lower() != param["type"]: + return False, f"[INVALID_ROW_FILTER] {col} is not of type {param['type']}" + t["row_filter"] = { + "function_name": f"{fcat}.{fsch}.{fname}", + "input_column_names": names, + } + return True, "" + + if re.match(r"DROP\s+ROW\s+FILTER\s*$", action, re.I): + if not t.get("row_filter"): + return False, "[ROW_FILTER_NOT_FOUND] the table has no row filter" + t.pop("row_filter") + return True, "" + + m = re.match(rf"ALTER\s+COLUMN\s+`?([^`\s]+)`?\s+SET\s+MASK\s+{fn}\s*$", action, re.I) + if m: + col, fcat, fsch, fname = m.groups() + idx = find(col) + if idx < 0: + return False, f"[UNRESOLVED_COLUMN] {col}" + func = fake.functions.get(f"{fcat}.{fsch}.{fname}") + if func is None: + return False, f"[ROUTINE_NOT_FOUND] {fcat}.{fsch}.{fname}" + ctype = cols[idx]["type"].lower() + if len(func["params"]) != 1 or func["params"][0]["type"] != ctype: + return False, f"[INVALID_COLUMN_MASK] the function must take one {ctype} argument" + if func["returns"].lower() != ctype: + return False, f"[INVALID_COLUMN_MASK] the function must return {ctype}" + cols[idx]["mask"] = {"function_name": f"{fcat}.{fsch}.{fname}"} + return True, "" + + m = re.match(r"ALTER\s+COLUMN\s+`?([^`\s]+)`?\s+DROP\s+MASK\s*$", action, re.I) + if m: + idx = find(m.group(1)) + if idx < 0: + return False, f"[UNRESOLVED_COLUMN] {m.group(1)}" + if not cols[idx].get("mask"): + return False, f"[COLUMN_MASK_NOT_FOUND] {m.group(1)} has no mask" + cols[idx].pop("mask") + return True, "" + m = re.match(r"ADD\s+COLUMN\s+(.*)$", action, re.S | re.I) if m: body = m.group(1).strip() @@ -407,6 +543,48 @@ def find(col: str) -> int: return False, f"fake_databricks: unsupported statement: {s[:200]}" +# ---------------------------------------------------------------------- a client +class _HttpError(Exception): + """What the SDK raises for an error status; ``status_code`` is what forge-cli reads.""" + + def __init__(self, status_code: int, doc: Dict[str, Any]): + super().__init__(doc.get("message") or str(status_code)) + self.status_code = status_code + self.error_code = doc.get("error_code") + + +class _ApiClient: + """``WorkspaceClient.api_client`` over plain HTTP: ``do(method, path, query=, body=)``.""" + + def __init__(self, endpoint: str): + self._endpoint = endpoint + + def do(self, method: str, path: str, query: Any = None, body: Any = None, **_: Any) -> Any: + import urllib.error + import urllib.parse + import urllib.request + + url = self._endpoint + path + ("?" + urllib.parse.urlencode(query) if query else "") + data = json.dumps(body).encode() if body is not None else None + request = urllib.request.Request(url, data=data, method=method) + request.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 + raw = response.read() + except urllib.error.HTTPError as exc: + try: + doc = json.loads(exc.read() or b"{}") + except ValueError: + doc = {} + raise _HttpError(exc.code, doc) from exc + return json.loads(raw) if raw else {} + + +class _Client: + def __init__(self, endpoint: str): + self.api_client = _ApiClient(endpoint) + + # ---------------------------------------------------------------------- the server class FakeDatabricks: def __init__( @@ -439,6 +617,13 @@ def __init__( self.queries: List[Tuple[str, str, str]] = [] # (method, path, raw query) self.unknown: List[Tuple[str, str]] = [] # 404'd because not implemented self.sql_log: List[str] = [] # every SQL statement received + # full name -> {params, returns, body, sql} of each SQL function CREATE FUNCTION made + self.functions: Dict[str, Dict[str, Any]] = {} + # ("sql", statement) and ("perm", full_name) in the order they arrived + self.timeline: List[Tuple[str, str]] = [] + self.failing: List[Tuple[str, str]] = [] # (substring, message): matching statements FAIL + # (method, path substring, status, message): matching requests are answered with an error + self.failing_routes: List[Tuple[str, str, int, str]] = [] self.replaced: List[str] = [] # tables a CREATE OR REPLACE hit while they existed self._marker = 0 self._job_seq = 0 @@ -559,6 +744,30 @@ def create_table( self._ensure_schema(cat, sch) self.tables[full_name] = _new_table(cols, tail, self._next_marker()) + def client(self) -> Any: + """A stand-in for ``databricks.sdk.WorkspaceClient``: only ``api_client.do``, over HTTP.""" + return _Client(self.endpoint) + + def fail_on(self, substring: str, message: str = "[INTERNAL_ERROR] injected failure") -> None: + """Make every later statement containing ``substring`` FAIL with ``message``.""" + self.failing.append((substring, message)) + + def fail_route( + self, method: str, path_part: str, status: int = 400, message: str = "injected failure" + ) -> None: + """Answer every later ``method`` request whose path matches the regex ``path_part`` with + an error (a 4xx, which the provider does not retry), to make a ``tofu apply`` fail part + way, or (a 429 or 503) to make one read fail.""" + self.failing_routes.append((method, path_part, status, message)) + + def function_in_use(self, full_name: str) -> bool: + for t in self.tables.values(): + used = [(t.get("row_filter") or {}).get("function_name")] + used += [(c.get("mask") or {}).get("function_name") for c in t["columns"]] + if full_name in used: + return True + return False + def add_job(self, settings: Dict[str, Any]) -> int: """Seed a job (brownfield) from its settings; returns the new job id.""" with self._lock: @@ -609,6 +818,9 @@ def grant_catalog(self, catalog: str, principal: str, *privileges: str) -> None: def _route( self, method: str, path: str, query: Dict[str, List[str]], body: Dict[str, Any] ) -> Tuple[int, Any]: + for fail_method, fail_path, status, message in self.failing_routes: + if method == fail_method and re.search(fail_path, path): + return _err(status, "INVALID_PARAMETER_VALUE", message) for m, rx, fn in self.routes: if m == method: mt = rx.match(path) @@ -636,6 +848,7 @@ def _register(self) -> None: a("GET", r"/api/2\.1/unity-catalog/schemas/([^/]+)", self._get_schema) a("PATCH", r"/api/2\.1/unity-catalog/schemas/([^/]+)", self._patch_schema) a("DELETE", r"/api/2\.1/unity-catalog/schemas/([^/]+)", self._delete_schema) + a("GET", r"/api/2\.1/unity-catalog/functions/([^/]+)", self._get_function) a("GET", r"/api/2\.1/unity-catalog/tables/([^/]+)", self._get_table) a("DELETE", r"/api/2\.1/unity-catalog/tables/([^/]+)", self._delete_table) a("GET", r"/api/2\.1/unity-catalog/permissions/([a-z_]+)/([^/]+)", self._get_perms) @@ -773,6 +986,61 @@ def _delete_schema(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: self._drop_tags("schemas", full) return 200, {} + # -- functions + def add_function( + self, + full_name: str, + params: Sequence[Tuple[str, str]], + returns: str, + body: str, + owner: str = "attacker@example.test", + ) -> None: + """Seed a SQL function as someone else made it, without a SQL statement (the stand-in's + ``sql_log`` stays as it was): ``params`` is ``[(name, type)]``.""" + with self._lock: + self.functions[full_name] = { + "params": [{"name": n, "type": t.lower()} for n, t in params], + "returns": returns, + "body": body, + "sql": "", + "owner": owner, + } + + def _get_function(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + """FunctionInfo, with the fields databricks-sdk-go v0.190.0 names: ``owner``, + ``routine_definition``, ``input_params.parameters[]`` and ``full_data_type``.""" + full = m.group(1) + fn = self.functions.get(full) + if fn is None: + return _err(404, "FUNCTION_DOES_NOT_EXIST", f"Function '{full}' does not exist.") + cat, sch, name = full.split(".") + returns = fn["returns"].lower() + return 200, { + "name": name, + "catalog_name": cat, + "schema_name": sch, + "full_name": full, + "function_id": f"fid-{full}", + "owner": fn.get("owner"), + "created_by": fn.get("owner"), + "routine_body": "SQL", + "routine_definition": fn["body"], + "parameter_style": "S", + "data_type": re.split(r"[(<]", returns)[0].upper(), + "full_data_type": returns, + "input_params": { + "parameters": [ + { + "name": p["name"], + "position": i, + "type_text": p["type"], + "type_name": re.split(r"[(<]", p["type"])[0].upper(), + } + for i, p in enumerate(fn["params"]) + ] + }, + } + # -- tables def _get_table(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: full = m.group(1) @@ -802,11 +1070,15 @@ def _table_doc(self, full: str) -> Dict[str, Any]: } if c.get("comment") is not None: col["comment"] = c["comment"] + if c.get("mask"): + col["mask"] = dict(c["mask"]) # partition columns are flagged on the columns themselves in the UC REST API if c["name"] in t["partitions"]: col["partition_index"] = t["partitions"].index(c["name"]) cols.append(col) + extra = {"row_filter": t["row_filter"]} if t.get("row_filter") else {} return { + **extra, "name": name, "catalog_name": cat, "schema_name": sch, @@ -922,6 +1194,7 @@ def _get_perms(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: return 200, self._assignments(self.grants.get((m.group(1), m.group(2)), {})) def _patch_perms(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + self.timeline.append(("perm", m.group(2))) g = self.grants.setdefault((m.group(1), m.group(2)), {}) for ch in b.get("changes") or []: cur = set(g.get(ch["principal"], [])) @@ -1086,7 +1359,9 @@ def _get_wh(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: def _exec_sql(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: sql = b.get("statement", "") self.sql_log.append(sql) - ok, err = _execute(self, sql) + self.timeline.append(("sql", sql)) + injected = next((msg for sub, msg in self.failing if sub in sql), None) + ok, err = (False, injected) if injected is not None else _execute(self, sql) sid = f"stmt-{len(self.statements) + 1}" if ok: doc: Dict[str, Any] = { diff --git a/tests/iac/test_iac_databricks.py b/tests/iac/test_iac_databricks.py index b1e527fdc..94d462496 100644 --- a/tests/iac/test_iac_databricks.py +++ b/tests/iac/test_iac_databricks.py @@ -244,11 +244,13 @@ def test_other_clouds_are_untouched(self): {"exposes.0.policy.privacy": {"rowLevelPolicy": {"a": 1}}}, "governance-unmapped", ), - ("row-filters", {"exposes.0.policy.authz.rowFilters": [{"a": 1}]}, "governance-unmapped"), + # rowFilters and columnRestrictions are mapped now (databricks_access.py): a malformed + # entry is refused as one, never dropped. + ("row-filters", {"exposes.0.policy.authz.rowFilters": [{"a": 1}]}, "row-filter-invalid"), ( "column-restr", {"exposes.0.policy.authz.columnRestrictions": [{"a": 1}]}, - "governance-unmapped", + "column-restriction-invalid", ), ( "lf", diff --git a/tests/iac/test_iac_databricks_access.py b/tests/iac/test_iac_databricks_access.py new file mode 100644 index 000000000..91e7b1e2b --- /dev/null +++ b/tests/iac/test_iac_databricks_access.py @@ -0,0 +1,721 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Databricks column masks and row filters: the SQL a contract compiles to. + +Pure: no ``tofu``, no network. ``policy.authz.columnRestrictions`` and ``rowFilters`` become +Unity Catalog SQL functions bound on the table (``databricks_access.py``). The exact SQL is +pinned for a Customer-Profile-360-shaped contract, the semantics are evaluated (DuckDB runs the +function bodies, transpiled, with ``is_account_group_member`` stubbed), the names are shown to +follow the definition, and every shape the mapping cannot express is a refusal. +""" + +from __future__ import annotations + +import hashlib +from typing import Any, Dict, Iterable, List, Tuple + +import pytest + +from fluid_build.iac.base import UnsupportedBindingError +from fluid_build.iac.providers import databricks as dbx +from fluid_build.iac.providers.databricks_access import ( + ALLOWED_FUNCTIONS, + TableEnforcement, + revoke_statements, +) + +pytestmark = [pytest.mark.unit, pytest.mark.databricks] + +from ._databricks_governed import ( # noqa: E402 + ANALYSTS, + ANALYSTS_GROUP, + DENIED, + DENY, + FILTER, + MASK, + STEWARDS, + STEWARDS_GROUP, + contract, +) + +ENV = {dbx.WAREHOUSE_ENV: "wh-0123456789abcdef"} + + +def compiled(doc: Dict[str, Any]) -> TableEnforcement: + result = dbx._compile(doc, ENV) + assert not result.refusals, [str(r) for r in result.refusals] + enforcement = result.tables[0].enforcement + assert enforcement is not None + return enforcement + + +def refusal(doc: Dict[str, Any]) -> UnsupportedBindingError: + result = dbx._compile(doc, ENV) + assert result.refusals, "expected a refusal" + return result.refusals[0] + + +def _sha8(definition: str) -> str: + return hashlib.sha256(definition.encode()).hexdigest()[:8] + + +# ── the exact SQL ─────────────────────────────────────────────────────────── +MEMBER = ( + "(is_account_group_member('fluid-demo-lab-analysts') " + "OR current_user() = 'fluid-demo-lab-analysts')" +) +ROW_BODY = ( + f"NOT {MEMBER} OR ({MEMBER} AND (`consent_data_analytics` = TRUE AND " + "(`is_suppressed` IS NULL OR `is_suppressed` = FALSE)))" +) +ROW_DEFINITION = ( + "(`consent_data_analytics` BOOLEAN, `is_suppressed` BOOLEAN) " + f"RETURNS BOOLEAN RETURN {ROW_BODY}" +) +NULLED = "CASE WHEN {m} THEN CAST(NULL AS {t}) ELSE `val` END" +MSISDN_BODY = ( + f"CASE WHEN {MEMBER} THEN CASE WHEN `val` IS NULL THEN CAST(NULL AS STRING) " + "WHEN length(`val`) <= 4 THEN sha2(`val`, 256) ELSE concat('XXXXX', right(`val`, 4)) END " + "ELSE `val` END" +) + + +def _mask_definition(sql_type: str, body: str) -> str: + return f"(`val` {sql_type}) RETURNS {sql_type} RETURN {body}" + + +def test_a_customer_profile_360_contract_compiles_to_this_sql(): + enforcement = compiled(contract()) + + rf = enforcement.row_filter + assert rf is not None + rf_name = f"main.cp360.fluid_rf_profile_{_sha8(ROW_DEFINITION)}" + assert rf.function.full_name == rf_name + assert rf.function.create == ( + "CREATE FUNCTION IF NOT EXISTS " + f"`main`.`cp360`.`fluid_rf_profile_{_sha8(ROW_DEFINITION)}`{ROW_DEFINITION}" + ) + assert rf.columns == ("consent_data_analytics", "is_suppressed") + assert rf.bind == ( + "ALTER TABLE `main`.`cp360`.`profile` SET ROW FILTER " + f"`main`.`cp360`.`fluid_rf_profile_{_sha8(ROW_DEFINITION)}` " + "ON (`consent_data_analytics`, `is_suppressed`)" + ) + + # The table's column order: customer_id, msisdn, email, score. + assert [m.column for m in enforcement.masks] == ["customer_id", "msisdn", "email", "score"] + expected = { + "customer_id": ("STRING", NULLED.format(m=MEMBER, t="STRING")), + "msisdn": ("STRING", MSISDN_BODY), + "email": ("STRING", NULLED.format(m=MEMBER, t="STRING")), + "score": ("DOUBLE", NULLED.format(m=MEMBER, t="DOUBLE")), + } + for mask in enforcement.masks: + sql_type, body = expected[mask.column] + definition = _mask_definition(sql_type, body) + name = f"fluid_mask_profile_{mask.column}_{_sha8(definition)}" + assert mask.function.full_name == f"main.cp360.{name}" + assert mask.function.create == ( + f"CREATE FUNCTION IF NOT EXISTS `main`.`cp360`.`{name}`{definition}" + ) + assert mask.bind == ( + f"ALTER TABLE `main`.`cp360`.`profile` ALTER COLUMN `{mask.column}` " + f"SET MASK `main`.`cp360`.`{name}`" + ) + assert enforcement.restricted == (ANALYSTS_GROUP,) + + +def test_a_contract_without_restrictions_has_no_enforcement(): + result = dbx._compile(contract(restrictions=None, filters=None), ENV) + assert not result.refusals and result.tables[0].enforcement is None + + +def test_the_expected_bindings_reach_the_managed_table_for_fluid_verify(): + tables, _jobs = dbx.managed_objects(contract(), ENV) + (table,) = tables + enforcement = compiled(contract()) + assert table.row_filter == ( + enforcement.row_filter.function.full_name, + ("consent_data_analytics", "is_suppressed"), + ) + assert table.masks == tuple((m.column, m.function.full_name) for m in enforcement.masks) + bare, _ = dbx.managed_objects(contract(restrictions=None, filters=None), ENV) + assert bare[0].row_filter is None and bare[0].masks == () + + +# ── names follow the definition ───────────────────────────────────────────── +def test_the_same_contract_names_the_same_functions(): + first, second = compiled(contract()), compiled(contract()) + assert first == second + + +def test_a_changed_predicate_is_a_new_row_filter_function_and_leaves_the_masks_alone(): + before = compiled(contract()) + after = compiled(contract(filters=[{**FILTER, "where": "consent_data_analytics = false"}])) + assert after.row_filter.function.full_name != before.row_filter.function.full_name + assert after.row_filter.columns == ("consent_data_analytics",) + assert after.masks == before.masks + + +def test_a_changed_mask_rule_renames_that_columns_function_only(): + before = compiled(contract()) + changed = {**MASK, "mask": "first_four"} + after = compiled(contract(restrictions=[DENY, changed])) + names = lambda e: {m.column: m.function.full_name for m in e.masks} # noqa: E731 + assert names(after)["msisdn"] != names(before)["msisdn"] + for column in DENIED: + assert names(after)[column] == names(before)[column] + assert after.row_filter == before.row_filter + + +def test_a_remapped_principal_is_a_new_function(): + before = compiled(contract()) + after = compiled(contract(principals={ANALYSTS: "other-group", STEWARDS: STEWARDS_GROUP})) + assert after.row_filter.function.full_name != before.row_filter.function.full_name + assert after.restricted == ("other-group",) + + +def test_a_functions_name_ends_in_the_hash_of_its_definition(): + rf = compiled(contract()).row_filter + assert rf.function.full_name.endswith(_sha8(ROW_DEFINITION)) + assert rf.function.full_name.rsplit(".", 1)[-1].startswith("fluid_rf_profile_") + + +# ── the semantics, evaluated ──────────────────────────────────────────────── +duckdb = pytest.importorskip("duckdb") + + +def _body(create: str) -> str: + return create.split(" RETURN ", 1)[1] + + +def _duck(sql: str) -> str: + import sqlglot + + return sqlglot.transpile(sql, read="databricks", write="duckdb")[0] + + +def _connection(groups: Iterable[str]): + con = duckdb.connect() + members = ", ".join("'" + g + "'" for g in groups) + con.execute(f"CREATE MACRO is_account_group_member(g) AS list_contains([{members}], g)") + con.execute("CREATE MACRO sha2(v, n) AS sha256(v)") + return con + + +def _filter_rows(enforcement: TableEnforcement, groups: Iterable[str], rows: List[Tuple]): + spec = enforcement.row_filter + con = _connection(groups) + params = ", ".join(f'"{c}"' for c in spec.columns) + con.execute(f"CREATE MACRO keep({params}) AS {_duck(_body(spec.function.create))}") + return [ + r for r in rows if con.execute(f"SELECT keep({', '.join('?' * len(r))})", r).fetchone()[0] + ] + + +def _masked(enforcement: TableEnforcement, column: str, groups: Iterable[str], value: Any): + spec = next(m for m in enforcement.masks if m.column == column) + con = _connection(groups) + con.execute(f'CREATE MACRO mask("val") AS {_duck(_body(spec.function.create))}') + return con.execute("SELECT mask(?)", [value]).fetchone()[0] + + +def test_the_restricted_group_reads_only_consented_rows_and_everyone_else_every_row(): + enforcement = compiled(contract()) + rows = [(True, None), (True, False), (True, True), (False, None), (None, False), (None, None)] + kept = _filter_rows(enforcement, [ANALYSTS_GROUP], rows) + assert kept == [(True, None), (True, False)] + # A steward (not restricted) and a user in no group read every row. + assert _filter_rows(enforcement, [STEWARDS_GROUP], rows) == rows + assert _filter_rows(enforcement, [], rows) == rows + + +def test_a_member_of_several_filtered_principals_reads_the_union_of_their_predicates(): + doc = contract( + restrictions=None, + filters=[ + {"principal": ANALYSTS, "where": "region = 'eu'"}, + {"principal": STEWARDS, "where": "region = 'us' AND consent_data_analytics = true"}, + ], + ) + enforcement = compiled(doc) + assert enforcement.row_filter.columns == ("region", "consent_data_analytics") + rows = [("eu", True), ("us", True), ("us", False), ("ap", True), (None, True)] + assert _filter_rows(enforcement, [ANALYSTS_GROUP], rows) == [("eu", True)] + assert _filter_rows(enforcement, [STEWARDS_GROUP], rows) == [("us", True)] + assert _filter_rows(enforcement, [ANALYSTS_GROUP, STEWARDS_GROUP], rows) == [ + ("eu", True), + ("us", True), + ] + assert _filter_rows(enforcement, ["somebody-else"], rows) == rows + assert enforcement.restricted == (ANALYSTS_GROUP, STEWARDS_GROUP) + + +@pytest.mark.parametrize( + "value,last_four,first_four", + [ + ("0791234567", "XXXXX4567", "0791XXXXX"), + ("12345", "XXXXX2345", "1234XXXXX"), + ("1234", hashlib.sha256(b"1234").hexdigest(), hashlib.sha256(b"1234").hexdigest()), + ("ab", hashlib.sha256(b"ab").hexdigest(), hashlib.sha256(b"ab").hexdigest()), + ("", hashlib.sha256(b"").hexdigest(), hashlib.sha256(b"").hexdigest()), + (None, None, None), + ], +) +def test_mask_rules_match_the_other_clouds_including_a_short_value(value, last_four, first_four): + for rule, expected in (("last_four", last_four), ("first_four", first_four)): + enforcement = compiled(contract(restrictions=[{**MASK, "mask": rule}], filters=None)) + assert _masked(enforcement, "msisdn", [ANALYSTS_GROUP], value) == expected, rule + # Every other reader sees the value. + assert _masked(enforcement, "msisdn", [STEWARDS_GROUP], value) == value, rule + + +def test_nullify_and_deny_return_null_and_others_read_the_value(): + nullify = compiled(contract(restrictions=[{**MASK, "mask": "nullify"}, DENY], filters=None)) + for column, value in (("msisdn", "0791234567"), ("email", "a@b.c"), ("score", 4.5)): + assert _masked(nullify, column, [ANALYSTS_GROUP], value) is None + assert _masked(nullify, column, [STEWARDS_GROUP], value) == value + + +def test_a_principal_in_a_deny_and_a_mask_gets_the_deny_and_a_mask_only_group_reads_masked(): + both = [ + {"principal": ANALYSTS, "columns": ["msisdn"], "access": "deny"}, + {"principal": ANALYSTS, "columns": ["msisdn"], "access": "mask", "mask": "last_four"}, + {"principal": STEWARDS, "columns": ["msisdn"], "access": "mask", "mask": "last_four"}, + ] + enforcement = compiled(contract(restrictions=both, filters=None)) + assert len(enforcement.masks) == 1, "one mask per column" + assert _masked(enforcement, "msisdn", [ANALYSTS_GROUP], "0791234567") is None + assert _masked(enforcement, "msisdn", [ANALYSTS_GROUP, STEWARDS_GROUP], "0791234567") is None + assert _masked(enforcement, "msisdn", [STEWARDS_GROUP], "0791234567") == "XXXXX4567" + assert _masked(enforcement, "msisdn", [], "0791234567") == "0791234567" + body = enforcement.masks[0].function.create + assert body.index("THEN CAST(NULL AS STRING) WHEN") < body.index("XXXXX"), "deny is first" + + +# ── principals ────────────────────────────────────────────────────────────── +def _both_ways(identity: str) -> str: + return f"is_account_group_member('{identity}') OR current_user() = '{identity}'" + + +def test_every_identity_is_tested_as_a_group_and_as_a_user_whatever_its_prefix(): + users = [ + {"principal": "user:ana@northwind.example", "columns": ["email"], "access": "deny"}, + {"principal": "serviceAccount:etl", "columns": ["score"], "access": "deny"}, + ] + doc = contract( + restrictions=users, + filters=None, + principals={ + "user:ana@northwind.example": "ana@northwind.com", + "serviceAccount:etl": "0f1e2d3c-1111-2222-3333-444455556666", + ANALYSTS: ANALYSTS_GROUP, + STEWARDS: STEWARDS_GROUP, + }, + ) + by_column = {m.column: m.function.create for m in compiled(doc).masks} + assert f"WHEN ({_both_ways('ana@northwind.com')}) THEN" in by_column["email"] + assert f"WHEN ({_both_ways('0f1e2d3c-1111-2222-3333-444455556666')}) THEN" in by_column["score"] + + +@pytest.mark.parametrize( + "identity", ["0f1e2d3c-1111-2222-3333-444455556666", "ana@northwind.com"], ids=["sp", "email"] +) +def test_a_group_principal_mapped_to_a_service_principal_or_a_user_still_matches_them(identity): + """binding.principals may map a ``group:`` principal to a service principal's application + ID or a user's email. The grant to that identity applies, so the restriction must too.""" + doc = contract(principals={ANALYSTS: identity, STEWARDS: STEWARDS_GROUP}) + enforcement = compiled(doc) + test = f"current_user() = '{identity}'" + assert test in enforcement.row_filter.function.create + assert all(test in m.function.create for m in enforcement.masks) + assert enforcement.restricted == (identity,) + + +def test_a_session_user_that_is_a_mapped_identity_and_in_no_group_is_restricted(): + """DuckDB's session user is ``duckdb``: map the group principal to it, and put it in no + group, as a service principal is.""" + doc = contract(principals={ANALYSTS: "duckdb", STEWARDS: STEWARDS_GROUP}) + enforcement = compiled(doc) + assert _masked(enforcement, "msisdn", [], "0791234567") == "XXXXX4567" + assert _masked(enforcement, "email", [], "a@b.c") is None + rows = [(True, None), (False, None), (None, None)] + assert _filter_rows(enforcement, [], rows) == [(True, None)] + # A group of that name restricts a member too: both tests are made. + assert _masked(enforcement, "email", ["duckdb"], "a@b.c") is None + + +def test_a_logical_principal_with_several_identities_matches_any_of_them(): + doc = contract( + restrictions=[DENY], filters=None, principals={ANALYSTS: ["g-one", "g-two"], STEWARDS: "s"} + ) + create = compiled(doc).masks[0].function.create + assert f"({_both_ways('g-one')} OR {_both_ways('g-two')})" in create + assert compiled(doc).restricted == ("g-one", "g-two") + + +def test_a_principal_mapped_to_no_identity_restricts_nothing_on_databricks(): + doc = contract(principals={ANALYSTS: [], STEWARDS: STEWARDS_GROUP}) + enforcement = compiled(doc) + assert enforcement.masks == () and enforcement.row_filter is None + assert enforcement.restricted == () + + +def test_principal_names_are_escaped_string_literals_and_identifiers(): + nasty = "o'brien\\" + doc = contract( + restrictions=[{"principal": "user:o@x.example", "columns": ["email"], "access": "deny"}], + filters=None, + principals={"user:o@x.example": nasty, ANALYSTS: ANALYSTS_GROUP, STEWARDS: "s"}, + ) + create = compiled(doc).masks[0].function.create + assert "current_user() = 'o\\'brien\\\\'" in create + assert "is_account_group_member('o\\'brien\\\\')" in create + # Databricks' own parser reads it back as the name, not as o + brien. + import sqlglot + + literal = sqlglot.parse_one("current_user() = 'o\\'brien\\\\'", read="databricks").expression + assert literal.name == nasty + (revoke,) = revoke_statements("main.cp360.profile", ["we`ird"]) + assert revoke == "REVOKE SELECT ON TABLE `main`.`cp360`.`profile` FROM `we``ird`" + + +def test_revoke_refuses_a_control_character_in_a_principal(): + with pytest.raises(ValueError): + revoke_statements("main.cp360.profile", ["a\nb"]) + + +# ── refusals ──────────────────────────────────────────────────────────────── +def _kinds(doc: Dict[str, Any]) -> str: + return refusal(doc).kind + + +def test_access_allow_is_refused_rather_than_blinding_every_other_reader(): + allow = {"principal": ANALYSTS, "columns": ["email"], "access": "allow"} + assert _kinds(contract(restrictions=[allow])) == "databricks-column-allow-unmapped" + + +@pytest.mark.parametrize( + "restriction,why", + [ + ({**MASK, "mask": "scramble"}, "unknown mask rule"), + ({**MASK, "mask": None}, "mask without a rule"), + ({**DENY, "mask": "last_four"}, "a rule on a deny"), + ({**DENY, "columns": ["nope"]}, "an undeclared column"), + ({**DENY, "access": "block"}, "unknown access"), + ({**DENY, "principal": ""}, "no principal"), + ({**DENY, "columns": []}, "no columns"), + ], +) +def test_a_restriction_the_mapping_cannot_express_is_refused(restriction, why): + error = refusal(contract(restrictions=[restriction])) + assert error.kind == "databricks-column-restriction-invalid", why + + +GUESTS = "group:guests@northwind.example" +GUESTS_GROUP = "fluid-demo-lab-guests" +THREE = {ANALYSTS: ANALYSTS_GROUP, STEWARDS: STEWARDS_GROUP, GUESTS: GUESTS_GROUP} + + +def _by_rule(rules: dict) -> list: + return [ + {"principal": principal, "columns": ["msisdn"], "access": "mask", "mask": rule} + for principal, rule in rules.items() + ] + + +def test_each_principal_gets_the_mask_rule_it_names_not_the_last_ones(): + """A nullify and B last_four on one column: A reads NULL, not the last four characters.""" + restrictions = _by_rule({ANALYSTS: "nullify", STEWARDS: "last_four"}) + enforcement = compiled(contract(restrictions=restrictions, filters=None)) + assert len(enforcement.masks) == 1, "one mask per column" + value = "0791234567" + assert _masked(enforcement, "msisdn", [ANALYSTS_GROUP], value) is None + assert _masked(enforcement, "msisdn", [STEWARDS_GROUP], value) == "XXXXX4567" + assert _masked(enforcement, "msisdn", ["nobody"], value) == value + # A member of both gets the strictest rule that matches. + assert _masked(enforcement, "msisdn", [ANALYSTS_GROUP, STEWARDS_GROUP], value) is None + + +def test_the_same_result_whichever_restriction_is_written_first(): + one = compiled(contract(restrictions=_by_rule({ANALYSTS: "nullify", STEWARDS: "last_four"}))) + two = compiled(contract(restrictions=_by_rule({STEWARDS: "last_four", ANALYSTS: "nullify"}))) + assert one.masks == two.masks + + +def test_mask_branches_are_ordered_strictest_first_deny_nullify_first_four_last_four(): + restrictions = [ + {"principal": GUESTS, "columns": ["msisdn"], "access": "deny"}, + *_by_rule({ANALYSTS: "last_four", STEWARDS: "first_four"}), + {"principal": "group:ops@northwind.example", "columns": ["msisdn"], "access": "mask", + "mask": "nullify"}, + ] # fmt: skip + principals = {**THREE, "group:ops@northwind.example": "fluid-demo-lab-ops"} + enforcement = compiled(contract(restrictions=restrictions, filters=None, principals=principals)) + body = enforcement.masks[0].function.create + deny = body.index(f"WHEN ({_both_ways(GUESTS_GROUP)}) THEN CAST(NULL") + nullify = body.index(f"WHEN ({_both_ways('fluid-demo-lab-ops')}) THEN CAST(NULL") + first = body.index(f"WHEN ({_both_ways(STEWARDS_GROUP)}) THEN CASE") + last = body.index(f"WHEN ({_both_ways(ANALYSTS_GROUP)}) THEN CASE") + assert deny < nullify < first < last + value = "0791234567" + groups = {"deny": GUESTS_GROUP, "nullify": "fluid-demo-lab-ops", "first": STEWARDS_GROUP, + "last": ANALYSTS_GROUP} # fmt: skip + assert _masked(enforcement, "msisdn", [groups["last"]], value) == "XXXXX4567" + assert _masked(enforcement, "msisdn", [groups["first"]], value) == "0791XXXXX" + assert _masked(enforcement, "msisdn", [groups["first"], groups["last"]], value) == "0791XXXXX" + assert _masked(enforcement, "msisdn", [groups["nullify"], groups["first"]], value) is None + assert _masked(enforcement, "msisdn", [groups["deny"], groups["nullify"]], value) is None + assert _masked(enforcement, "msisdn", list(groups.values()), value) is None + + +def test_a_principal_naming_two_rules_for_a_column_gets_the_stricter(): + restrictions = _by_rule({ANALYSTS: "last_four"}) + [ + {"principal": ANALYSTS, "columns": ["msisdn"], "access": "mask", "mask": "first_four"} + ] + enforcement = compiled(contract(restrictions=restrictions, filters=None)) + assert _masked(enforcement, "msisdn", [ANALYSTS_GROUP], "0791234567") == "0791XXXXX" + + +def test_gcp_and_aws_still_refuse_two_masking_rules_for_one_column(): + from fluid_build.iac.column_access import restrictions_for + + exposure = contract(restrictions=_by_rule({ANALYSTS: "nullify", STEWARDS: "last_four"}))[ + "exposes" + ][0] + with pytest.raises(UnsupportedBindingError, match="one masking rule"): + restrictions_for(exposure) + assert len(restrictions_for(exposure, rule_per_principal=True)) == 2 + + +def test_each_text_rule_is_checked_against_the_column_type(): + on_score = [ + {"principal": ANALYSTS, "columns": ["score"], "access": "mask", "mask": "nullify"}, + {"principal": STEWARDS, "columns": ["score"], "access": "mask", "mask": "first_four"}, + ] + error = refusal(contract(restrictions=on_score, filters=None)) + assert error.kind == "databricks-mask-type" and "first_four" in str(error) + assert compiled(contract(restrictions=on_score[:1], filters=None)).masks[0].column == "score" + + +def test_a_text_mask_on_a_non_string_column_is_refused_and_nullify_is_not(): + bad = {"principal": ANALYSTS, "columns": ["score"], "access": "mask", "mask": "last_four"} + assert _kinds(contract(restrictions=[bad])) == "databricks-mask-type" + ok = {**bad, "mask": "nullify"} + assert compiled(contract(restrictions=[ok], filters=None)).masks[0].column == "score" + + +@pytest.mark.parametrize("principal", ["analysts", "domain:northwind.example", "role:analyst"]) +def test_a_principal_with_no_sql_test_is_refused(principal): + doc = contract( + restrictions=[{"principal": principal, "columns": ["email"], "access": "deny"}], + filters=None, + principals={principal: "x", ANALYSTS: ANALYSTS_GROUP, STEWARDS: STEWARDS_GROUP}, + ) + assert _kinds(doc) == "databricks-principal-kind" + + +def test_an_unmapped_principal_is_refused_as_every_databricks_grant_is(): + doc = contract(principals={STEWARDS: STEWARDS_GROUP}) + assert _kinds(doc) == "principal-unmapped" + + +def test_a_placeholder_principal_is_refused_when_nothing_maps_it(): + doc = contract() + doc["exposes"][0]["binding"].pop("principals") + assert _kinds(doc) == "principal-placeholder" + + +def test_unmapped_principals_resolve_as_written_without_a_map(): + doc = contract(restrictions=[DENY], filters=None) + doc["exposes"][0]["binding"].pop("principals") + doc["exposes"][0]["policy"]["authz"]["readers"] = [] + doc["exposes"][0]["policy"]["authz"]["columnRestrictions"][0]["principal"] = "group:analysts" + assert compiled(doc).restricted == ("analysts",) + + +def test_masking_landing_rules_and_binding_governance_are_still_refused(): + doc = contract() + doc["exposes"][0]["policy"]["privacy"] = {"masking": [{"column": "msisdn"}]} + assert _kinds(doc) == "databricks-governance-unmapped" + doc = contract() + doc["exposes"][0]["binding"]["governance"] = {"lakeFormation": {"grants": []}} + assert _kinds(doc) == "databricks-governance-unmapped" + doc = contract() + doc["exposes"][0]["policy"]["privacy"] = {"rowLevelPolicy": {"a": 1}} + assert _kinds(doc) == "databricks-governance-unmapped" + + +# ── predicates ────────────────────────────────────────────────────────────── +INJECTIONS = [ + "region = 'eu'; DROP TABLE main.cp360.profile", + "region = 'eu' OR 1=1; --", + "region = 'eu' -- everyone", + "region = 'eu' /* all */ OR true", + "region = 'a\\' OR 1=1 --'", + "region = (SELECT max(region) FROM other)", + "region IN (SELECT region FROM other)", + "EXISTS (SELECT 1)", + "region = 'eu' UNION SELECT 'x'", + "DROP TABLE x", + "current_user() = 'ana'", + "is_member('admins')", + "is_account_group_member('admins')", + "reflect('java.lang.Runtime', 'exec') = 'x'", + "unknown_column = 1", + "other.region = 'eu'", + "main.cp360.profile.region = 'eu'", + "1 = 1", + "true", + "region", + "'eu'", + "", + " ", + "region = ", + "region = 'eu'\n; DROP TABLE x", + "region = 'eu'\x00", + "*", + "region = 'eu' AND hash(region) = 1", + "region = ${jndi}", + # The shared reader's tokenizer cannot read a backslash-escaped quote: refused, no traceback. + "region = 'it\\'s'", +] + + +@pytest.mark.parametrize("where", INJECTIONS) +def test_a_predicate_that_is_not_one_boolean_expression_over_the_columns_is_refused(where): + doc = contract(restrictions=None, filters=[{"principal": ANALYSTS, "where": where}]) + assert _kinds(doc) == "databricks-row-filter-invalid", repr(where) + + +def test_a_lone_boolean_column_is_a_predicate_and_any_other_lone_column_is_not(): + boolean = contract( + restrictions=None, filters=[{"principal": ANALYSTS, "where": "consent_data_analytics"}] + ) + assert compiled(boolean).row_filter.columns == ("consent_data_analytics",) + other = contract(restrictions=None, filters=[{"principal": ANALYSTS, "where": "score"}]) + assert _kinds(other) == "databricks-row-filter-invalid" + + +def test_a_function_that_returns_a_value_is_not_a_predicate(): + doc = contract(restrictions=None, filters=[{"principal": ANALYSTS, "where": "lower(region)"}]) + assert _kinds(doc) == "databricks-row-filter-invalid" + + +@pytest.mark.parametrize( + "where", + [ + "region = 'eu'", + "region IN ('eu', 'uk') AND NOT is_suppressed", + "lower(region) = 'eu'", + "upper(trim(region)) LIKE 'E%'", + "coalesce(consent_data_analytics, false) = true", + "score BETWEEN 1 AND 5 OR score IS NULL", + "length(region) > 1 AND abs(score) >= 0.5", + ], +) +def test_ordinary_predicates_compile(where): + enforcement = compiled( + contract(restrictions=None, filters=[{"principal": ANALYSTS, "where": where}]) + ) + assert enforcement.row_filter is not None + + +def test_every_allowed_function_is_a_scalar_one(): + assert set(ALLOWED_FUNCTIONS) == {"lower", "upper", "length", "trim", "coalesce", "abs"} + + +def test_a_predicate_is_written_back_from_the_tree_not_spliced(): + where = "region = 'eu' and (is_suppressed is null)" + enforcement = compiled( + contract(restrictions=None, filters=[{"principal": ANALYSTS, "where": where}]) + ) + body = enforcement.row_filter.function.create + assert "`region` = 'eu' AND (`is_suppressed` IS NULL)" in body + assert " " not in body and " and " not in body + + +def test_a_string_literal_with_a_backslash_cannot_break_out_of_the_function_body(): + where = "region = 'a\\\\'" # the literal a\\ (an escaped backslash), then the quote closes + enforcement = compiled( + contract(restrictions=None, filters=[{"principal": ANALYSTS, "where": where}]) + ) + assert "`region` = 'a\\\\'" in enforcement.row_filter.function.create + + +def test_the_shared_readers_refusals_are_kept_under_the_databricks_kind(): + doc = contract( + restrictions=None, + filters=[FILTER, {"principal": ANALYSTS, "where": "region = 'eu'"}], + ) + error = refusal(doc) + assert error.kind == "databricks-row-filter-invalid" + assert "a second time" in str(error) + + +def test_a_row_filter_on_a_column_the_table_lacks_is_refused(): + doc = contract(restrictions=None, filters=[{"principal": ANALYSTS, "where": "nope = 1"}]) + assert _kinds(doc) == "databricks-row-filter-invalid" + + +def test_a_contract_column_in_another_case_binds_the_tables_lower_case_column(): + doc = contract( + restrictions=[{"principal": ANALYSTS, "columns": ["Email"], "access": "deny"}], + filters=[{"principal": ANALYSTS, "where": "Region = 'eu'"}], + columns=[("Email", "string"), ("Region", "string")], + ) + enforcement = compiled(doc) + assert [m.column for m in enforcement.masks] == ["email"] + assert enforcement.row_filter.columns == ("region",) + + +def test_validate_reports_every_refusal_with_the_emitters_message(): + allow = {"principal": ANALYSTS, "columns": ["email"], "access": "allow"} + errors, _ = dbx.validate_databricks_binding(contract(restrictions=[allow]), ENV) + assert errors and "allow list" in errors[0] + + +# ── generate iac refuses a module a plain tofu apply would leave unprotected ─ +def test_the_plugin_refuses_a_standalone_module_for_a_contract_with_masks_or_filters(monkeypatch): + monkeypatch.setenv(dbx.WAREHOUSE_ENV, ENV[dbx.WAREHOUSE_ENV]) + from fluid_build.iac import IAC_PLUGINS + + plugin = IAC_PLUGINS["databricks"] + refused = plugin.standalone_module_refusal(contract()) + assert refused.kind == "databricks-enforcement-outside-module" + assert "plain tofu apply" in str(refused) and "no mask or row filter" in str(refused) + assert "fluid apply" in " ".join(refused.remediation) + assert "exposes[0] (profile)" in str(refused) + + +def test_a_contract_without_restrictions_or_with_a_refusal_of_its_own_gets_no_such_refusal( + monkeypatch, +): + monkeypatch.setenv(dbx.WAREHOUSE_ENV, ENV[dbx.WAREHOUSE_ENV]) + from fluid_build.iac import IAC_PLUGINS + + plugin = IAC_PLUGINS["databricks"] + assert plugin.standalone_module_refusal(contract(restrictions=None, filters=None)) is None + # A contract that does not compile is left to emit, which raises its own refusal. + allow = {"principal": ANALYSTS, "columns": ["email"], "access": "allow"} + assert plugin.standalone_module_refusal(contract(restrictions=[allow])) is None + + +def test_emit_stays_pure_and_still_emits_the_module_for_fluid_apply(monkeypatch): + monkeypatch.setenv(dbx.WAREHOUSE_ENV, ENV[dbx.WAREHOUSE_ENV]) + from fluid_build.iac import IAC_PLUGINS + + resources = IAC_PLUGINS["databricks"].emit(contract()) + assert "databricks_sql_table" in resources and "databricks_grant" in resources diff --git a/tests/iac/test_iac_databricks_enforce.py b/tests/iac/test_iac_databricks_enforce.py new file mode 100644 index 000000000..5d4180c2b --- /dev/null +++ b/tests/iac/test_iac_databricks_enforce.py @@ -0,0 +1,1293 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Databricks column masks and row filters: the hooks that bind them, against the stand-in. + +No ``tofu``: the plugin's ``pre_apply`` / ``post_apply`` hooks and the engine's gate around +them are driven against ``_fake_databricks.FakeDatabricks`` (statements, Tables API reads), +through its plain-HTTP stand-in for the SDK client. Pinned: what runs before the apply and +what after, a dry run that prints and executes nothing, a second run that executes nothing, +a changed predicate that binds a new function, a removal that drops, a failed binding that +revokes ``SELECT`` and fails, and a pre-apply failure that stops the run. +""" + +from __future__ import annotations + +from typing import Any, Dict, Iterator, List, Optional + +import pytest + +from fluid_build.cli import _apply_opentofu_engine as engine +from fluid_build.cli._common import CLIError +from fluid_build.iac import IAC_PLUGINS +from fluid_build.iac.base import UnsupportedBindingError +from fluid_build.iac.providers import databricks as dbx +from fluid_build.iac.providers import databricks_enforce as enforce + +from ._databricks_governed import ( + ANALYSTS, + ANALYSTS_GROUP, + COLUMNS, + DENY, + FILTER, + MASK, + STEWARDS, + STEWARDS_GROUP, + contract, +) +from ._fake_databricks import DEPLOY_USER, FakeDatabricks + +pytestmark = [pytest.mark.unit, pytest.mark.databricks] + +TABLE = "main.cp360.profile" +PLUGIN = IAC_PLUGINS["databricks"] + + +@pytest.fixture +def fake(monkeypatch) -> Iterator[FakeDatabricks]: + server = FakeDatabricks().start() + monkeypatch.setenv(dbx.WAREHOUSE_ENV, server.warehouse_id) + monkeypatch.setenv("DATABRICKS_TOKEN", "dapi-secret-token-value") # pragma: allowlist secret + try: + yield server + finally: + server.stop() + + +def seed(fake: FakeDatabricks, *, columns=COLUMNS) -> None: + fake.create_table(TABLE, [(n, t, True) for n, t in columns]) + fake.grant("table", TABLE, ANALYSTS_GROUP, "SELECT") + fake.grant("table", TABLE, STEWARDS_GROUP, "SELECT") + + +class Run: + """The hooks against ``fake``, with everything they print kept.""" + + def __init__(self, fake: FakeDatabricks): + self.fake = fake + self.said: List[str] = [] + self.start = len(fake.sql_log) + + def pre(self, doc: Dict[str, Any], **kw: Any) -> None: + PLUGIN.pre_apply(doc, say=self.said.append, client_factory=self.fake.client, **kw) + + def post(self, doc: Dict[str, Any]) -> None: + PLUGIN.post_apply(doc, say=self.said.append, client_factory=self.fake.client) + + @property + def text(self) -> str: + return "\n".join(self.said) + + def statements(self) -> List[str]: + """The statements the warehouse got since this run began.""" + return self.fake.sql_log[self.start :] + + +def _verbs(statements: List[str]) -> List[str]: + return [" ".join(s.split()[:2]) + (" " + s.split()[4] if s.startswith("ALTER") else "") + for s in statements] # fmt: skip + + +def live(fake: FakeDatabricks) -> Dict[str, Any]: + status, doc = fake._get_table(type("M", (), {"group": lambda self, i: TABLE})(), {}, {}) + assert status == 200 + return doc + + +def live_masks(fake: FakeDatabricks) -> Dict[str, str]: + return {c["name"]: c["mask"]["function_name"] for c in live(fake)["columns"] if c.get("mask")} + + +# ── binding ───────────────────────────────────────────────────────────────── +def test_pre_apply_binds_the_filter_and_masks_on_a_table_that_exists(fake): + seed(fake) + run = Run(fake) + run.pre(contract()) + + doc = live(fake) + assert doc["row_filter"]["function_name"].startswith("main.cp360.fluid_rf_profile_") + assert doc["row_filter"]["input_column_names"] == ["consent_data_analytics", "is_suppressed"] + assert sorted(live_masks(fake)) == ["customer_id", "email", "msisdn", "score"] + assert all(f.startswith("main.cp360.fluid_mask_profile_") for f in live_masks(fake).values()) + assert len(fake.functions) == 5 + # Every function is created before the statement that binds it. + statements = run.statements() + for create in [s for s in statements if s.startswith("CREATE FUNCTION")]: + name = create.split("`")[5] + bind = next(i for i, s in enumerate(statements) if f"`{name}`" in s and "ALTER" in s) + assert statements.index(create) < bind + assert not any(s.startswith(("DROP", "REVOKE")) for s in statements) + + +def test_pre_apply_leaves_a_table_the_apply_has_yet_to_create_to_post_apply(fake): + run = Run(fake) + run.pre(contract()) + assert fake.sql_log == [] + + seed(fake) # the apply creates it, empty + run.post(contract()) + assert live(fake)["row_filter"]["function_name"].startswith("main.cp360.fluid_rf_profile_") + assert len(live_masks(fake)) == 4 + + +def test_a_dry_run_prints_every_statement_in_its_phase_and_executes_none(fake): + seed(fake) + run = Run(fake) + run.pre(contract(), dry_run=True) + assert fake.sql_log == [] + assert "dry-run: none of these statements ran." in run.text + assert "before apply: create row filter function main.cp360.fluid_rf_profile_" in run.text + assert "SET ROW FILTER" in run.text and "SET MASK" in run.text + assert "after apply" not in run.text + + other = Run(FakeDatabricks().start()) + try: + other.pre(contract(), dry_run=True) + assert "(not created yet)" in other.text + assert "after apply: bind row filter" in other.text + assert "before apply" not in other.text + assert other.fake.sql_log == [] + finally: + other.fake.stop() + + +def test_a_second_run_executes_nothing(fake): + seed(fake) + run = Run(fake) + run.pre(contract()) + run.post(contract()) + done = len(fake.sql_log) + again = Run(fake) + again.pre(contract()) + again.post(contract()) + assert len(fake.sql_log) == done + assert again.said == [] + + +def test_a_changed_predicate_binds_a_new_function_and_keeps_the_old_one(fake): + seed(fake) + Run(fake).pre(contract()) + old = live(fake)["row_filter"]["function_name"] + masks = live_masks(fake) + + changed = contract(filters=[{**FILTER, "where": "consent_data_analytics = false"}]) + run = Run(fake) + run.pre(changed) + run.post(changed) + + new = live(fake)["row_filter"] + assert new["function_name"] != old + assert new["input_column_names"] == ["consent_data_analytics"] + assert old in fake.functions, "a superseded function is not dropped" + assert live_masks(fake) == masks + assert not any(s.startswith("DROP") for s in run.statements()) + assert len([s for s in run.statements() if s.startswith("CREATE FUNCTION")]) == 1 + + +def test_a_changed_mask_rule_rebinds_only_that_column(fake): + seed(fake) + Run(fake).pre(contract()) + before = live_masks(fake) + changed = contract(restrictions=[DENY, {**MASK, "mask": "first_four"}]) + Run(fake).pre(changed) + after = live_masks(fake) + assert after["msisdn"] != before["msisdn"] + assert {k: v for k, v in after.items() if k != "msisdn"} == { + k: v for k, v in before.items() if k != "msisdn" + } + + +def test_a_binding_someone_changed_by_hand_is_set_back(fake): + seed(fake) + Run(fake).pre(contract()) + fake.tables[TABLE]["row_filter"]["input_column_names"] = [ + "is_suppressed", + "consent_data_analytics", + ] + run = Run(fake) + run.pre(contract()) + assert live(fake)["row_filter"]["input_column_names"] == [ + "consent_data_analytics", + "is_suppressed", + ] + assert any("SET ROW FILTER" in s for s in run.statements()) + + +def test_a_restriction_removed_from_the_contract_is_dropped_after_the_apply_and_said_to_widen( + fake, +): + seed(fake) + Run(fake).pre(contract()) + narrower = contract(restrictions=[DENY], filters=None) + run = Run(fake) + run.pre(narrower) + assert not any(s.startswith("ALTER") for s in run.statements()), "pre_apply never widens" + run.post(narrower) + + doc = live(fake) + assert "row_filter" not in doc + assert "msisdn" not in live_masks(fake) and "email" in live_masks(fake) + assert "ALTER TABLE `main`.`cp360`.`profile` DROP ROW FILTER" in run.statements() + assert any("DROP MASK" in s for s in run.statements()) + assert "access widened (no data lost, not gated): drop row filter" in run.text + assert "access widened (no data lost, not gated): drop mask" in run.text + + # Removing the last restriction drops the rest, from a contract that names none. + bare = contract(restrictions=None, filters=None) + last = Run(fake) + last.post(bare) + assert live_masks(fake) == {} + + +def test_a_filter_or_mask_an_admin_set_by_hand_is_never_dropped(fake): + seed(fake) + run_sql = enforce.SqlRunner(fake.client(), fake.warehouse_id) + run_sql.run( + "CREATE FUNCTION `main`.`cp360`.`admin_rows`(`region` STRING) RETURNS BOOLEAN RETURN true" + ) + run_sql.run( + "ALTER TABLE `main`.`cp360`.`profile` SET ROW FILTER `main`.`cp360`.`admin_rows` ON (`region`)" + ) + run_sql.run("CREATE FUNCTION `main`.`cp360`.`admin_mask`(`v` STRING) RETURNS STRING RETURN `v`") + run_sql.run( + "ALTER TABLE `main`.`cp360`.`profile` ALTER COLUMN `email` SET MASK `main`.`cp360`.`admin_mask`" + ) + done = len(fake.sql_log) + run = Run(fake) + run.post(contract(restrictions=None, filters=None)) + assert len(fake.sql_log) == done + assert live(fake)["row_filter"]["function_name"] == "main.cp360.admin_rows" + assert live_masks(fake) == {"email": "main.cp360.admin_mask"} + + +def test_a_filter_that_needs_a_column_the_apply_adds_is_held_by_a_deny_all_filter_until_then(fake): + seed(fake, columns=[c for c in COLUMNS if c[0] != "is_suppressed"]) + run = Run(fake) + run.pre(contract()) + deny_all = _expected().deny_all.function.full_name + # The table exists and holds rows: the restricted group reads none until the real filter binds. + assert live(fake)["row_filter"] == {"function_name": deny_all, "input_column_names": []} + assert len(live_masks(fake)) == 4, "the masks needed no new column" + + fake.tables[TABLE]["columns"].append( + {"name": "is_suppressed", "type": "boolean", "nullable": True} + ) + run.post(contract()) + assert live(fake)["row_filter"]["input_column_names"] == [ + "consent_data_analytics", + "is_suppressed", + ] + assert live(fake)["row_filter"]["function_name"] == _expected().row_filter.function.full_name + + +def test_the_interim_filter_is_bound_once_and_before_anything_else(fake): + seed(fake, columns=[c for c in COLUMNS if c[0] != "is_suppressed"]) + first = Run(fake) + first.pre(contract()) + statements = first.statements() + deny = next(i for i, s in enumerate(statements) if "SET ROW FILTER" in s) + assert "deny_all" in statements[deny], "the first filter bound is the deny-all one" + assert all("SET MASK" not in s for s in statements[:deny]) + again = Run(fake) + again.pre(contract()) + assert again.statements() == [], "a second run binds nothing: the interim filter is in place" + + +def test_no_interim_filter_when_the_real_one_can_bind_now_or_the_contract_has_none(fake): + seed(fake) + Run(fake).pre(contract()) + assert "deny_all" not in live(fake)["row_filter"]["function_name"] + assert not any("deny_all" in f for f in fake.functions) + bare = Run(FakeDatabricks().start()) + try: + seed(bare.fake, columns=[c for c in COLUMNS if c[0] != "is_suppressed"]) + bare.pre(contract(filters=None)) + assert not any("deny_all" in f for f in bare.fake.functions) + finally: + bare.fake.stop() + + +def test_a_dry_run_shows_the_interim_filter_before_the_apply(fake): + seed(fake, columns=[c for c in COLUMNS if c[0] != "is_suppressed"]) + run = Run(fake) + run.pre(contract(), dry_run=True) + assert "before apply: bind deny-all row filter" in run.text + assert "after apply: bind row filter" in run.text + assert fake.sql_log == [] + + +def test_a_hostile_deny_all_function_stops_the_run_before_the_apply(fake): + seed(fake, columns=[c for c in COLUMNS if c[0] != "is_suppressed"]) + _plant(fake, _expected().deny_all.function, body="true") + with pytest.raises(UnsupportedBindingError, match="its body is 'true'"): + Run(fake).pre(contract()) + assert "row_filter" not in live(fake) + + +def test_a_foreign_row_filter_is_not_replaced_by_the_interim_filter(fake): + seed(fake, columns=[c for c in COLUMNS if c[0] != "is_suppressed"]) + theirs = _admin_filter(fake) + with pytest.raises(UnsupportedBindingError, match="did not create"): + Run(fake).pre(contract()) + assert live(fake)["row_filter"]["function_name"] == theirs + + +def test_a_create_or_replace_table_keeps_the_bindings_in_the_stand_in(fake): + seed(fake) + Run(fake).pre(contract()) + runner = enforce.SqlRunner(fake.client(), fake.warehouse_id) + cols = ", ".join(f"`{n}` {t.upper()}" for n, t in COLUMNS) + runner.run(f"CREATE OR REPLACE TABLE `main`.`cp360`.`profile` ({cols}) USING DELTA") + assert fake.replaced == [TABLE] + assert "row_filter" in live(fake) and len(live_masks(fake)) == 4 + + +def test_unity_catalog_refuses_to_replace_a_function_a_filter_uses(fake): + seed(fake) + Run(fake).pre(contract()) + name = live(fake)["row_filter"]["function_name"] + cat, sch, fn = name.split(".") + runner = enforce.SqlRunner(fake.client(), fake.warehouse_id) + with pytest.raises(enforce.StatementFailed, match="FUNCTION_IN_USE"): + runner.run( + f"CREATE OR REPLACE FUNCTION `{cat}`.`{sch}`.`{fn}`(`a` BOOLEAN) RETURNS BOOLEAN RETURN true" + ) + + +def test_no_statement_this_module_sends_replaces_a_function(fake): + seed(fake) + run = Run(fake) + run.pre(contract()) + run.pre(contract(filters=[{**FILTER, "where": "consent_data_analytics = false"}])) + assert not any("REPLACE FUNCTION" in s for s in fake.sql_log) + assert all( + s.startswith("CREATE FUNCTION IF NOT EXISTS") for s in fake.sql_log if "FUNCTION" in s + ) + + +# ── failing safe ──────────────────────────────────────────────────────────── +def test_a_pre_apply_failure_stops_the_run_and_revokes_nothing(fake): + seed(fake) + fake.fail_on("SET ROW FILTER", "[INSUFFICIENT_PERMISSIONS] not the owner") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.pre(contract()) + assert exc.value.kind == "databricks-enforcement-failed" + assert "[INSUFFICIENT_PERMISSIONS] not the owner" in str(exc.value) + assert "SET ROW FILTER" in str(exc.value) and "the apply did not run" in str(exc.value) + assert not any(s.startswith("REVOKE") for s in fake.sql_log) + assert fake.grants[("table", TABLE)][ANALYSTS_GROUP] == ["SELECT"] + + +def test_a_post_apply_failure_binds_a_deny_all_filter_for_the_restricted_and_revokes_nothing(fake): + seed(fake) + fake.fail_on("SET MASK", "[INTERNAL_ERROR] warehouse stopped") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.post(contract()) + + deny_all = _expected().deny_all.function.full_name + assert live(fake)["row_filter"]["function_name"] == deny_all + assert live(fake)["row_filter"]["input_column_names"] == [] + assert not any(s.startswith("REVOKE") for s in fake.sql_log), "no grant was touched" + assert fake.grants[("table", TABLE)][ANALYSTS_GROUP] == ["SELECT"] + assert "fail-safe: binding a deny-all row filter" in run.text + message = str(exc.value) + assert "[INTERNAL_ERROR] warehouse stopped" in message + assert f"A deny-all row filter ({deny_all}) is bound" in message + assert "whatever grant gives them the table" in message + assert "SELECT was not revoked from anyone" in message + assert "re-run fluid apply" in " ".join(exc.value.remediation) + + +def test_a_failed_revoke_is_reported_as_not_revoked(fake): + seed(fake) + fake.fail_on("SET ROW FILTER") # the real filter and the deny-all filter both fail + fake.fail_on("REVOKE", "[PERMISSION_DENIED] cannot revoke") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.post(contract()) + assert "NOT REVOKED: REVOKE SELECT" in run.text + assert "could NOT be revoked, run them yourself, now" in str(exc.value) + assert "Nothing was revoked" in str(exc.value) + assert fake.grants[("table", TABLE)][ANALYSTS_GROUP] == ["SELECT"] + + +def test_a_table_missing_after_the_apply_is_revoked_and_fails(fake): + run = Run(fake) + with pytest.raises(UnsupportedBindingError, match="does not exist after the apply"): + run.post(contract()) + + +def test_post_apply_goes_on_to_the_next_table_after_a_failure(fake): + seed(fake) + second = contract() + second["exposes"].append( + { + **contract()["exposes"][0], + "exposeId": "profile2", + "binding": { + **contract()["exposes"][0]["binding"], + "location": {"database": "main", "schema": "cp360", "table": "profile2"}, + }, + } + ) + fake.create_table("main.cp360.profile2", [(n, t, True) for n, t in COLUMNS]) + fake.fail_on("fluid_mask_profile", "[INTERNAL_ERROR] x") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.post(second) + assert "main.cp360.profile, main.cp360.profile2" in str(exc.value) + + +# ── the client ────────────────────────────────────────────────────────────── +def test_a_contract_with_restrictions_needs_the_sdk_and_a_warehouse(fake, monkeypatch): + def no_sdk(): + raise ImportError("databricks") + + with pytest.raises(UnsupportedBindingError, match="databricks-sdk is not installed"): + PLUGIN.pre_apply(contract(), say=print, client_factory=no_sdk) + + monkeypatch.delenv(dbx.WAREHOUSE_ENV) + seed(fake) + # Without the warehouse the contract is refused at compile, before any client. + with pytest.raises(UnsupportedBindingError) as exc: + PLUGIN.pre_apply(contract(), say=print, client_factory=fake.client) + assert exc.value.kind == "databricks-warehouse-unset" + + +def test_a_contract_without_restrictions_needs_no_sdk_and_says_nothing(fake): + said: List[str] = [] + + def no_sdk(): + raise ImportError("databricks") + + bare = contract(restrictions=None, filters=None) + PLUGIN.post_apply(bare, say=said.append, client_factory=no_sdk) + PLUGIN.pre_apply(bare, say=said.append, client_factory=no_sdk) + PLUGIN.pre_apply(bare, say=said.append, dry_run=True, client_factory=no_sdk) + assert said == [] + + +def test_a_client_that_cannot_be_made_for_a_contract_without_restrictions_is_a_note(fake): + said: List[str] = [] + + def broken(): + raise RuntimeError("no route to host") + + PLUGIN.post_apply( + contract(restrictions=None, filters=None), say=said.append, client_factory=broken + ) + assert any("not checked for removal" in line for line in said) + + +def test_a_read_failure_is_an_error_naming_the_table(fake): + class Broken: + class api_client: # noqa: N801 + @staticmethod + def do(*_a: Any, **_k: Any) -> Any: + raise RuntimeError("boom dapi-secret-token-value") + + with pytest.raises(UnsupportedBindingError) as exc: + PLUGIN.pre_apply(contract(), say=print, client_factory=lambda: Broken()) + assert "main.cp360.profile" in str(exc.value) + assert "dapi-secret-token-value" not in str(exc.value) + + +class _Scripted: + """A client whose statement endpoint answers from a script.""" + + def __init__(self, *docs: Dict[str, Any]): + self.docs = list(docs) + self.calls: List[Any] = [] + self.api_client = self + + def do(self, method: str, path: str, **kw: Any) -> Dict[str, Any]: + self.calls.append((method, path, kw.get("body"))) + if path.endswith("/cancel"): + return {} + return self.docs.pop(0) if len(self.docs) > 1 else self.docs[0] + + +def _doc(state: str, message: Optional[str] = None) -> Dict[str, Any]: + doc: Dict[str, Any] = {"statement_id": "s1", "status": {"state": state}} + if message: + doc["status"]["error"] = {"message": message} + return doc + + +def test_a_statement_that_is_still_running_is_polled_until_it_succeeds(): + client = _Scripted(_doc("PENDING"), _doc("RUNNING"), _doc("SUCCEEDED")) + enforce.SqlRunner(client, "wh", poll=0, sleep=lambda _s: None).run("ALTER TABLE x") + methods = [(m, p) for m, p, _b in client.calls] + assert methods[0] == ("POST", "/api/2.0/sql/statements/") + assert methods[1:] == [("GET", "/api/2.0/sql/statements/s1")] * 2 + body = client.calls[0][2] + assert body["warehouse_id"] == "wh" and body["statement"] == "ALTER TABLE x" + assert body["on_wait_timeout"] == "CONTINUE" + + +@pytest.mark.parametrize("state", ["FAILED", "CANCELED", "CLOSED"]) +def test_a_statement_that_does_not_succeed_fails_with_the_servers_message(state, monkeypatch): + monkeypatch.setenv("DATABRICKS_TOKEN", "dapi-secret-token-value") # pragma: allowlist secret + client = _Scripted(_doc(state, "bad thing with dapi-secret-token-value in it")) + with pytest.raises(enforce.StatementFailed) as exc: + enforce.SqlRunner(client, "wh", sleep=lambda _s: None).run("x") + assert state in str(exc.value) and "bad thing" in str(exc.value) + assert "dapi-secret-token-value" not in str(exc.value), "a credential is never logged" + + +def test_a_statement_that_never_finishes_is_cancelled_and_fails(): + client = _Scripted(_doc("RUNNING")) + with pytest.raises(enforce.StatementFailed, match="did not finish"): + enforce.SqlRunner(client, "wh", timeout=-1, sleep=lambda _s: None).run("x") + assert client.calls[-1][1] == "/api/2.0/sql/statements/s1/cancel" + + +# ── the plan, without a server ────────────────────────────────────────────── +def _enf(): + result = dbx._compile(contract(), {dbx.WAREHOUSE_ENV: "wh"}) + return result.tables[0].enforcement + + +def test_a_table_already_enforcing_exactly_this_has_nothing_to_do(): + enforcement = _enf() + columns = [{"name": n, "type_text": t} for n, t in COLUMNS] + for mask in enforcement.masks: + next(c for c in columns if c["name"] == mask.column)["mask"] = { + "function_name": mask.function.full_name.upper() # case is not significant + } + live_table = { + "columns": columns, + "row_filter": { + "function_name": "`" + enforcement.row_filter.function.full_name + "`", + "input_column_names": [c.upper() for c in enforcement.row_filter.columns], + }, + } + plan = enforce.plan_table(TABLE, enforcement, live_table) + assert plan.bind == [] and plan.drop == [] + + +def test_a_gate_that_is_not_there_is_everything_to_bind(): + plan = enforce.plan_table(TABLE, _enf(), None) + assert not plan.exists and len(plan.bind) == 15 and plan.drop == [] + + +# ── the engine's gate ─────────────────────────────────────────────────────── +class _Recorder: + def __init__(self, raises: Optional[Exception] = None): + self.calls: List[Any] = [] + self.raises = raises + + def pre_apply(self, contract, say=None, dry_run=False): + self.calls.append(("pre_apply", dry_run)) + if self.raises: + raise self.raises + + def post_apply(self, contract, say=None): + self.calls.append(("post_apply", None)) + if self.raises: + raise self.raises + + +def test_the_engine_calls_the_hook_a_plugin_has_and_skips_one_without(): + rec = _Recorder() + engine._enforce_access(rec, "pre_apply", {}) + engine._enforce_access(rec, "pre_apply", {}, dry_run=True) + engine._enforce_access(rec, "post_apply", {}) + assert rec.calls == [("pre_apply", False), ("pre_apply", True), ("post_apply", None)] + engine._enforce_access(object(), "pre_apply", {}) # a plugin with no hook: nothing happens + + +@pytest.mark.parametrize("hook", ["pre_apply", "post_apply"]) +def test_a_refusal_from_a_hook_is_a_typed_error_with_its_fix(hook): + error = enforce.EnforcementError("it broke", ("fix it",)) + with pytest.raises(CLIError) as exc: + engine._enforce_access(_Recorder(error), hook, {}) + assert exc.value.exit_code == 1 and exc.value.event == "opentofu_access_enforcement_failed" + assert exc.value.context["kind"] == "databricks-enforcement-failed" + assert exc.value.context["remediation"] == ["fix it"] + + +def test_the_engine_runs_the_hooks_around_tofu_apply_in_this_order(): + source = __import__("inspect").getsource(engine._apply_via_opentofu) + pre = source.index('_enforce_access(plugin, "pre_apply", contract)') + keep = source.index("imported.keep()") + apply = source.index("runner.tofu_apply(") + post = source.index('_enforce_access(plugin, "post_apply", contract)') + gate = source.index("_enforce_data_loss_gate(") + dry = source.index('_enforce_access(plugin, "pre_apply", contract, dry_run=True)') + assert gate < dry < source.index('cprint("\\ndry-run: plan only') + assert gate < pre < keep < apply < post + + +# ── after a tofu apply that failed part way ───────────────────────────────── +def test_after_a_failed_apply_post_apply_binds_what_exists_and_skips_a_missing_table(fake): + seed(fake) + run = Run(fake) + PLUGIN.post_apply( + contract(), say=run.said.append, client_factory=fake.client, after_failed_apply=True + ) + assert live(fake)["row_filter"]["function_name"].startswith("main.cp360.fluid_rf_profile_") + assert len(live_masks(fake)) == 4 + + # A table the failed apply never created is skipped: no failure, no revoke. + other = Run(FakeDatabricks().start()) + try: + PLUGIN.post_apply( + contract(), + say=other.said.append, + client_factory=other.fake.client, + after_failed_apply=True, + ) + assert other.fake.sql_log == [] + assert "not created by the failed apply" in other.text + assert "NOT REVOKED" not in other.text and "fail-safe" not in other.text + finally: + other.fake.stop() + + +def test_after_a_failed_apply_nothing_is_dropped_even_what_the_contract_no_longer_asks(fake): + seed(fake) + Run(fake).pre(contract()) + narrower = contract(restrictions=[DENY], filters=None) + run = Run(fake) + PLUGIN.post_apply( + narrower, say=run.said.append, client_factory=fake.client, after_failed_apply=True + ) + assert not any(s.startswith(("DROP", "ALTER")) for s in run.statements()) + assert "row_filter" in live(fake) and "msisdn" in live_masks(fake) + # The same contract after a successful apply does drop them. + PLUGIN.post_apply(narrower, say=run.said.append, client_factory=fake.client) + assert "row_filter" not in live(fake) and "msisdn" not in live_masks(fake) + + +def test_a_table_without_restrictions_is_not_even_read_after_a_failed_apply(fake): + seed(fake) + Run(fake).pre(contract()) + PLUGIN.post_apply( + contract(restrictions=None, filters=None), + say=print, + client_factory=fake.client, + after_failed_apply=True, + ) + assert "row_filter" in live(fake) and len(live_masks(fake)) == 4 + + +def test_a_binding_that_fails_after_a_failed_apply_locks_the_table_and_says_so(fake): + seed(fake) + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] warehouse stopped") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + PLUGIN.post_apply( + contract(), say=run.said.append, client_factory=fake.client, after_failed_apply=True + ) + assert str(exc.value).startswith("the apply failed part way") + # The deny-all filter could not be bound either: SELECT is revoked from every grantee. + assert fake.grants[("table", TABLE)][ANALYSTS_GROUP] == [] + assert fake.grants[("table", TABLE)][STEWARDS_GROUP] == [] + + +def test_the_engine_prints_an_enforcement_failure_after_a_failed_apply_and_never_raises_it( + capsys, +): + rec = _Recorder(enforce.EnforcementError("it broke", ("fix it",))) + rec.post_apply = lambda contract, say=None, after_failed_apply=False: ( # type: ignore + rec.calls.append(("post_apply", after_failed_apply)), + (_ for _ in ()).throw(enforce.EnforcementError("it broke", ("fix it",))), + ) + engine._enforce_after_failed_apply(rec, {}) + assert rec.calls == [("post_apply", True)] + out = capsys.readouterr().out + assert "it broke" in out and "fix it" in out + + boom = _Recorder() + boom.post_apply = lambda contract, say=None, after_failed_apply=False: 1 / 0 # type: ignore + engine._enforce_after_failed_apply(boom, {}) # a bug in the hook never hides the apply's error + assert "ZeroDivisionError" in capsys.readouterr().out + engine._enforce_after_failed_apply(object(), {}) # no hook: nothing happens + + +def test_the_engine_binds_before_it_raises_the_apply_failure(): + source = __import__("inspect").getsource(engine._apply_via_opentofu) + bind = source.index("_enforce_after_failed_apply(plugin, contract)") + failed = source.index('"opentofu_apply_failed"') + assert source.index("runner.tofu_apply(") < bind < failed + + +# ── a function someone else made under the hashed name ────────────────────── +def _expected(): + return dbx._compile(contract(), {dbx.WAREHOUSE_ENV: "wh"}).tables[0].enforcement + + +def _params(fn): + return [(n, t) for n, t in fn.params] + + +def _plant(fake, fn, *, owner="attacker@example.test", body=None, returns=None, params=None): + fake.add_function( + fn.full_name, + _params(fn) if params is None else params, + returns or fn.returns, + fn.body if body is None else body, + owner, + ) + + +def test_a_function_pre_created_under_a_mask_name_is_never_bound(fake): + """``RETURN val`` under the computable name, made by someone with CREATE FUNCTION.""" + seed(fake) + mask = next(m for m in _expected().masks if m.column == "email") + _plant(fake, mask.function, body="`val`") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.pre(contract()) + message = str(exc.value) + assert mask.function.full_name in message + assert "attacker@example.test" in message and "its body is '`val`'" in message + assert "It was not bound" in message + assert not any( + f"SET MASK `{mask.function.full_name.replace('.', '`.`')}`" in s for s in run.statements() + ) + assert "email" not in live_masks(fake) + assert fake.functions[mask.function.full_name]["body"] == "`val`", "theirs is left as it is" + + +def test_a_function_pre_created_under_the_row_filter_name_is_never_bound(fake): + seed(fake) + row = _expected().row_filter + _plant(fake, row.function, body="true") + with pytest.raises(UnsupportedBindingError, match="its body is 'true'"): + Run(fake).pre(contract()) + assert "row_filter" not in live(fake) + + +def test_a_function_with_the_expected_body_but_another_owner_is_not_bound(fake): + seed(fake) + row = _expected().row_filter + _plant(fake, row.function, owner="someone-else@example.test") + with pytest.raises(UnsupportedBindingError) as exc: + Run(fake).pre(contract()) + assert "its owner is someone-else@example.test" in str(exc.value) + assert DEPLOY_USER in str(exc.value) + assert "row_filter" not in live(fake) + + +@pytest.mark.parametrize( + "change,fragment", + [ + ({"returns": "INT"}, "it returns int"), + ({"params": [("val", "INT")]}, "its parameters are"), + ({"params": []}, "its parameters are"), + ], + ids=["return-type", "parameter-type", "no-parameters"], +) +def test_a_function_with_other_parameters_or_return_type_is_not_bound(fake, change, fragment): + seed(fake) + mask = next(m for m in _expected().masks if m.column == "email") + _plant(fake, mask.function, owner=DEPLOY_USER, **change) + with pytest.raises(UnsupportedBindingError, match=fragment): + Run(fake).pre(contract()) + assert "email" not in live_masks(fake) + + +def test_a_function_that_only_differs_in_whitespace_or_case_of_types_is_bound(fake): + seed(fake) + row = _expected().row_filter + _plant( + fake, + row.function, + owner=DEPLOY_USER.upper(), + body=" " + row.function.body.replace(" ", " ") + "\n", + params=[(n, t.lower()) for n, t in row.function.params], + ) + Run(fake).pre(contract()) + assert live(fake)["row_filter"]["function_name"] == row.function.full_name + + +def test_a_function_that_was_made_by_this_module_passes_its_own_check(fake): + seed(fake) + run = Run(fake) + run.pre(contract()) + assert live(fake)["row_filter"]["function_name"] == _expected().row_filter.function.full_name + assert all( + s.startswith(("CREATE FUNCTION IF NOT EXISTS", "ALTER TABLE")) for s in run.statements() + ) + + +def test_the_identity_is_read_once_per_run(fake): + seed(fake) + Run(fake).pre(contract()) + me = [r for r in fake.requests if r == ("GET", "/api/2.0/preview/scim/v2/Me")] + assert len(me) == 1 + + +def test_a_hostile_function_after_a_successful_apply_is_a_failed_bind_with_the_fail_safe(fake): + seed(fake) + mask = next(m for m in _expected().masks if m.column == "email") + _plant(fake, mask.function, body="`val`") + run = Run(fake) + with pytest.raises(UnsupportedBindingError): + run.post(contract()) + assert "email" not in live_masks(fake) + assert "attacker@example.test" in run.text, "what differs is printed" + # The fail-safe ran: the (genuine) deny-all filter blocks the restricted group. + assert live(fake)["row_filter"]["function_name"] == _expected().deny_all.function.full_name + + +def test_function_problem_names_every_difference_and_none_for_the_expected_function(): + row = _expected().row_filter.function + info = { + "owner": DEPLOY_USER, + "routine_body": "SQL", + "routine_definition": row.body, + "full_data_type": "boolean", + "input_params": { + "parameters": [{"name": n, "type_text": t.lower()} for n, t in row.params] + }, + } + assert enforce.function_problem(info, row, {DEPLOY_USER}) is None + assert "does not exist" in enforce.function_problem(None, row, {DEPLOY_USER}) + bad = dict(info, owner="x", routine_definition="true", full_data_type="int") + problem = enforce.function_problem(bad, row, {DEPLOY_USER}) + assert "owner is x" in problem and "its body is 'true'" in problem + assert "it returns int" in problem + + +# ── a binding an admin made by hand is never replaced silently ────────────── +def _admin_filter(fake: FakeDatabricks, table: str = TABLE) -> str: + cat, sch, name = table.split(".") + sql = enforce.SqlRunner(fake.client(), fake.warehouse_id) + sql.run( + f"CREATE FUNCTION `{cat}`.`{sch}`.`admin_rows_{name}`(`region` STRING) " + "RETURNS BOOLEAN RETURN true" + ) + sql.run( + f"ALTER TABLE `{cat}`.`{sch}`.`{name}` SET ROW FILTER " + f"`{cat}`.`{sch}`.`admin_rows_{name}` ON (`region`)" + ) + return f"{cat}.{sch}.admin_rows_{name}" + + +def _admin_mask(fake: FakeDatabricks, column: str) -> str: + sql = enforce.SqlRunner(fake.client(), fake.warehouse_id) + sql.run( + f"CREATE FUNCTION `main`.`cp360`.`admin_mask_{column}`(`v` STRING) " + "RETURNS STRING RETURN `v`" + ) + sql.run( + f"ALTER TABLE `main`.`cp360`.`profile` ALTER COLUMN `{column}` SET MASK " + f"`main`.`cp360`.`admin_mask_{column}`" + ) + return f"main.cp360.admin_mask_{column}" + + +def test_a_row_filter_set_by_hand_is_not_replaced_by_the_contracts(fake): + seed(fake) + theirs = _admin_filter(fake) + done = len(fake.sql_log) + with pytest.raises(UnsupportedBindingError) as exc: + Run(fake).pre(contract()) + message = str(exc.value) + assert TABLE in message and theirs in message and "did not create" in message + assert "DROP ROW FILTER" in " ".join(exc.value.remediation) + assert "policy.authz.rowFilters" in " ".join(exc.value.remediation) + assert len(fake.sql_log) == done, "nothing was changed, not even a function created" + assert live(fake)["row_filter"]["function_name"] == theirs + + +def test_a_mask_set_by_hand_is_not_replaced_and_the_message_names_the_column(fake): + seed(fake) + theirs = _admin_mask(fake, "email") + done = len(fake.sql_log) + with pytest.raises(UnsupportedBindingError) as exc: + Run(fake).pre(contract()) + assert f"{TABLE}.email" in str(exc.value) and theirs in str(exc.value) + assert "DROP MASK" in " ".join(exc.value.remediation) + assert len(fake.sql_log) == done + assert live_masks(fake) == {"email": theirs} + + +def test_a_refusal_on_a_later_table_changes_nothing_on_an_earlier_one(fake): + seed(fake) + fake.create_table("main.cp360.profile2", [(n, t, True) for n, t in COLUMNS]) + doc = contract() + doc["exposes"].append( + { + **contract()["exposes"][0], + "exposeId": "profile2", + "binding": { + **contract()["exposes"][0]["binding"], + "location": {"database": "main", "schema": "cp360", "table": "profile2"}, + }, + } + ) + _admin_filter(fake, "main.cp360.profile2") + done = len(fake.sql_log) + with pytest.raises(UnsupportedBindingError, match="profile2"): + Run(fake).pre(doc) + assert len(fake.sql_log) == done + assert "row_filter" not in live(fake), "the first table was not bound either" + + +def test_a_binding_set_by_hand_on_something_the_contract_does_not_restrict_is_left_alone(fake): + seed(fake) + theirs = _admin_mask(fake, "region") + Run(fake).pre(contract(restrictions=[DENY], filters=None)) + assert live_masks(fake)["region"] == theirs + assert "email" in live_masks(fake) + + +def test_a_filter_this_module_made_is_still_replaced_without_a_fuss(fake): + seed(fake) + Run(fake).pre(contract()) + Run(fake).pre(contract(filters=[{**FILTER, "where": "consent_data_analytics = false"}])) + assert live(fake)["row_filter"]["input_column_names"] == ["consent_data_analytics"] + + +def test_a_foreign_binding_found_after_the_apply_is_a_failed_bind_with_the_fail_safe(fake): + seed(fake) + theirs = _admin_filter(fake) + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.post(contract()) + assert theirs in str(exc.value) + assert live(fake)["row_filter"]["function_name"] == theirs, "theirs was not replaced" + assert fake.grants[("table", TABLE)][ANALYSTS_GROUP] == [] + + +def test_a_dry_run_reports_the_refusal_too(fake): + seed(fake) + _admin_filter(fake) + with pytest.raises(UnsupportedBindingError, match="did not create"): + Run(fake).pre(contract(), dry_run=True) + + +# ── a Tables API read that fails ──────────────────────────────────────────── +def _two_tables() -> Dict[str, Any]: + doc = contract() + doc["exposes"].append( + { + **contract()["exposes"][0], + "exposeId": "profile2", + "binding": { + **contract()["exposes"][0]["binding"], + "location": {"database": "main", "schema": "cp360", "table": "profile2"}, + }, + } + ) + return doc + + +def _seed_two(fake: FakeDatabricks) -> None: + seed(fake) + fake.create_table("main.cp360.profile2", [(n, t, True) for n, t in COLUMNS]) + fake.grant("table", "main.cp360.profile2", ANALYSTS_GROUP, "SELECT") + + +@pytest.mark.parametrize("status", [503, 429]) +@pytest.mark.parametrize( + "after_failed_apply", [False, True], ids=["after-apply", "after-failed-apply"] +) +def test_a_read_that_fails_for_one_table_is_a_failed_bind_for_it_and_the_next_table_is_tried( + fake, status, after_failed_apply +): + _seed_two(fake) + fake.fail_route("GET", r"/tables/main\.cp360\.profile$", status, "temporarily unavailable") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + PLUGIN.post_apply( + _two_tables(), + say=run.said.append, + client_factory=fake.client, + after_failed_apply=after_failed_apply, + ) + # T1 could not be read, so a filter on it is unknown and the deny-all filter is not bound + # over it: its grants are revoked (every grantee but the owner), not left granted and unbound. + assert fake.grants[("table", TABLE)][ANALYSTS_GROUP] == [] + assert fake.grants[("table", TABLE)][STEWARDS_GROUP] == [] + assert "could not read main.cp360.profile " in str(exc.value) + assert "revoked: REVOKE SELECT ON TABLE `main`.`cp360`.`profile`" in run.text + assert "the table could not be read, so a filter on it is unknown" in str(exc.value) + # T2 was still tried: it is bound, and keeps its grant. + live2 = fake.tables["main.cp360.profile2"] + assert live2["row_filter"]["function_name"].startswith("main.cp360.fluid_rf_profile2_") + assert fake.grants[("table", "main.cp360.profile2")][ANALYSTS_GROUP] == ["SELECT"] + assert "main.cp360.profile2" not in str(exc.value).split("could not be put in place")[0] + + +def test_a_read_that_fails_for_a_table_without_restrictions_is_a_note_and_the_rest_go_on(fake): + _seed_two(fake) + bare = _two_tables() + first = bare["exposes"][0]["policy"]["authz"] + first.pop("columnRestrictions") + first.pop("rowFilters") + fake.fail_route("GET", r"/tables/main\.cp360\.profile$", 503, "temporarily unavailable") + run = Run(fake) + PLUGIN.post_apply(bare, say=run.said.append, client_factory=fake.client) + assert "could not be read, so its earlier masks" in run.text + assert "row_filter" in fake.tables["main.cp360.profile2"], "the next table was bound" + assert not any(s.startswith("REVOKE") for s in run.statements()) + + +def test_a_read_failure_before_the_apply_still_stops_the_run(fake): + seed(fake) + fake.fail_route("GET", r"/tables/main\.cp360\.profile$", 503, "temporarily unavailable") + with pytest.raises(UnsupportedBindingError, match="could not read main.cp360.profile"): + Run(fake).pre(contract()) + + +# ── the fail-safe: a deny-all filter, then a lock ─────────────────────────── +def _holders(fake: FakeDatabricks) -> Dict[str, List[str]]: + return {p: v for p, v in fake.grants[("table", TABLE)].items() if v} + + +def test_the_deny_all_filter_hides_every_row_from_the_restricted_and_none_from_anyone_else(): + duck = pytest.importorskip("duckdb") + import sqlglot + + spec = _expected().deny_all + assert spec.columns == () and spec.function.params == () + assert spec.bind.endswith("ON ()") + body = spec.function.create.split(" RETURN ", 1)[1] + sql = sqlglot.transpile(body, read="databricks", write="duckdb")[0] + + def keeps(groups): + con = duck.connect() + members = ", ".join("'" + g + "'" for g in groups) + con.execute(f"CREATE MACRO is_account_group_member(g) AS list_contains([{members}], g)") + return con.execute(f"SELECT {sql}").fetchone()[0] + + assert keeps([ANALYSTS_GROUP]) is False + assert keeps([STEWARDS_GROUP]) is True + assert keeps([]) is True + assert _expected().deny_all.function.full_name.startswith( + "main.cp360.fluid_rf_profile_deny_all_" + ) + + +def test_a_failed_bind_leaves_a_deny_all_filter_that_the_next_apply_replaces(fake): + seed(fake) + fake.fail_on("SET MASK", "[INTERNAL_ERROR] warehouse stopped") + with pytest.raises(UnsupportedBindingError): + Run(fake).post(contract()) + deny_all = _expected().deny_all.function.full_name + assert live(fake)["row_filter"]["function_name"] == deny_all + + fake.failing.clear() + Run(fake).pre(contract()) + assert live(fake)["row_filter"]["function_name"] == _expected().row_filter.function.full_name + assert len(live_masks(fake)) == 4 + assert deny_all in fake.functions, "the deny-all function is kept: harmless metadata" + + +def test_a_hostile_function_under_the_deny_all_name_is_not_bound_and_the_table_is_locked(fake): + seed(fake) + spec = _expected().deny_all + _plant(fake, spec.function, body="true") + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") # the real filter fails too + fake.fail_on("fluid_mask_profile", "[INTERNAL_ERROR] x") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.post(contract()) + assert "its body is 'true'" in run.text + assert "row_filter" not in live(fake), "the hostile deny-all was never bound" + assert "The deny-all row filter could not be bound" in str(exc.value) + assert _holders(fake) == {}, "so SELECT was revoked from every grantee" + + +def test_when_the_deny_all_filter_fails_select_is_revoked_from_every_grantee_but_the_owner(fake): + seed(fake) + fake.grant("table", TABLE, "other-group", "SELECT") # a path the restricted member is on + fake.grant("table", TABLE, "power-group", "ALL_PRIVILEGES") + fake.grant("table", TABLE, "fake-user@example.test", "SELECT") # the table's owner + fake.grant("table", TABLE, "modifier", "MODIFY") + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.post(contract()) + held = _holders(fake) + assert held == {"fake-user@example.test": ["SELECT"], "modifier": ["MODIFY"]} + revokes = sorted(s for s in fake.sql_log if s.startswith("REVOKE")) + assert "REVOKE ALL PRIVILEGES ON TABLE `main`.`cp360`.`profile` FROM `power-group`" in revokes + assert "REVOKE SELECT ON TABLE `main`.`cp360`.`profile` FROM `other-group`" in revokes + assert not any("fake-user" in s or "modifier" in s for s in revokes) + message = str(exc.value) + assert "every grantee but its owner" in message + assert "other-group" in message and "power-group" in message + assert "No grant inherited from the schema or the catalog was found open" in message + + +def test_a_grant_inherited_from_the_schema_or_catalog_is_named_with_what_to_run(fake): + seed(fake) + fake.grant("schema", "main.cp360", "schema-readers", "SELECT") + fake.grant("catalog", "main", "catalog-admins", "ALL_PRIVILEGES") + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.post(contract()) + message = str(exc.value) + assert "Still open:" in message + assert ( + "schema-readers holds SELECT on SCHEMA main.cp360, which a table-level REVOKE cannot " + "close; run: REVOKE SELECT ON SCHEMA `main`.`cp360` FROM `schema-readers`" + ) in message + assert "run: REVOKE ALL PRIVILEGES ON CATALOG `main` FROM `catalog-admins`" in message + assert "No grant inherited" not in message + + +def test_a_foreign_row_filter_is_not_replaced_by_the_deny_all_filter_either(fake): + seed(fake) + theirs = _admin_filter(fake) + fake.fail_on("SET MASK", "[INTERNAL_ERROR] x") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.post(contract()) + assert live(fake)["row_filter"]["function_name"] == theirs + assert f"it would replace {theirs}, a row filter this module did not create" in str(exc.value) + assert _holders(fake) == {}, "locked by revoking instead" + + +def test_a_grant_listing_that_fails_revokes_only_the_restricted_identities_and_says_so(fake): + seed(fake) + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") + fake.fail_route("GET", r"/permissions/table/", 503, "unavailable") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.post(contract()) + assert fake.grants[("table", TABLE)][ANALYSTS_GROUP] == [] + assert fake.grants[("table", TABLE)][STEWARDS_GROUP] == ["SELECT"] + assert "Its grants could not be listed, so only the restricted identities were revoked" in str( + exc.value + ) + + +def test_a_contract_whose_restricted_principals_have_no_identity_has_nothing_to_close(fake): + seed(fake) + doc = contract(principals={ANALYSTS: [], STEWARDS: STEWARDS_GROUP}) + enforcement = dbx._compile(doc, {dbx.WAREHOUSE_ENV: "wh"}).tables[0].enforcement + assert enforcement.deny_all is None + + +# ── a dbt job that writes a table not yet protected is created paused ─────── +def _job_contract(trigger=None, *, outputs=("profile",), doc=None): + doc = doc or contract() + doc["builds"] = [ + { + "id": "daily", + "engine": "dbt", + "repository": "https://github.com/o/r?tag=v1", + "outputs": list(outputs), + "properties": {"model": "profile"}, + "execution": { + "trigger": trigger or {"type": "schedule", "cron": "0 2 * * *"}, + "runtime": {"platform": "databricks"}, + }, + } + ] + return doc + + +def _emitted_job(doc): + resources = PLUGIN.emit(doc) + (job,) = resources["databricks_job"].values() + return job + + +@pytest.fixture +def workspace_client(fake, monkeypatch): + monkeypatch.setattr(enforce, "default_client", fake.client) + return fake + + +@pytest.mark.parametrize( + "trigger,block", + [ + ({"type": "schedule", "cron": "*/5 * * * *"}, "schedule"), + ({"type": "continuous"}, "continuous"), + ], + ids=["frequent-cron", "continuous"], +) +def test_a_job_writing_a_table_that_does_not_exist_yet_is_created_paused( + workspace_client, trigger, block +): + job = _emitted_job(_job_contract(trigger)) + assert job[block]["pause_status"] == "PAUSED" + notices = "\n".join(PLUGIN.plan_notices(_job_contract(trigger))) + assert "is created paused" in notices and "main.cp360.profile" in notices + assert "next fluid apply" in notices and "run-now" in notices + + +def test_a_job_is_unpaused_once_the_live_bindings_match(workspace_client): + fake = workspace_client + seed(fake) + assert _emitted_job(_job_contract())["schedule"]["pause_status"] == "PAUSED" + Run(fake).pre(contract()) + job = _emitted_job(_job_contract()) + assert job["schedule"]["pause_status"] == "UNPAUSED" + assert not any("is created paused" in n for n in PLUGIN.plan_notices(_job_contract())) + + +def test_a_job_stays_paused_while_a_binding_is_missing_or_differs(workspace_client): + fake = workspace_client + seed(fake) + Run(fake).pre(contract()) + del fake.tables[TABLE]["row_filter"] + assert _emitted_job(_job_contract())["schedule"]["pause_status"] == "PAUSED" + Run(fake).pre(contract()) + assert _emitted_job(_job_contract())["schedule"]["pause_status"] == "UNPAUSED" + Run(fake).pre(contract(filters=[{**FILTER, "where": "consent_data_analytics = false"}])) + assert ( + _emitted_job(_job_contract())["schedule"]["pause_status"] == "PAUSED" + ), "another predicate" + + +def test_a_job_stays_paused_while_a_bound_function_is_not_the_expected_one(workspace_client): + fake = workspace_client + seed(fake) + Run(fake).pre(contract()) + name = _expected().masks[0].function.full_name + fake.functions[name]["owner"] = "attacker@example.test" + assert _emitted_job(_job_contract())["schedule"]["pause_status"] == "PAUSED" + + +def test_a_job_is_paused_when_the_live_state_cannot_be_read(workspace_client, monkeypatch): + fake = workspace_client + seed(fake) + Run(fake).pre(contract()) + fake.fail_route("GET", r"/tables/main\.cp360\.profile$", 503, "unavailable") + assert _emitted_job(_job_contract())["schedule"]["pause_status"] == "PAUSED" + fake.failing_routes.clear() + + def no_sdk(): + raise ImportError("databricks") + + monkeypatch.setattr(enforce, "default_client", no_sdk) + assert _emitted_job(_job_contract())["schedule"]["pause_status"] == "PAUSED" + + +def test_a_manual_job_and_a_job_writing_no_restricted_table_are_left_alone(workspace_client): + fake = workspace_client + manual = _emitted_job(_job_contract({"type": "manual"})) + assert "schedule" not in manual and "continuous" not in manual + plain = contract(restrictions=None, filters=None) + assert _emitted_job(_job_contract(doc=plain))["schedule"]["pause_status"] == "UNPAUSED" + assert fake.requests == [], "no live read when nothing restricted is written by a job" + + +def test_pure_compilation_never_reads_the_workspace_or_pauses(fake): + compiled = dbx._compile(_job_contract(), {dbx.WAREHOUSE_ENV: "wh"}) + (job,) = compiled.resources["databricks_job"].values() + assert job["schedule"]["pause_status"] == "UNPAUSED" + assert fake.requests == [] diff --git a/tests/iac/test_iac_databricks_fake_e2e.py b/tests/iac/test_iac_databricks_fake_e2e.py index 8d49af42e..35bef8651 100644 --- a/tests/iac/test_iac_databricks_fake_e2e.py +++ b/tests/iac/test_iac_databricks_fake_e2e.py @@ -34,8 +34,17 @@ only its assignment without ``--allow-data-loss``, and an assignment already in the workspace (state lost, or set by hand) is adopted, not a failed create. +A contract's column restrictions and row filters become Unity Catalog SQL functions bound on +the table, run on the warehouse outside OpenTofu (``databricks_enforce.py``): apply binds the +filter and masks, a second apply executes no statement, a table that already exists is bound +before ``tofu apply`` can grant on it, a changed predicate binds a new function, a removed +restriction is dropped, a binding that fails revokes SELECT from the restricted principals and +fails the run (and the next apply binds first and grants again), and a failure before the apply +stops it with nothing applied. + Not proven here: what real Databricks accepts (SQL dialect, warehouse state, -privilege names, Unity Catalog storage); that is the manual live tier. +privilege names, Unity Catalog storage, that CREATE FUNCTION IF NOT EXISTS and SET MASK / SET +ROW FILTER behave as the stand-in models them); that is the manual live tier. Skipped unless ``tofu`` is on PATH; ``tofu init`` needs the registry, a provider cache or a filesystem mirror for ``databricks/databricks``. @@ -43,6 +52,7 @@ from __future__ import annotations +import importlib.util from pathlib import Path from typing import Iterator, List @@ -526,3 +536,437 @@ def test_diff_initialises_with_the_committed_lock(workspace, fake, capfd): rc, out = run_fluid(capfd, "diff", path) assert rc == 0, out assert lock.read_text() == dependency_lock(IAC_PLUGINS["databricks"]) + + +# ── column masks and row filters ──────────────────────────────────────────── +def _sdk_missing() -> bool: + try: + return importlib.util.find_spec("databricks.sdk") is None + except ModuleNotFoundError: + return True + + +requires_sdk = pytest.mark.skipif( + _sdk_missing(), reason="databricks-sdk (the [databricks] extra) is not installed" +) + +EU = "region = 'eu'" + + +def _restricted( + *, + where: str = EU, + deny: bool = True, + mask: bool = True, + row_filter: bool = True, + readers=("group:analysts", "group:bi"), + columns=None, +) -> dict: + """``orders`` with ``amount`` denied and ``region`` masked to the analysts, who also read + only some rows. ``bi`` is not restricted.""" + contract = make_contract(readers=readers, **({"columns": columns} if columns else {})) + authz = contract["exposes"][0]["policy"]["authz"] + restrictions = [] + if deny: + restrictions.append( + {"principal": "group:analysts", "columns": ["amount"], "access": "deny"} + ) + if mask: + restrictions.append( + { + "principal": "group:analysts", + "columns": ["region"], + "access": "mask", + "mask": "last_four", + } + ) + if restrictions: + authz["columnRestrictions"] = restrictions + if row_filter: + authz["rowFilters"] = [{"principal": "group:analysts", "where": where}] + return contract + + +def _live(fake: FakeDatabricks) -> dict: + return fake.tables[TABLE] + + +def _masks(fake: FakeDatabricks) -> dict: + return {c["name"]: c["mask"]["function_name"] for c in _live(fake)["columns"] if c.get("mask")} + + +def _brownfield(fake: FakeDatabricks) -> None: + fake.create_table(TABLE, LIVE_COLUMNS, properties={"delta.columnMapping.mode": "name"}) + + +@requires_sdk +def test_apply_binds_the_filter_and_masks_and_a_second_apply_executes_no_statement( + workspace, fake, capfd +): + path = write_contract(workspace, _restricted()) + + rc, out = apply_contract(capfd, path) + assert rc == 0, out + row_filter = _live(fake)["row_filter"] + assert row_filter["function_name"].startswith("main.sales.fluid_rf_orders_") + assert row_filter["input_column_names"] == ["region"] + assert sorted(_masks(fake)) == ["amount", "region"] + assert all(f in fake.functions for f in [row_filter["function_name"], *_masks(fake).values()]) + # The table was created by this apply, so everything was bound after it, empty. + assert fake.grants[("table", TABLE)] == {"analysts": ["SELECT"], "bi": ["SELECT"]} + assert "masks and row filter" in out and fake.unknown == [] + + statements = len(fake.sql_log) + rc, out = apply_contract(capfd, path) + assert rc == 0, out + assert "tofu plan: +0 ~0 -0" in out + assert len(fake.sql_log) == statements, "a second apply executes no statement" + assert fake.unknown == [] + + +@requires_sdk +def test_a_table_that_exists_is_bound_before_the_apply_can_grant_on_it(workspace, fake, capfd): + _brownfield(fake) + rc, out = apply_contract(capfd, write_contract(workspace, _restricted())) + assert rc == 0, out + assert "row_filter" in _live(fake) and len(_masks(fake)) == 2 + assert fake.replaced == [] + + timeline = fake.timeline + last_bind = max( + i for i, (kind, what) in enumerate(timeline) if kind == "sql" and "SET " in what + ) + first_grant = min( + i for i, (kind, what) in enumerate(timeline) if kind == "perm" and what.endswith("orders") + ) + assert last_bind < first_grant, "every binding was made before the first grant on the table" + + +@requires_sdk +def test_a_dry_run_prints_the_statements_and_executes_none(workspace, fake, capfd): + _brownfield(fake) + rc, out = apply_contract(capfd, write_contract(workspace, _restricted()), "--mode", "dry-run") + assert rc == 0, out + assert "before apply: bind row filter" in out and "SET ROW FILTER" in out + assert "dry-run: none of these statements ran." in out and "dry-run: plan only" in out + assert fake.sql_log == [] and fake.functions == {} and "row_filter" not in _live(fake) + + +@requires_sdk +def test_a_changed_predicate_binds_a_new_function_and_keeps_the_old_one(workspace, fake, capfd): + assert apply_contract(capfd, write_contract(workspace, _restricted()))[0] == 0 + old = _live(fake)["row_filter"]["function_name"] + masks = _masks(fake) + + rc, out = apply_contract(capfd, write_contract(workspace, _restricted(where="region = 'uk'"))) + assert rc == 0, out + new = _live(fake)["row_filter"]["function_name"] + assert new != old and old in fake.functions and new in fake.functions + assert _masks(fake) == masks + assert not any(s.startswith("DROP") for s in fake.sql_log) + assert not any("REPLACE FUNCTION" in s for s in fake.sql_log) + + +@requires_sdk +def test_a_removed_restriction_is_dropped_and_said_to_widen_access(workspace, fake, capfd): + assert apply_contract(capfd, write_contract(workspace, _restricted()))[0] == 0 + + rc, out = apply_contract( + capfd, write_contract(workspace, _restricted(mask=False, row_filter=False)) + ) + assert rc == 0, out + assert "access widened (no data lost, not gated): drop row filter" in out + assert "access widened (no data lost, not gated): drop mask" in out + assert "row_filter" not in _live(fake) + assert sorted(_masks(fake)) == ["amount"] + + rc, out = apply_contract( + capfd, write_contract(workspace, _restricted(deny=False, mask=False, row_filter=False)) + ) + assert rc == 0, out + assert _masks(fake) == {} and "row_filter" not in _live(fake) + statements = len(fake.sql_log) + assert apply_contract(capfd, write_contract(workspace, make_contract()))[0] == 0 + assert len(fake.sql_log) == statements + + +@requires_sdk +def test_a_failed_binding_with_no_deny_all_locks_the_table_and_the_next_apply_fixes_it( + workspace, fake, capfd +): + path = write_contract(workspace, _restricted()) + # Neither the real filter nor the deny-all filter can be bound. + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] warehouse stopped") + + rc, out = apply_contract(capfd, path) + assert rc == 1, out + assert "opentofu_access_enforcement_failed" in out + assert "warehouse stopped" in out and "every grantee but its owner" in out + assert "revoked: REVOKE SELECT ON TABLE `main`.`sales`.`orders` FROM `analysts`" in out + assert fake.grants[("table", TABLE)]["analysts"] == [] + assert fake.grants[("table", TABLE)]["bi"] == [], "revoked from every grantee but the owner" + assert "row_filter" not in _live(fake) + + fake.failing.clear() + mark = len(fake.timeline) + rc, out = apply_contract(capfd, path) + assert rc == 0, out + assert "row_filter" in _live(fake) and len(_masks(fake)) == 2 + assert fake.grants[("table", TABLE)] == {"analysts": ["SELECT"], "bi": ["SELECT"]} + again = fake.timeline[mark:] + bound = next(i for i, (k, w) in enumerate(again) if k == "sql" and "SET ROW FILTER" in w) + granted = next(i for i, (k, w) in enumerate(again) if k == "perm" and w.endswith("orders")) + assert bound < granted, "the table is bound before the grant is put back" + + +@requires_sdk +def test_a_failed_mask_leaves_a_deny_all_filter_and_every_grant_in_place_until_the_next_apply( + workspace, fake, capfd +): + path = write_contract(workspace, _restricted()) + fake.fail_on("SET MASK", "[INTERNAL_ERROR] warehouse stopped") + + rc, out = apply_contract(capfd, path) + assert rc == 1, out + assert "A deny-all row filter (main.sales.fluid_rf_orders_deny_all_" in out + assert "SELECT was not revoked from anyone" in out + deny_all = _live(fake)["row_filter"] + assert deny_all["function_name"].startswith("main.sales.fluid_rf_orders_deny_all_") + assert deny_all["input_column_names"] == [] + assert fake.grants[("table", TABLE)] == {"analysts": ["SELECT"], "bi": ["SELECT"]} + assert not any(s.startswith("REVOKE") for s in fake.sql_log) + + fake.failing.clear() + rc, out = apply_contract(capfd, path) + assert rc == 0, out + assert _live(fake)["row_filter"]["function_name"].startswith("main.sales.fluid_rf_orders_") + assert "deny_all" not in _live(fake)["row_filter"]["function_name"] + assert len(_masks(fake)) == 2 + + +@requires_sdk +def test_a_failure_before_the_apply_stops_it_with_nothing_applied(workspace, fake, capfd): + _brownfield(fake) + assert apply_contract(capfd, write_contract(workspace, _restricted()))[0] == 0 + # A new reader, and a changed predicate whose function cannot be made. + fake.fail_on("CREATE FUNCTION", "[INSUFFICIENT_PERMISSIONS] cannot create a function") + contract = _restricted( + where="region = 'uk'", readers=("group:analysts", "group:bi", "group:new") + ) + + rc, out = apply_contract(capfd, write_contract(workspace, contract)) + assert rc == 1, out + assert "opentofu_access_enforcement_failed" in out and "the apply did not run" in out + assert "new" not in fake.grants[("table", TABLE)], "tofu apply never ran" + assert not any(s.startswith("REVOKE") for s in fake.sql_log) + assert fake.grants[("table", TABLE)]["analysts"] == ["SELECT"] + + +@requires_sdk +def test_a_contract_with_an_invalid_predicate_is_refused_before_anything_runs( + workspace, fake, capfd +): + contract = _restricted(where="region = 'eu'; DROP TABLE main.sales.orders") + rc, out = apply_contract(capfd, write_contract(workspace, contract)) + assert rc == 1 and "databricks-row-filter-invalid" in out + assert fake.sql_log == [] and fake.tables == {} + + +@requires_sdk +def test_an_apply_that_fails_after_creating_the_table_still_binds_it_and_fails_with_the_apply_error( + workspace, fake, capfd +): + # The table is created, then the grant on it fails: a later run of a job that fills the + # table would read it with no mask or filter if the failed apply stopped there. + fake.fail_route("PATCH", "/permissions/table/", message="grant refused") + rc, out = apply_contract(capfd, write_contract(workspace, _restricted())) + + assert rc == 1, out + assert "opentofu_apply_failed" in out, "the run fails with the apply's own error" + assert "opentofu_access_enforcement_failed" not in out + assert TABLE in fake.tables, "the failed apply did create the table" + assert "row_filter" in _live(fake) and sorted(_masks(fake)) == ["amount", "region"] + assert "binding the masks and row filter on the tables that exist" in out + assert not any(s.startswith(("DROP", "REVOKE")) for s in fake.sql_log) + + +@requires_sdk +def test_an_apply_that_fails_before_creating_the_table_binds_nothing_and_says_nothing_alarming( + workspace, fake, capfd +): + fake.fail_route("POST", "/unity-catalog/schemas", message="no schema for you") + rc, out = apply_contract(capfd, write_contract(workspace, _restricted())) + + assert rc == 1 and "opentofu_apply_failed" in out, out + assert TABLE not in fake.tables and fake.sql_log == [] + assert "not created by the failed apply" in out + assert "NOT REVOKED" not in out + + +@requires_sdk +def test_a_failed_apply_never_drops_a_binding_the_contract_no_longer_asks_for( + workspace, fake, capfd +): + assert apply_contract(capfd, write_contract(workspace, _restricted()))[0] == 0 + before = _live(fake)["row_filter"]["function_name"] + mark = len(fake.sql_log) + + # No restriction now, and a new reader whose grant fails: dropping the filter would widen + # access, and waits for an apply that succeeds. + fake.fail_route("PATCH", "/permissions/table/", message="grant refused") + contract = _restricted( + mask=False, + deny=False, + row_filter=False, + readers=("group:analysts", "group:bi", "group:new"), + ) + rc, out = apply_contract(capfd, write_contract(workspace, contract)) + assert rc == 1 and "opentofu_apply_failed" in out, out + assert _live(fake)["row_filter"]["function_name"] == before + assert sorted(_masks(fake)) == ["amount", "region"] + assert not any(s.startswith("DROP") for s in fake.sql_log[mark:]) + assert "access widened" not in out + + fake.failing_routes.clear() + rc, out = apply_contract(capfd, write_contract(workspace, contract)) + assert rc == 0, out + assert "row_filter" not in _live(fake) and _masks(fake) == {}, "dropped once it succeeded" + + +@requires_sdk +def test_a_function_pre_created_under_the_hashed_name_stops_the_apply_before_anything_is_applied( + workspace, fake, capfd +): + from fluid_build.iac.providers import databricks as dbx + + _brownfield(fake) + contract = _restricted() + path = write_contract(workspace, contract) + enforcement = ( + dbx._compile(contract, {dbx.WAREHOUSE_ENV: fake.warehouse_id}).tables[0].enforcement + ) + mask = enforcement.masks[0] + # Someone with CREATE FUNCTION took the name, with a body that returns the value as it is. + fake.add_function( + mask.function.full_name, + [("val", mask.function.params[0][1])], + mask.function.returns, + "`val`", + ) + + rc, out = apply_contract(capfd, path) + + assert rc == 1, out + assert "opentofu_access_enforcement_failed" in out and "It was not bound" in out + assert "attacker@example.test" in out + assert mask.column not in _masks(fake), "the hostile function was never bound" + assert ("table", TABLE) not in fake.grants, "tofu apply never ran: nothing was granted" + + +@requires_sdk +def test_a_row_filter_set_by_hand_stops_the_apply_and_is_left_in_place(workspace, fake, capfd): + _brownfield(fake) + from fluid_build.iac.providers.databricks_enforce import SqlRunner + + sql = SqlRunner(fake.client(), fake.warehouse_id) + theirs = "main.sales.admin_rows" + sql.run( + "CREATE FUNCTION `main`.`sales`.`admin_rows`(`region` STRING) RETURNS BOOLEAN RETURN true" + ) + sql.run( + "ALTER TABLE `main`.`sales`.`orders` SET ROW FILTER `main`.`sales`.`admin_rows` " + "ON (`region`)" + ) + rc, out = apply_contract(capfd, write_contract(workspace, _restricted())) + assert rc == 1, out + assert "opentofu_access_enforcement_failed" in out and "did not create" in out + assert theirs in out and "Nothing was changed" in out + assert _live(fake)["row_filter"]["function_name"] == theirs + assert ("table", TABLE) not in fake.grants, "tofu apply never ran" + + +@requires_sdk +def test_a_filter_needing_a_column_the_apply_adds_is_held_by_a_deny_all_filter_before_it( + workspace, fake, capfd +): + _brownfield(fake) # the table holds rows and has no ``channel`` column yet + contract = _restricted( + where="channel = 'web'", columns=COLUMNS + [("channel", "string", False)] + ) + rc, out = apply_contract(capfd, write_contract(workspace, contract)) + assert rc == 0, out + row_filter = _live(fake)["row_filter"] + assert row_filter["input_column_names"] == ["channel"] + assert "deny_all" not in row_filter["function_name"], "the real filter replaced the interim one" + + timeline = fake.timeline + interim = next( + i + for i, (k, w) in enumerate(timeline) + if k == "sql" and "SET ROW FILTER" in w and "deny_all" in w + ) + first_grant = min( + i for i, (k, w) in enumerate(timeline) if k == "perm" and w.endswith("orders") + ) + add_column = next(i for i, (k, w) in enumerate(timeline) if k == "sql" and "ADD COLUMN" in w) + real = max(i for i, (k, w) in enumerate(timeline) if k == "sql" and "SET ROW FILTER" in w) + assert interim < add_column < real, "deny-all, then the new column, then the real filter" + assert interim < first_grant + + +def test_generate_iac_refuses_a_module_a_plain_tofu_apply_would_leave_unmasked( + workspace, fake, capfd +): + path = write_contract(workspace, _restricted()) + + rc, out = run_fluid(capfd, "generate", "iac", path, "--out", "iac") + assert rc == 1, out + assert "databricks-enforcement-outside-module" in out + assert "plain tofu apply" in out and "fluid apply" in out + assert not (workspace / "iac" / "main.tf.json").exists(), "no module was written" + + # validate, plan and a dry-run apply are unaffected, and so is a contract without any. + assert run_fluid(capfd, "validate", path)[0] == 0 + assert run_fluid(capfd, "plan", path, "--out", "plan.json")[0] == 0 + plain = write_contract(workspace, make_contract()) + rc, out = run_fluid(capfd, "generate", "iac", plain, "--out", "iac") + assert rc == 0, out + assert (workspace / "iac" / "main.tf.json").is_file() + + +@requires_sdk +def test_a_dbt_job_created_with_its_restricted_table_is_paused_until_the_next_apply( + workspace, fake, capfd +): + contract = _restricted() + contract["builds"] = [ + { + "id": "transform", + "engine": "dbt", + "pattern": "hybrid-reference", + "repository": "https://github.com/o/r?tag=v1", + "outputs": ["orders"], + "properties": {"model": "orders", "select": "orders+"}, + "execution": { + "trigger": {"type": "schedule", "cron": "*/5 * * * *"}, + "runtime": {"platform": "databricks"}, + }, + } + ] + path = write_contract(workspace, contract) + + rc, out = apply_contract(capfd, path) + assert rc == 0, out + (job,) = fake.jobs.values() + assert job["settings"]["schedule"]["pause_status"] == "PAUSED" + assert "is created paused" in out and "run-now" in out + assert "row_filter" in _live(fake), "the table was bound in the same apply" + + rc, out = apply_contract(capfd, path) + assert rc == 0, out + assert "tofu plan: +0 ~1 -0" in out, "only the job changes: it is unpaused" + (job,) = fake.jobs.values() + assert job["settings"]["schedule"]["pause_status"] == "UNPAUSED" + assert "is created paused" not in out + + assert "tofu plan: +0 ~0 -0" in apply_contract(capfd, path, "--mode", "dry-run")[1] From edafd66b806c74d17529c78751c8b6b2c185a529 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:31:39 +0100 Subject: [PATCH 04/26] chore(ci): bump reviewdog/action-actionlint from 1.80.6 to 1.81.1 (#782) Bumps [reviewdog/action-actionlint](https://github.com/reviewdog/action-actionlint) from 1.80.6 to 1.81.1. - [Release notes](https://github.com/reviewdog/action-actionlint/releases) - [Commits](https://github.com/reviewdog/action-actionlint/compare/48e36f54d8da326b83ab061aee2cc5e01b5ca46f...619a81ec64ff8c0211cd6146eb2b21e8f1d7f440) --- updated-dependencies: - dependency-name: reviewdog/action-actionlint dependency-version: 1.81.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/actionlint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index 00f6adf80..d764510b0 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -30,7 +30,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Run actionlint - uses: reviewdog/action-actionlint@48e36f54d8da326b83ab061aee2cc5e01b5ca46f # v1.80.6 + uses: reviewdog/action-actionlint@619a81ec64ff8c0211cd6146eb2b21e8f1d7f440 # v1.81.1 with: fail_level: error From f322f8b9dcd1ebce4a38817cd826593d567de8cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:40:01 +0100 Subject: [PATCH 05/26] chore(ci): bump github/codeql-action init and analyze to 4.39.1 together (#745) Dependabot split the 4.39.1 bump into #745 (init) and #747 (analyze). CodeQL fails any run whose steps are on different versions, so each PR alone was red on all three Analyze jobs. This one now bumps both to the same pinned SHA (9b3254085494b55a0f7c4c2b3d2d52cc712ce370, checked against the upstream v4.39.1 tag), and dependabot.yml groups github/codeql-action* so init, analyze and upload-sarif arrive in one PR from now on. --- .github/dependabot.yml | 8 ++++++++ .github/workflows/codeql.yml | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index db68620d8..ffd83f2d6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,3 +19,11 @@ updates: prefix: "chore(ci)" labels: - ci + groups: + # Dependabot treats codeql-action's init, analyze and upload-sarif as + # separate dependencies and bumps each in its own PR. CodeQL fails any + # run whose steps are on different versions ("Loaded a configuration + # file for version X, but running version Y"), so they move together. + codeql-action: + patterns: + - "github/codeql-action*" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7a7e63a56..9cccf9fb0 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -38,12 +38,12 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: github/codeql-action/init@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + - uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: languages: ${{ matrix.language }} build-mode: none queries: security-extended config-file: ./.github/codeql/codeql-config.yml - - uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + - uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: category: "/language:${{ matrix.language }}" From c929542ee64bdd0a3dbb2338b29ace089ac34a9b Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:07:03 +0100 Subject: [PATCH 06/26] fix: doctor --strict, an honest roadmap, and a readable, redacted OpenTofu apply failure (#786) * feat(doctor): --strict exits 1 when doctor reports "Action needed" fluid doctor printed "Action needed" for a copilot that is not ready and still exited 0, so a CI gate on doctor could not fail. With --strict it exits 1 whenever the summary says "Action needed", in the text report and with --json, whose payload now carries action_needed and copilot_ready. The default exit code is unchanged; --strict has no effect with --features-only, which prints no summary. Found by an external hands-on assessment of 0.26.0. * fix(roadmap): stop claiming a 1.0 release; name every provider in the description fluid roadmap printed "forge-cli v1.0 shipped 2026-04-23" while the package is 0.x and classified Beta. The roadmap now says so and calls its "v1" headings an internal milestone, and a test fails if it announces a 1.x release while the package is below 1.0. The package description named GCP, Snowflake and Local; it now also names AWS and Databricks. Found by an external hands-on assessment of 0.26.0. * fix(apply): a failed OpenTofu apply names its diagnostic and the way on, redacted The runner uses -json, so the error a failed apply printed was the last 800 characters of diagnostic JSON lines, with no hint. The error now reads the first error diagnostic (summary, resource address, provider detail), falls back to the first "Error:" or the exit code, and says that OpenTofu records state as it goes, so fixing the cause and re-running fluid apply normally continues; it names the workdir for `tofu state list`, and only a state-lock error points at `tofu force-unlock`. The raw tail goes to --debug. Every value a CLIError prints is now redacted and has its control characters spelled out, as the other error handlers already did: the new detail field carries provider text, which can hold a credential. Found by an external hands-on assessment of 0.26.0. * fix(cli): error values go through the full secret redactor; a locked plan says how to clear it The top-level CLIError handler ran context values through _console's narrow pattern only (bare password/token/secret/api key followed by : or =), which leaves an AKIA access key id, a Bearer token, a URL password, JSON-quoted and *_secret_key shapes and env-literal secrets in clear. Context values and suggestions now go through redact_secret_text first, the redactor the logs already use. _apply_failure redacts the diagnostic before cutting it at 800 characters, since a credential split at the limit (a URL that lost its @) matches no pattern. A stale state lock first surfaces at tofu plan, which printed only the raw tail; _plan_failure now redacts it and, for a state-lock error only, says to wait for the other run or clear it with tofu force-unlock in the named workdir. Found by the security review of this branch. * test: allowlist the deliberate fake credentials the secret scan flags detect-secrets failed Lint & Format on the AWS-example key and the admin:s3cretpw DSN these redaction tests feed in on purpose. Each value now carries `# pragma: allowlist secret` on every line that holds it, not just the first: detect-secrets reports one finding per value per file, so allowlisting only lines 690/696 and 412/413 would have surfaced the copies at 695/701 and 424 on the next run. Two lines were pre-split so the pragma stays on the value. With the comment appended, black would otherwise rewrap them and leave the pragma on a closing bracket, where detect-secrets does not look. --- CHANGELOG.md | 21 +++ fluid_build/_error_catalog.py | 9 ++ fluid_build/cli/__init__.py | 16 ++- fluid_build/cli/_apply_opentofu_engine.py | 86 +++++++++++- fluid_build/cli/doctor.py | 24 +++- fluid_build/copilot/roadmap.md | 20 +-- pyproject.toml | 2 +- .../test_apply_reports_to_command_center.py | 131 ++++++++++++++++++ tests/cli/test_cli_output_keeps_brackets.py | 38 +++++ tests/iac/test_iac_databricks_enforce.py | 2 +- tests/test_doctor_strict.py | 63 +++++++++ tests/test_roadmap_claims.py | 55 ++++++++ 12 files changed, 445 insertions(+), 22 deletions(-) create mode 100644 tests/test_doctor_strict.py create mode 100644 tests/test_roadmap_claims.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cb39e912..65fd3d436 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **`fluid doctor --strict` turns the "Action needed" summary into exit 1.** `fluid doctor` printed + "Action needed" for a copilot that is not ready and still exited 0, so a CI gate could not tell. + With `--strict` it exits 1 whenever the summary says "Action needed", in the text report and with + `--json`, whose payload now carries `action_needed` and `copilot_ready`. The default exit code is + unchanged, and `--strict` has no effect with `--features-only`, which prints no summary. + - **`fluid publish` tells the Command Center which workspace holds a Databricks table.** The product's `metadata.workspace_host` is now `https://[:port]` taken from `DATABRICKS_HOST`, normalised and checked as the provider checks it (https, or plain http only to a loopback @@ -126,6 +132,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **A failed `fluid apply` on OpenTofu printed raw JSON and no way forward.** The runner uses + `-json`, so the error was the last 800 characters of diagnostic lines. It now reads the + diagnostic (the summary, the resource address and the provider's detail) and says that OpenTofu + records state as it goes, so fixing the cause and re-running `fluid apply` normally continues; + `tofu state list` in the working directory shows what exists, and a state-lock error also + points at `tofu force-unlock`, as does a plan that fails on a held state lock. The raw output + tail is logged under `--debug`. Every value a `CLIError` prints (context and suggestions) now + goes through the full secret redactor, so a provider message carrying an access key id, a + bearer token or a URL password is masked, and has its control characters spelled out, as the + other error handlers already did. +- **`fluid roadmap` claimed a 1.0 release and the package description left providers out.** The + roadmap said "forge-cli v1.0 shipped"; the package is 0.x Beta, and the roadmap now says so and + calls its "v1" headings an internal milestone. A test fails if it announces a 1.x release while + the package is below 1.0. The package description named GCP, Snowflake and Local and now names + AWS and Databricks too. - **A contract's governance labels were dropped silently on Databricks.** The Databricks emitter never read `labels`, so `data-classification: confidential`, the `gdpr-article`, and the sovereignty jurisdiction, regulation and residency reached AWS (tags, Glue parameters) and GCP diff --git a/fluid_build/_error_catalog.py b/fluid_build/_error_catalog.py index 7e9a7bd16..30c31226f 100644 --- a/fluid_build/_error_catalog.py +++ b/fluid_build/_error_catalog.py @@ -277,6 +277,15 @@ def slug_for(event: str) -> str: ["Run 'fluid plan ' and inspect the emitted plan for the failing action"], None, ), + "opentofu_apply_failed": ( + [ + "OpenTofu records state as it goes, so resources created before the failure are " + + "normally tracked", + "Fix the cause named above, then re-run 'fluid apply' to continue", + "Re-run with --debug to see the raw tofu output", + ], + None, + ), "opentofu_adoption_failed": ( [ "Check the provider credentials and network access, then re-run 'fluid apply'", diff --git a/fluid_build/cli/__init__.py b/fluid_build/cli/__init__.py index f65db246a..6e4c65356 100644 --- a/fluid_build/cli/__init__.py +++ b/fluid_build/cli/__init__.py @@ -35,6 +35,7 @@ # transitive deps back into this package. from fluid_build.observability.secret_redactor import ( install_secret_redacting_filter, + redact_secret_text, register_secrets_from_environ, ) @@ -467,20 +468,31 @@ def main(argv: Optional[List[str]] = None) -> int: # ``tofu state rm
`` line runs it without its address. from rich.text import Text + from fluid_build._console import _redact_str, literal_text + message = Text(f"❌ {e.message}", style="red") if slug: message.append(f" [{slug}]", style="dim") context.console.print(message) if e.context: for key, value in e.context.items(): + # Provider text (a tofu diagnostic's detail) lands here: redacted + # and with control characters spelled out, as the handlers below. context.console.print( - Text.assemble(" ", (f"{key}:", "dim"), f" {value}"), soft_wrap=True + Text.assemble( + " ", + (f"{key}:", "dim"), + literal_text(_redact_str(redact_secret_text(f" {value}"))), + ), + soft_wrap=True, ) suggestions = getattr(e, "suggestions", None) or [] if suggestions: context.console.print("\n[yellow]💡 Suggestions:[/yellow]") for suggestion in suggestions: - context.console.print(Text(f" • {suggestion}")) + context.console.print( + literal_text(_redact_str(redact_secret_text(f" • {suggestion}"))) + ) docs_url = getattr(e, "docs_url", None) if docs_url: # soft_wrap: a docs link is a URL, and Rich word-wraps at the terminal diff --git a/fluid_build/cli/_apply_opentofu_engine.py b/fluid_build/cli/_apply_opentofu_engine.py index 584d301f0..d20b70a50 100644 --- a/fluid_build/cli/_apply_opentofu_engine.py +++ b/fluid_build/cli/_apply_opentofu_engine.py @@ -28,6 +28,7 @@ import json import logging +import shlex from contextlib import contextmanager from dataclasses import dataclass, field, replace from pathlib import Path @@ -56,6 +57,7 @@ records_backend, ) from fluid_build.iac.state_migration import reconcile_state_key as _reconcile_state +from fluid_build.observability.secret_redactor import redact_secret_text from ._apply_cc_report import current_report from ._common import CLIError, load_contract_with_overlay, resolve_env_templates_in_contract @@ -314,11 +316,11 @@ def _apply_via_opentofu(args, logger: logging.Logger, contract: Optional[Mapping # A table the failed apply created, its grants and the job that fills it can all # exist with no mask or filter: bind what exists (never drop) before failing. _enforce_after_failed_apply(plugin, contract) - raise CLIError( - 1, - "opentofu_apply_failed", - {"error": _tail(apply_result.stderr or apply_result.stdout)}, + logger.debug( + "opentofu apply raw output tail: %s", + _tail(apply_result.stderr or apply_result.stdout), ) + raise _apply_failure(apply_result, str(workdir)) applied = runner.change_summary(apply_result) record(applied) @@ -1137,7 +1139,17 @@ def _plan_failure( explained = _explained_failure(plugin, contract, output, workdir, env) if explained is not None: return explained - return CLIError(1, "opentofu_plan_failed", {"error": _tail(plan.stderr or plan.stdout)}) + err = CLIError( + 1, + "opentofu_plan_failed", + {"error": _tail(redact_secret_text(plan.stderr or plan.stdout or ""))}, + ) + if _mentions_state_lock(output): + err.suggestions = list(err.suggestions) + [ + "The state is locked: wait for the other run, or clear a stale lock with " + f"'tofu force-unlock ' in {shlex.quote(workdir)}" + ] + return err def _guard_region_move( @@ -1532,6 +1544,70 @@ def _tail(text: str, limit: int = 800) -> str: return (text or "")[-limit:] +def _diagnostics(events: Any) -> List[Mapping[str, Any]]: + """The error diagnostics in a ``tofu -json`` stream, in the order printed.""" + found: List[Mapping[str, Any]] = [] + for event in events or []: + diag = event.get("diagnostic") if isinstance(event, Mapping) else None + if isinstance(diag, Mapping) and diag.get("severity", "error") == "error": + found.append(diag) + return found + + +def _mentions_state_lock(text: str) -> bool: + """True when tofu says the state lock is the problem. + + Matched on the phrases tofu prints ("Error acquiring the state lock", + "Error locking state"), never on a bare "lock", which "block" contains. + """ + low = text.lower() + return "state lock" in low or "locking state" in low + + +def _apply_failure(result: Any, workdir: str) -> CLIError: + """The error for a failed ``tofu apply``: what failed, on which resource, and the way on. + + ``-json`` puts every diagnostic on stdout as one JSON line, so the raw tail + was a run of ``{"@level":...}`` objects. The diagnostics are read back into + a summary, the resource address and the provider's detail; output that holds + none (a crash, a timeout) falls back to the first ``Error:``, else the exit + code. Tofu records state as it goes, so what was created before the failure + is normally tracked and a re-run converges: the suggestions say so. + """ + diagnostics = _diagnostics(getattr(result, "events", None)) + context: Dict[str, Any] = {} + locked = False + if diagnostics: + first = diagnostics[0] + # Redacted before it is cut: a credential split at the limit (a URL that lost + # its ``@``) matches no pattern. + address = redact_secret_text(str(first.get("address") or "").strip()) + detail = redact_secret_text(str(first.get("detail") or "").strip()) + summary = redact_secret_text(str(first.get("summary") or "").strip()) or "tofu apply failed" + context["error"] = summary + (f" ({address})" if address else "") + if address: + context["resource"] = address + if detail: + context["detail"] = detail[:800] + if len(diagnostics) > 1: + context["more_errors"] = len(diagnostics) - 1 + locked = _mentions_state_lock(summary + " " + detail) + else: + output = result.stderr or result.stdout or "" + context["error"] = ( + _first_error(redact_secret_text(output), 800) + if "Error:" in output + else f"tofu apply exited {getattr(result, 'returncode', 1)}" + ) + locked = _mentions_state_lock(output) + err = CLIError(1, "opentofu_apply_failed", context) + hints = [f"To see what exists, run 'tofu state list' in {shlex.quote(workdir)}"] + if locked: + hints.append("The state is locked: wait for the other run, or 'tofu force-unlock '") + err.suggestions = list(err.suggestions) + hints + return err + + def _first_error(text: str, limit: int = 400) -> str: """``tofu`` output from its first ``Error:`` on, where the cause is stated. diff --git a/fluid_build/cli/doctor.py b/fluid_build/cli/doctor.py index d4a99f4f8..003bee39b 100644 --- a/fluid_build/cli/doctor.py +++ b/fluid_build/cli/doctor.py @@ -470,6 +470,15 @@ def register(subparsers: argparse._SubParsersAction): # The previous gap (UX audit finding H9): every kill switch was # documented only in CLAUDE.md and source docstrings, never on the # CLI itself. + p.add_argument( + "--strict", + action="store_true", + help=( + "Exit 1 when the summary is 'Action needed' (any built-in check not ready, " + "including copilot readiness); by default only a failed feature check does. " + "Ignored with --features-only, which reports feature checks alone" + ), + ) p.add_argument( "--env", action="store_true", @@ -508,6 +517,9 @@ def run(args, logger: logging.Logger) -> int: # backend resolution errors, and never costs the happy path more # than a few ms. store_backend_status = _inspect_store_backend() + # --strict: the "Action needed" summary becomes the exit code, so a copilot + # that is not ready fails a CI gate instead of printing a warning and exiting 0. + needs_action = getattr(args, "strict", False) is True and not copilot_readiness.ready # If features-only mode, just show features and exit if getattr(args, "features_only", False): @@ -523,13 +535,17 @@ def run(args, logger: logging.Logger) -> int: import sys as _sys _json.dump( - {"store_backend": store_backend_status}, + { + "store_backend": store_backend_status, + "copilot_ready": bool(copilot_readiness.ready), + "action_needed": not feature_checks_ok or not copilot_readiness.ready, + }, _sys.stdout, indent=2, default=str, ) _sys.stdout.write("\n") - return 0 if feature_checks_ok else 1 + return 0 if feature_checks_ok and not needs_action else 1 resolved_script = _resolve_extended_diagnostic_script() extended_available = resolved_script is not None @@ -553,7 +569,7 @@ def run(args, logger: logging.Logger) -> int: ) if not extended_requested: - return 0 if feature_checks_ok else 1 + return 0 if feature_checks_ok and not needs_action else 1 if resolved_script is None: raise _extended_diagnostic_error( @@ -602,7 +618,7 @@ def run(args, logger: logging.Logger) -> int: info(logger, "doctor_ok") # Return success only if both feature checks and infrastructure checks pass - return 0 if feature_checks_ok else 1 + return 0 if feature_checks_ok and not needs_action else 1 except subprocess.TimeoutExpired: raise CLIError(1, "doctor_timeout", context={"timeout": 300}) diff --git a/fluid_build/copilot/roadmap.md b/fluid_build/copilot/roadmap.md index 11103ffb7..26f0739e2 100644 --- a/fluid_build/copilot/roadmap.md +++ b/fluid_build/copilot/roadmap.md @@ -1,13 +1,15 @@ # forge-cli roadmap -forge-cli v1.0 shipped 2026-04-23. The Lean v1 scope previewed several v1.1+ -milestones early; they are listed under "Shipped in v1.0" below and have been -retargeted out of the upcoming-milestone track so the banner and `fluid -roadmap` point at the next truly unshipped milestone. +forge-cli is a 0.x Beta release. The "v1.0" milestones below are internal roadmap +labels: the Lean v1 scope was completed 2026-04-23, but no 1.0 package has been +published. The Lean v1 scope previewed several v1.1+ milestones early; they are +listed under "Completed in the Lean v1 milestone" below and have been retargeted out of the +upcoming-milestone track so the banner and `fluid roadmap` point at the next +truly unshipped milestone. -## Shipped in v1.0 (previewed from v1.1+ roadmap) +## Completed in the Lean v1 milestone (previewed from v1.1+ roadmap) -The following milestones were pulled forward into v1.0 and need no further +The following milestones were pulled forward into the Lean v1 milestone and need no further work: - **v1.1 — Team Collaboration** · shared cache backends (`SqliteBackend`, @@ -22,11 +24,11 @@ work: favour of `LogicalAgent`. - **v1.4 — Richer CLI Surface** · `fluid forge data-model diff`, `--emit-dimensional-variants`, `--emit-ddl-dir`, `--emit-osi-sidecar` - (default on), and `--deterministic` all ship in v1.0. + (default on), and `--deterministic` all landed in the Lean v1 milestone. ### v1.0.1 follow-up (landed 2026-04-24) -Sprint-local hardening that shipped the day after v1.0, driven by live +Sprint-local hardening that landed the day after the Lean v1 milestone, driven by live Phase-4 Gemini integration findings: - **Retry envelope on every staged LLM call** — `BaseStageAgent.call()` @@ -140,7 +142,7 @@ below. Directive from the user: "nothing left behind." ### V1 world-class hardening (landed 2026-04-25) Closes the rest of the original-plan gaps that were "shipped without -tests" and pins the v1.0 public API surface so v1.x users can rely on +tests" and pins the Lean v1 public API surface so later 1.x users can rely on their imports surviving any internal refactor. - **Plan gap A2 closed** — `forge_datamodel/dv2/hash_keys.py` and diff --git a/pyproject.toml b/pyproject.toml index 110b9d1ef..e4693e5ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "data-product-forge" dynamic = ["version"] -description = "FLUID Build System: plan/apply/visualize data products across providers (GCP, Snowflake, Local) with policy & contract tests." +description = "FLUID Build System: plan/apply/visualize data products across providers (AWS, GCP, Snowflake, Databricks, Local) with policy & contract tests." readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" diff --git a/tests/cli/test_apply_reports_to_command_center.py b/tests/cli/test_apply_reports_to_command_center.py index b3e8e8a7c..c60e76a41 100644 --- a/tests/cli/test_apply_reports_to_command_center.py +++ b/tests/cli/test_apply_reports_to_command_center.py @@ -602,3 +602,134 @@ def test_a_build_id_that_is_not_in_the_contract_is_the_reason( assert patch["body"]["result"]["error_event"] == "build_not_found:no_such_build" assert patch["body"]["current_phase"] == "build" assert build["calls"] == 0 + + +def test_a_failed_apply_reads_the_json_diagnostic_and_says_how_to_continue( + product, tofu, monkeypatch +): + stream = [ + {"@level": "info", "@message": "google_bigquery_dataset.d: Creation complete"}, + { + "@level": "error", + "type": "diagnostic", + "diagnostic": { + "severity": "error", + "summary": "googleapi: Error 403: Access Denied", + "detail": "Permission bigquery.tables.create denied on dataset d", + "address": "google_bigquery_table.orders", + }, + }, + ] + stdout = "\n".join(json.dumps(e) for e in stream) + monkeypatch.setattr( + runner, "tofu_apply", lambda *a, **k: runner.TofuResult("apply", 1, stdout, "", stream) + ) + + with pytest.raises(CLIError) as exc: + _apply(product) + + assert exc.value.event == "opentofu_apply_failed" + assert exc.value.context["error"] == ( + "googleapi: Error 403: Access Denied (google_bigquery_table.orders)" + ) + assert exc.value.context["resource"] == "google_bigquery_table.orders" + assert "bigquery.tables.create denied" in exc.value.context["detail"] + assert "@level" not in json.dumps(exc.value.context) + suggestions = "\n".join(exc.value.suggestions) + assert "re-run 'fluid apply'" in suggestions + assert "tofu state list" in suggestions + + +@pytest.mark.parametrize( + "result, expected", + [ + (runner.TofuResult("apply", 7, "", "", []), "tofu apply exited 7"), + ( + runner.TofuResult("apply", 1, '{"@level":"warn"}', "", [{"@level": "warn"}]), + "tofu apply exited 1", + ), + ( + runner.TofuResult("apply", 1, "", "Error: state lock held", []), + "Error: state lock held", + ), + ], +) +def test_a_failed_apply_without_a_diagnostic_never_prints_raw_json(result, expected): + err = engine._apply_failure(result, "/work/my dir") + + assert err.context["error"] == expected + assert "'tofu state list' in '/work/my dir'" in "\n".join(err.suggestions) + assert ("force-unlock" in "\n".join(err.suggestions)) == ("lock" in expected) + + +@pytest.mark.parametrize( + "summary, locked", + [ + ("Error acquiring the state lock", True), + ("Error locking state: ConditionalCheckFailed", True), + ("Unsupported block type", False), + ("Missing required argument in backend block", False), + ], +) +def test_only_a_state_lock_error_suggests_force_unlock(summary, locked): + events = [{"diagnostic": {"severity": "error", "summary": summary, "detail": ""}}] + + err = engine._apply_failure(runner.TofuResult("apply", 1, "", "", events), "/w") + + assert ("force-unlock" in "\n".join(err.suggestions)) is locked + + +def test_a_non_string_diagnostic_field_does_not_raise(): + events = [{"diagnostic": {"severity": "error", "summary": 403, "detail": None, "address": 7}}] + + err = engine._apply_failure(runner.TofuResult("apply", 1, "", "", events), "/w") + + assert err.context["error"] == "403 (7)" + + +_LEAKS = ("AKIAIOSFODNN7EXAMPLE", "abcdef0123456789token", "s3cretpw") # pragma: allowlist secret + + +def _leaky() -> str: + return ( + "access key AKIAIOSFODNN7EXAMPLE, Authorization: Bearer abcdef0123456789token, " # pragma: allowlist secret + "dsn postgres://admin:s3cretpw@db.example.com/orders" # pragma: allowlist secret + ) + + +def test_a_failed_apply_diagnostic_is_redacted_before_it_is_cut(): + detail = ( + "x" * 780 + + " postgres://admin:s3cretpw@db.example.com/orders " # pragma: allowlist secret + + _leaky() + ) + events = [{"diagnostic": {"severity": "error", "summary": _leaky(), "detail": detail}}] + + err = engine._apply_failure(runner.TofuResult("apply", 1, "", "", events), "/w") + + shown = json.dumps(err.context) + assert not [leak for leak in _LEAKS if leak in shown] + + +def test_a_failed_apply_without_a_diagnostic_is_redacted(): + err = engine._apply_failure(runner.TofuResult("apply", 1, "", "Error: " + _leaky(), []), "/w") + + assert not [leak for leak in _LEAKS if leak in err.context["error"]] + + +@pytest.mark.parametrize( + "output, locked", + [ + ("Error: Error acquiring the state lock\n" + _leaky(), True), + ("Error: could not read block device " + _leaky(), False), + ], +) +def test_a_failed_plan_is_redacted_and_names_how_to_clear_a_held_lock(output, locked): + plan = runner.TofuResult("plan", 1, "", output, []) + + err = engine._plan_failure(object(), {}, plan, "/work/my dir", {}) + + assert err.event == "opentofu_plan_failed" + assert not [leak for leak in _LEAKS if leak in err.context["error"]] + hint = "\n".join(err.suggestions) + assert ("tofu force-unlock ' in '/work/my dir'" in hint) is locked diff --git a/tests/cli/test_cli_output_keeps_brackets.py b/tests/cli/test_cli_output_keeps_brackets.py index 1e8b5eecf..fc181d7e1 100644 --- a/tests/cli/test_cli_output_keeps_brackets.py +++ b/tests/cli/test_cli_output_keeps_brackets.py @@ -393,6 +393,44 @@ def test_cli_error_message_context_and_suggestions_print_verbatim(monkeypatch, c assert "[dim]" not in out and "[red]" not in out +def test_cli_error_context_is_redacted_and_control_characters_are_spelled_out(monkeypatch, capsys): + """A provider's text reaches the context (a tofu diagnostic's detail): no secret, no escape.""" + err = CLIError(1, "opentofu_apply_failed", {"detail": "login failed password=hunter2\x1b[31m"}) + _raise_from_command(monkeypatch, err) + + rc = cli_mod.main(["validate", "unused.fluid.yaml"]) + + out = capsys.readouterr().out + assert rc == 1 + assert "hunter2" not in out + assert "\x1b" not in out + assert "detail:" in out + + +def test_cli_error_prints_no_credential_shape_the_full_redactor_knows(monkeypatch, capsys): + leaky = ( + "AKIAIOSFODNN7EXAMPLE Authorization: Bearer abcdef0123456789token " # pragma: allowlist secret + "postgres://admin:s3cretpw@db.example.com/orders" # pragma: allowlist secret + ) + err = CLIError(1, "opentofu_apply_failed", {"detail": leaky}) + err.suggestions = [leaky] + _raise_from_command(monkeypatch, err) + + rc = cli_mod.main(["validate", "unused.fluid.yaml"]) + + out = _unwrapped(capsys.readouterr().out) + assert rc == 1 + assert not [ + s + for s in ( + "AKIAIOSFODNN7EXAMPLE", # pragma: allowlist secret + "abcdef0123456789token", + "s3cretpw", + ) + if s in out + ] + + def test_fluid_cli_error_format_for_user_prints_brackets_literally(): """``FluidCLIError.format_for_user`` (the top-level handler's renderer) had the same flaw: ``exposes[orders]`` and ``pkg[gcp]`` lost their brackets.""" diff --git a/tests/iac/test_iac_databricks_enforce.py b/tests/iac/test_iac_databricks_enforce.py index 5d4180c2b..280e79e4e 100644 --- a/tests/iac/test_iac_databricks_enforce.py +++ b/tests/iac/test_iac_databricks_enforce.py @@ -722,7 +722,7 @@ def test_the_engine_prints_an_enforcement_failure_after_a_failed_apply_and_never def test_the_engine_binds_before_it_raises_the_apply_failure(): source = __import__("inspect").getsource(engine._apply_via_opentofu) bind = source.index("_enforce_after_failed_apply(plugin, contract)") - failed = source.index('"opentofu_apply_failed"') + failed = source.index("raise _apply_failure(") assert source.index("runner.tofu_apply(") < bind < failed diff --git a/tests/test_doctor_strict.py b/tests/test_doctor_strict.py new file mode 100644 index 000000000..09624e592 --- /dev/null +++ b/tests/test_doctor_strict.py @@ -0,0 +1,63 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fluid doctor --strict``: the "Action needed" summary becomes the exit code.""" + +import argparse +import logging +from unittest.mock import patch + +import pytest + +from fluid_build.cli import doctor +from fluid_build.llm.providers import LlmReadinessCheck + + +def _run(strict: bool, json_out: bool, ready: bool) -> int: + parser = argparse.ArgumentParser() + doctor.register(parser.add_subparsers()) + argv = ["doctor"] + (["--strict"] if strict else []) + (["--json"] if json_out else []) + args = parser.parse_args(argv) + readiness = LlmReadinessCheck(ready=ready, error=None if ready else "no provider") + with ( + patch.object(doctor, "_check_fluid_features", return_value=(True, [])), + patch.object(doctor, "_check_copilot_readiness", return_value=readiness), + patch.object(doctor, "_inspect_store_backend", return_value={"backend": "file"}), + patch.object(doctor, "_resolve_extended_diagnostic_script", return_value=None), + ): + return doctor.run(args, logging.getLogger("test")) + + +def test_json_payload_says_why_strict_would_fail(capsys): + import json + + assert _run(strict=True, json_out=True, ready=False) == 1 + payload = json.loads(capsys.readouterr().out) + assert payload["copilot_ready"] is False + assert payload["action_needed"] is True + + +@pytest.mark.parametrize("json_out", [False, True], ids=["text", "json"]) +def test_strict_exits_1_when_a_built_in_check_is_not_ready(json_out, capsys): + assert _run(strict=True, json_out=json_out, ready=False) == 1 + + +@pytest.mark.parametrize("json_out", [False, True], ids=["text", "json"]) +def test_default_still_exits_0_when_a_built_in_check_is_not_ready(json_out, capsys): + assert _run(strict=False, json_out=json_out, ready=False) == 0 + + +@pytest.mark.parametrize("json_out", [False, True], ids=["text", "json"]) +def test_strict_exits_0_when_everything_is_ready(json_out, capsys): + assert _run(strict=True, json_out=json_out, ready=True) == 0 diff --git a/tests/test_roadmap_claims.py b/tests/test_roadmap_claims.py new file mode 100644 index 000000000..4ff35fe38 --- /dev/null +++ b/tests/test_roadmap_claims.py @@ -0,0 +1,55 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fluid roadmap`` must not announce a release the package has not made.""" + +import re +from importlib import resources + +import pytest + +import fluid_build + +# "forge-cli v1.0 shipped", "v2.1 shipped": a release claim, not a milestone label. +_RELEASE_CLAIM = re.compile( + r"\bv(\d+)\.\d+(?:\.\d+)?\s+(?:(?:has\s+been|was|has)\s+)?shipped\b", re.IGNORECASE +) + + +def _major(version: str) -> int: + match = re.match(r"\d+", version) + return int(match.group()) if match else 0 + + +def test_roadmap_does_not_claim_a_release_beyond_the_package_version(): + text = resources.files("fluid_build.copilot").joinpath("roadmap.md").read_text("utf-8") + package_major = _major(fluid_build.__version__) + claimed = [int(m.group(1)) for m in _RELEASE_CLAIM.finditer(text)] + assert all( + major <= package_major for major in claimed + ), f"roadmap.md claims a v{max(claimed)} release but the package is {fluid_build.__version__}" + + +@pytest.mark.parametrize( + "claim", + ["forge-cli v1.0 shipped 2026-04-23.", "v1.0 was shipped", "v2.1 has been shipped"], +) +def test_the_guard_catches_the_old_claim(claim): + assert _RELEASE_CLAIM.search(claim) + + +def test_no_section_or_sentence_presents_v1_as_a_release(): + text = resources.files("fluid_build.copilot").joinpath("roadmap.md").read_text("utf-8") + assert "Shipped in v1.0" not in text + assert "shipped the day after v1.0" not in text From 2e9db7e3dec2414e01f88bd4dd260928a0fdc735 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:26:31 +0100 Subject: [PATCH 07/26] test: make welcome-scan-dependent tests independent of probe timing (#792) run_welcome_scan drops any probe result that misses its 0.4s/1.0s budget, and pick_mode takes its default from those results. On a loaded runner the workspace probe overran and test_pick_mode_default_picks_from_product_when_ workspace_has_products saw "ai" instead of "from_product" (PR #782, Python 3.14 coverage leg). Add a deterministic_welcome_scan fixture that lifts the budgets and points the dbt probe at "no dbt" (it sizes its `dbt --version` subprocess from the same budget), and apply it to the four modules whose tests assert on probe results. Exposure audit: with every probe result dropped, 13 tests in those four modules fail; the other test files that reach the scan do not. A new test pins the original failure with a deliberately slow workspace probe. The product budget is unchanged. --- tests/conftest.py | 26 ++++++++++++++++++++++ tests/test_forge_mode_picker.py | 28 ++++++++++++++++++++++++ tests/test_welcome_scan.py | 4 ++++ tests/test_welcome_scan_coding_agents.py | 2 +- tests/test_world_class_interview.py | 4 ++++ 5 files changed, 63 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index e12231b15..cfa6b6c4d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -204,6 +204,32 @@ def _disable_copilot_self_eval(monkeypatch): monkeypatch.setenv("FLUID_COPILOT_SELF_EVAL", "0") +@pytest.fixture +def deterministic_welcome_scan(monkeypatch): + """Make ``run_welcome_scan`` return every probe's result, however slow the runner. + + The scan runs its probes in threads and discards any result that misses a + 0.4s per-probe / 1.0s total budget: right for a CLI prompt, wrong for a test + that asserts on what a probe reports (workspace products, sample data, ...). + On a loaded runner (coverage, ``-n auto``) the probe overruns, its field + keeps the "nothing detected" default, and the mode picker falls back to + ``ai`` where the test expects ``from_product`` or ``refine``. + + The budget only decides which results are kept, never how long the scan runs + (it still joins every probe thread), so wall-clock assertions are unaffected. + The dbt probe sizes its ``dbt --version`` subprocess from the same budget, so + it is pointed at "no dbt installed" instead of waiting out a slow dbt-core; + tests of the dbt probe patch ``_resolve_dbt_executable`` themselves. + + Opt in with ``pytestmark = pytest.mark.usefixtures("deterministic_welcome_scan")``. + """ + monkeypatch.setattr("fluid_build.cli._welcome_scan._PROBE_TIMEOUT_S", 30.0) + monkeypatch.setattr("fluid_build.cli._welcome_scan._TOTAL_TIMEOUT_S", 60.0) + monkeypatch.setattr( + "fluid_build.build_runners.dbt.runner._resolve_dbt_executable", lambda: None + ) + + @pytest.fixture(autouse=True) def _stop_leaked_monitoring_threads(): """Drain every MonitoringSystem's background worker threads after each diff --git a/tests/test_forge_mode_picker.py b/tests/test_forge_mode_picker.py index 1f5060987..27a3c37a6 100644 --- a/tests/test_forge_mode_picker.py +++ b/tests/test_forge_mode_picker.py @@ -35,6 +35,10 @@ should_show_picker, ) +# pick_mode derives its default from the welcome scan's threaded probes; without +# this a slow runner drops the workspace probe and every default falls back to "ai". +pytestmark = pytest.mark.usefixtures("deterministic_welcome_scan") + # --------------------------------------------------------------------------- # should_show_picker — trigger conditions # --------------------------------------------------------------------------- @@ -176,6 +180,30 @@ def test_pick_mode_default_picks_from_product_when_workspace_has_products(tmp_pa assert chosen == "from_product" +def test_pick_mode_default_survives_a_slow_workspace_probe(tmp_path, silent_console, monkeypatch): + """A workspace probe slower than the scan's default 0.4s budget keeps its result. + + Deterministic stand-in for a loaded CI runner, where the probe overran and the + default fell back to ``ai`` (``assert 'ai' == 'from_product'``). Holds only while + ``deterministic_welcome_scan`` is applied to this module. + """ + import time + + from fluid_build.cli import _welcome_scan + + real_probe = _welcome_scan._probe_workspace + + def _slow_probe(start): + time.sleep(0.6) # past the default 0.4s _PROBE_TIMEOUT_S + return real_probe(start) + + monkeypatch.setattr(_welcome_scan, "_probe_workspace", _slow_probe) + (tmp_path / "contract.fluid.yaml").write_text("fluidVersion: '0.7.3'\n") + args = _bare_args() + chosen = pick_mode(args, console=silent_console, input_fn=lambda _: "", target_dir=tmp_path) + assert chosen == "refine" + + @pytest.mark.parametrize( "key_to_pick,expected_attrs", [ diff --git a/tests/test_welcome_scan.py b/tests/test_welcome_scan.py index 0c0868ae0..89bc946f8 100644 --- a/tests/test_welcome_scan.py +++ b/tests/test_welcome_scan.py @@ -28,6 +28,10 @@ run_welcome_scan, ) +# The scan drops any probe result that misses its budget; these tests assert on +# what the probes report, so they must not race the clock on a loaded runner. +pytestmark = pytest.mark.usefixtures("deterministic_welcome_scan") + @pytest.fixture(autouse=True) def isolate_home(tmp_path, monkeypatch): diff --git a/tests/test_welcome_scan_coding_agents.py b/tests/test_welcome_scan_coding_agents.py index 8691aede1..a9057304c 100644 --- a/tests/test_welcome_scan_coding_agents.py +++ b/tests/test_welcome_scan_coding_agents.py @@ -32,7 +32,7 @@ run_welcome_scan, ) -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.usefixtures("deterministic_welcome_scan")] def _which_factory(present): diff --git a/tests/test_world_class_interview.py b/tests/test_world_class_interview.py index fed57715d..6b2d8b57d 100644 --- a/tests/test_world_class_interview.py +++ b/tests/test_world_class_interview.py @@ -44,6 +44,10 @@ run_world_class_bootstrap, ) +# collect_signals reads the welcome scan's threaded probes; a slow runner would +# drop one and leave the signal at its "nothing detected" default. +pytestmark = pytest.mark.usefixtures("deterministic_welcome_scan") + # --------------------------------------------------------------------------- # Stubs # --------------------------------------------------------------------------- From d7cbea678a22644457301dc7c1abf2373d21c1b5 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:27:23 +0100 Subject: [PATCH 08/26] security: validate_ident let a name with a trailing newline through (#788) * security: validate_ident let a name with a trailing newline through Python's `$` also matches just before a trailing newline, so `_SAFE_IDENT.match("orders\n")` succeeded and validate_ident returned "orders\n" unchanged. Every DDL emitter routes identifiers through it, and a name written into a line-based file (a Java properties file, as in #784) gains a second line. The four whole-string allowlists in _sql_safety.py now end in `\Z`. Only _SAFE_IDENT changes behaviour: the type-name, type-parameter and RLS-expression helpers already strip their input, so `\Z` there keeps a later edit that drops the strip from reopening the gap. tests/providers/test_sql_safety_type_param.py pins validate_ident against "\n", "\r\n" and an embedded "\n", and each pattern against a trailing "\n". * docs(changelog): validate_ident refuses a name with a trailing newline * chore(secrets): move the two _sql_safety baseline hits to their new lines --- .secrets.baseline | 6 +-- CHANGELOG.md | 10 ++++ fluid_build/providers/_sql_safety.py | 14 ++++-- tests/providers/test_sql_safety_type_param.py | 48 ++++++++++++++++++- 4 files changed, 69 insertions(+), 9 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 233ab9f48..d2bd5129f 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -436,14 +436,14 @@ "filename": "fluid_build/providers/_sql_safety.py", "hashed_secret": "fb2620e2ed7e530fcb1ea0a2b2456751057a5178", "is_verified": false, - "line_number": 244 + "line_number": 248 }, { "type": "Secret Keyword", "filename": "fluid_build/providers/_sql_safety.py", "hashed_secret": "6d613a1ee01eec4c0f8ca66df0db71dca0c6e1cf", "is_verified": false, - "line_number": 245 + "line_number": 249 } ], "fluid_build/providers/datamesh_manager/README.md": [ @@ -2154,5 +2154,5 @@ } ] }, - "generated_at": "2026-10-31T09:00:16Z" + "generated_at": "2026-11-03T19:58:17Z" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 65fd3d436..5afdd08dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -197,6 +197,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 passes when the region matches the declaration, fails with the fix when it does not, and is skipped when the contract declares none. +### Security + +- **`validate_ident` let a name with a trailing newline through.** Python's `$` also matches just + before a trailing newline, so the identifier allowlist in `providers/_sql_safety.py`, which the + DDL and OpenTofu emitters route names through, accepted `orders\n` and passed it on with the + newline: `fluid generate iac` wrote `CREATE EXTERNAL SCHEMA IF NOT EXISTS spectrum_events\n ...` + into `main.tf.json`. The module's allowlists now end in `\Z`, so such a name is refused with + `Invalid SQL identifier: 'spectrum_events\n'`. That includes a name that reaches it with a + trailing newline from an environment variable or a file; strip the value where it is set. + ## [0.26.0] — 2026-11-01 ### Added diff --git a/fluid_build/providers/_sql_safety.py b/fluid_build/providers/_sql_safety.py index 6722aac88..2d2b30539 100644 --- a/fluid_build/providers/_sql_safety.py +++ b/fluid_build/providers/_sql_safety.py @@ -17,7 +17,11 @@ import re from typing import Any, Mapping, Optional -_SAFE_IDENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +# The whole-string allowlists in this module end in ``\Z``, not ``$``. Python's +# ``$`` also matches just before a trailing newline, so ``validate_ident`` used +# to return ``"orders\n"`` unchanged, and that newline split any line-based +# file the name was written into (a Java properties file, a log line). +_SAFE_IDENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*\Z") # Identifier-quote characters for every supported dialect belong in the # safe set: ``"`` (ANSI / Snowflake), ``[`` ``]`` (T-SQL), and ``` ` ``` # (BigQuery / MySQL). The backtick was the missing one — BigQuery table @@ -28,7 +32,7 @@ # be rejected and the ``query`` tool fails on BigQuery. A backtick can't # open a statement or a comment, so the ``;`` / ``--`` / ``/* */`` and # blocked-keyword guards below still hold. -_SAFE_EXPR_CHARS = re.compile(r"^[A-Za-z0-9_\s().,<>=!'+\-*/%|&\"`:\[\]]+$") +_SAFE_EXPR_CHARS = re.compile(r"^[A-Za-z0-9_\s().,<>=!'+\-*/%|&\"`:\[\]]+\Z") _BLOCKED_EXPR_TOKENS = re.compile( r"(?i)\b(" r"alter|call|copy|create|delete|drop|execute|grant|insert|merge|put|remove|" @@ -132,7 +136,7 @@ def dialect_uses_backslash_escapes(dialect: str | None) -> bool: # at most one comma so a contract cannot smuggle DDL through a type parameter # (identifiers/literals are already routed through ``validate_ident`` / # ``quote_string_literal``). -_SQL_TYPE_PARAM_PAYLOAD = re.compile(r"^[0-9]+(\s*,\s*[0-9]+)?$") +_SQL_TYPE_PARAM_PAYLOAD = re.compile(r"^[0-9]+(\s*,\s*[0-9]+)?\Z") # A *whole* column-type token (not just the ``(...)`` payload) such as # ``INT`` / ``VARCHAR(255)`` / ``NUMBER(18,4)`` / ``TIMESTAMP_NTZ`` / @@ -143,7 +147,7 @@ def dialect_uses_backslash_escapes(dialect: str | None) -> bool: # set deliberately excludes ``;`` ``"`` ``-`` ``*`` ``/`` ``=`` so a type # token cannot carry a statement terminator, an identifier-quote, or a # comment sequence into the surrounding ``CREATE TABLE`` DDL. -_SQL_TYPE_NAME = re.compile(r"^[A-Za-z0-9_ ()',]+$") +_SQL_TYPE_NAME = re.compile(r"^[A-Za-z0-9_ ()',]+\Z") class SqlTypeError(ValueError): @@ -161,7 +165,7 @@ def validate_sql_type_name(type_name: str) -> str: (e.g. ``VARCHAR(255)``) directly into ``CREATE TABLE`` SQL. ``validate_ident`` is too strict for parameterised / multi-word types (it forbids parens and spaces), so this applies the conservative type allowlist - ``^[A-Za-z0-9_ ()',]+$`` and additionally fails closed on comment + ``^[A-Za-z0-9_ ()',]+\\Z`` and additionally fails closed on comment sequences (``--`` / ``/*`` / ``*/``) — defence-in-depth mirroring :func:`validate_sql_expression_allowlist`. Raises :class:`SqlTypeError` (a ``ValueError`` subclass) on anything outside the allowlist so callers diff --git a/tests/providers/test_sql_safety_type_param.py b/tests/providers/test_sql_safety_type_param.py index 912fe0ac3..eba0a2ca6 100644 --- a/tests/providers/test_sql_safety_type_param.py +++ b/tests/providers/test_sql_safety_type_param.py @@ -21,11 +21,22 @@ at most one comma), so a contract cannot smuggle a statement terminator, comment sequence, or nested DDL through a type parameter. Each test here FAILS if that guard is removed or loosened. + +It also pins how the module's whole-string allowlists anchor. Python's ``$`` +also matches just before a trailing newline, so a ``$``-anchored pattern let +``validate_ident("orders\\n")`` through. """ import pytest -from fluid_build.providers._sql_safety import SqlTypeError, validate_sql_type_param_payload +from fluid_build.providers._sql_safety import ( + _SAFE_IDENT, + _SQL_TYPE_NAME, + _SQL_TYPE_PARAM_PAYLOAD, + SqlTypeError, + validate_ident, + validate_sql_type_param_payload, +) class TestValidateSqlTypeParamPayload: @@ -54,3 +65,38 @@ def test_rejects_non_str_input(self): or silently mis-handle it.""" with pytest.raises(SqlTypeError): validate_sql_type_param_payload(100) # type: ignore[arg-type] + + +class TestValidateIdentLineBreaks: + @pytest.mark.parametrize( + "name", + ["orders\n", "orders\r\n", "ord\ners"], + ids=["trailing-lf", "trailing-crlf", "embedded-lf"], + ) + def test_rejects_line_break(self, name): + """An identifier with a line break anywhere in it is rejected. Only the + trailing ``\\n`` used to pass, because ``$`` matches just before it; the + other two pin that a fix neither strips the input nor matches it line + by line.""" + with pytest.raises(ValueError, match="Invalid SQL identifier"): + validate_ident(name) + + +class TestAllowlistAnchors: + @pytest.mark.parametrize( + ("pattern", "value"), + [ + (_SAFE_IDENT, "orders"), + (_SQL_TYPE_NAME, "VARCHAR(255)"), + (_SQL_TYPE_PARAM_PAYLOAD, "18,4"), + ], + ids=["ident", "type-name", "type-param"], + ) + def test_pattern_refuses_trailing_newline(self, pattern, value): + """Each whole-string allowlist refuses a trailing newline by itself. + ``validate_sql_type_name`` and ``validate_sql_type_param_payload`` strip + their input first, so the pattern is what holds if a later edit drops + that ``.strip()``. ``_SAFE_EXPR_CHARS`` is left out: its class contains + ``\\s``, which admits a newline anyway.""" + assert pattern.match(value) + assert pattern.match(value + "\n") is None From 01497207a59ad4dcd79338a7e94566b9510ee337 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:31:14 +0100 Subject: [PATCH 09/26] ci: freeze dependency weight and startup cost before the diet (#791) * ci: freeze dependency weight and startup cost before the diet Guards at today's numbers, so nothing grows while later PRs move litellm, mcp and openlineage out of the base install. - scripts/ci/dep_budget.py and .github/workflows/dep-budget.yml resolve the base install with `uv pip compile` for linux/3.12, linux/3.10 and windows/3.12 and fail above a committed ceiling (88/86/89), naming the packages added and removed against a baseline. It is its own workflow (PRs touching pyproject, main, weekly), because an upstream release can move the count with no change here. - scripts/ci/bare_install_smoke.sh, called from ci.yml build-check and release.yml, caps the bare wheel's build_parser() import cost (869) and runs validate, plan, bundle, generate iac and doctor on an AWS example. - .importlinter-external is an import-linter forbidden contract with external packages: litellm, mcp, openlineage and boto3 stay off the core modules. The 8 lazy routes that exist today are listed one by one. - ruff TID253 bans module-level imports of the heavy SDKs, with 4 exemptions. It does not see an import inside a module-level try:, which the config says. - deptry 0.25.1 joins [dev]. CI runs it as advisory until 0.29.0. - tests/perf/test_startup_budget.py lowers MAX_MODULES from 1100 to 895 (852 on 3.12, plus 5%) and adds a discover_providers() probe capped at 916. Both counts print in the CI log. Borrowed, not built: uv, import-linter (include_external_packages, ignore_imports), ruff flake8-tidy-imports, deptry. * ci: keep the bare-install smoke out of the release build job release.yml's build job holds id-token: write and attestations: write. Running the wheel and its ~88 transitive dependencies (resolved at release time, not hash-locked) through validate, plan, bundle, generate iac and doctor there widened an exposure that `fluid init` already had. A failing doctor check could also block a tag. A tag's merge commit has already run the same smoke on the same wheel build in ci.yml build-check. So release.yml goes back to its --version and init check, and the smoke runs only in CI. * ci: the dependency budget fails loudly and ratchets Fixes from the defect hunt on the budget guard: - A uv failure (a resolver conflict or an index outage) now prints uv's own stderr. CalledProcessError's message left it out. - `name @ url` requirements are counted. Any line the parser does not understand raises, so no package slips past the budget uncounted. - --update compares against the committed baseline before rewriting it, so an over-ceiling run still names the packages added and removed. - A count under its ceiling prints a notice to lower the ceiling. - Pushes to main run the workflow only when pyproject.toml, the script, the baselines or the workflow change, as PRs already did. Upstream drift still shows in the weekly run without turning unrelated pushes red. The bare-install smoke now resolves the venv path to an absolute one, because its probe runs from /tmp, and removes its temp dir on exit. --- .github/workflows/ci.yml | 15 ++ .github/workflows/dep-budget.yml | 47 ++++++ .importlinter-external | 48 ++++++ pyproject.toml | 49 +++++- scripts/ci/bare_install_smoke.sh | 56 +++++++ scripts/ci/dep_budget.py | 157 ++++++++++++++++++ .../ci/dep_budget_baselines/linux-py310.txt | 86 ++++++++++ .../ci/dep_budget_baselines/linux-py312.txt | 88 ++++++++++ .../ci/dep_budget_baselines/windows-py312.txt | 89 ++++++++++ tests/perf/test_startup_budget.py | 44 ++++- tests/test_ci_dep_budget.py | 89 ++++++++++ 11 files changed, 764 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/dep-budget.yml create mode 100644 .importlinter-external create mode 100755 scripts/ci/bare_install_smoke.sh create mode 100644 scripts/ci/dep_budget.py create mode 100644 scripts/ci/dep_budget_baselines/linux-py310.txt create mode 100644 scripts/ci/dep_budget_baselines/linux-py312.txt create mode 100644 scripts/ci/dep_budget_baselines/windows-py312.txt create mode 100644 tests/test_ci_dep_budget.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eed1bcca7..78c3e82b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,16 @@ jobs: pip install -e ".[dev]" - name: Run import-linter contracts run: lint-imports + # Separate config: this one sets include_external_packages = True so it + # can forbid heavy third-party SDKs on the light-CLI modules. + - name: Run import-linter contracts (external packages) + run: lint-imports --config .importlinter-external + # deptry (MIT): declared-vs-imported dependency hygiene, configured in + # [tool.deptry]. ADVISORY for 0.27.0 and 0.28.0 (continue-on-error), then + # BLOCKING from 0.29.0: delete the continue-on-error line at that release. + - name: Dependency hygiene (deptry, advisory) + continue-on-error: true + run: deptry . # ── mypy --strict on the security-critical hotspot allowlist ───────── # Incremental strict-typing gate. The allowlist lives in ONE place — the @@ -599,6 +609,11 @@ jobs: /tmp/smoke/bin/fluid init smoke-test --template hello-world test -f smoke-test/contract.fluid.yaml || { echo "::error::scaffold failed"; exit 1; } echo "Scaffold verified" + # The wheel above is installed with no extras. Gate its cold-import cost + # (a module-count ceiling on build_parser()) and prove the offline + # pipeline stages run on that bare install, from a neutral directory. + - name: Bare-install smoke (module ceiling + offline pipeline) + run: bash scripts/ci/bare_install_smoke.sh /tmp/smoke examples/aws-s3-glue-athena/contract.fluid.yaml # ────────────────────────────────────────────────────────────── # DuckDB end-to-end integration. Free, secret-less, runs on every PR diff --git a/.github/workflows/dep-budget.yml b/.github/workflows/dep-budget.yml new file mode 100644 index 000000000..3493b28ce --- /dev/null +++ b/.github/workflows/dep-budget.yml @@ -0,0 +1,47 @@ +# Dependency budget — the base install may not resolve to more packages than +# its committed ceiling (scripts/ci/dep_budget.py). +# +# Not a required PR check on purpose: an upstream release can add a transitive +# package with no change here, and that must not block or redden unrelated +# work. It runs on PRs and pushes to main that touch something that can change +# the count, and weekly, so upstream growth is seen within a week. On failure +# the log names the packages added and removed against +# scripts/ci/dep_budget_baselines/. + +name: Dependency budget + +on: + pull_request: + branches: [main] + paths: + - pyproject.toml + - scripts/ci/dep_budget.py + - scripts/ci/dep_budget_baselines/** + - .github/workflows/dep-budget.yml + push: + branches: [main] + paths: + - pyproject.toml + - scripts/ci/dep_budget.py + - scripts/ci/dep_budget_baselines/** + - .github/workflows/dep-budget.yml + schedule: + - cron: "41 6 * * 1" # Mondays 06:41 UTC + workflow_dispatch: + +permissions: + contents: read + +jobs: + dep-budget: + name: Package count ceiling + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + # Pinned: a different uv can resolve a different tree. + - run: pip install "uv==0.11.23" + - name: Resolve the base install for each target and compare to its ceiling + run: python scripts/ci/dep_budget.py diff --git a/.importlinter-external b/.importlinter-external new file mode 100644 index 000000000..d6f78315b --- /dev/null +++ b/.importlinter-external @@ -0,0 +1,48 @@ +# Heavy third-party SDKs must not be reachable from the light-CLI modules, +# directly or through any chain, unless the edge is a function-local import. +# Run with: lint-imports --config .importlinter-external +# +# Separate from [tool.importlinter] in pyproject.toml, whose contracts keep +# include_external_packages = false. Here external packages are in the graph, +# and grimp counts function-local imports as edges: every lazy route that +# exists today is listed in ignore_imports with the reason it is lazy. A new +# route fails the check; a stale ignore also fails (unmatched ignores error). +# Forbidden modules must be root-level names (import-linter rule). +# Borrowed: seddonym/import-linter (BSD-2), already a dev dependency. + +[importlinter] +root_package = fluid_build +include_external_packages = True + +[importlinter:contract:light-cli-external] +name = validate/plan/bundle/apply cold path imports no heavy SDK at module scope +type = forbidden +source_modules = + fluid_build.schema_manager + fluid_build.forge.core.plan_digest + fluid_build.iac + fluid_build.cli.validate + fluid_build.cli.plan + fluid_build.cli.bundle + fluid_build.cli.apply +forbidden_modules = + litellm + mcp + openlineage + boto3 +ignore_imports = + # boto3: each of these imports sits inside a function, guarded by + # try/except ImportError, and runs only once an AWS code path is taken. + fluid_build.secrets -> boto3 + fluid_build.iac.providers.aws -> boto3 + fluid_build.credentials.adapters -> boto3 + # providers/aws: the three modules below import boto3 inside functions + # (try/except ImportError); listed one by one so a new eager import + # elsewhere in the tree is still caught. + fluid_build.providers.aws.provider -> boto3 + fluid_build.providers.aws.util.auth -> boto3 + fluid_build.providers.aws.util.config -> boto3 + fluid_build.cli._auth_provider_impls -> boto3 + # openlineage: imported inside the emit helpers, so it loads only when a + # build runner actually emits a lineage event. + fluid_build.build_runners._lineage -> openlineage diff --git a/pyproject.toml b/pyproject.toml index e4693e5ab..452110404 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -428,6 +428,11 @@ dev = [ # ``cli/__init__.py`` at import time) and that the tier-0 # ``fluid_build._net`` leaf module stays free of fluid_build upstreams. "import-linter>=2.0", + # Dependency hygiene (deptry, MIT): declared-but-unused (DEP002), imported- + # but-undeclared (DEP001/DEP003), dev-only imports in runtime code (DEP004). + # Config in ``[tool.deptry]`` below; advisory in CI for 0.27.0 and 0.28.0, + # blocking from 0.29.0. + "deptry==0.25.1", # The BigQuery read/load tests for embedded-SQL builds (tests/build_runners/ # test_embedded_sql_bigquery.py) stream Arrow batches through a fake client; # the unit lanes install ``.[dev,local]``, which has no pyarrow, so without @@ -617,7 +622,15 @@ target-version = "py310" # fallbacks when Rich is unavailable) are allowed via per-file-ignores # or a ``# noqa: T201`` comment that explains the exemption. # See CODE_REVIEW C-011. -select = ["E", "F", "I", "B", "T201"] +# +# TID253 (flake8-tidy-imports banned-module-level-imports) keeps the heavy SDKs +# off the import-time path: they must be imported inside the function that uses +# them, so ``fluid --help`` / validate / plan stay light. Borrowed from ruff +# rather than a custom check. Gap, measured: TID253 does NOT flag an import +# inside a module-level ``try:`` block (the guarded-optional-import idiom), so a +# green run proves less than it looks. The runtime guards in +# tests/perf/test_startup_budget.py and ``.importlinter-external`` cover that. +select = ["E", "F", "I", "B", "T201", "TID253"] ignore = [ "E203", # whitespace before ':' — Black-compatible "E501", # line-too-long — handled by Black where possible @@ -639,9 +652,20 @@ ignore = [ "F811", # redefined-while-unused — overload patterns ] +[tool.ruff.lint.flake8-tidy-imports] +banned-module-level-imports = [ + "litellm", + "mcp", + "openlineage", + "boto3", + "botocore", + "google.cloud", + "snowflake.connector", + "databricks.sdk", +] + [tool.ruff.lint.per-file-ignores] # Tests, doctor scripts, and one-shot tools are allowed bare print(). -"tests/**" = ["T201"] "scripts/**" = ["T201"] # The console renderer owns print() fallbacks by design. The canonical # home is the tier-0 leaf ``fluid_build/_console.py``; ``cli/console.py`` @@ -653,6 +677,27 @@ ignore = [ # The error printer is the last-resort path when the Rich panel render # itself fails — it MUST use bare print(..., file=sys.stderr). "fluid_build/cli/errors.py" = ["T201"] +# TID253: tests may import the SDKs they exercise at module scope. +"tests/**" = ["T201", "TID253"] +# TID253: the MCP output port is a server that needs the SDK to exist; it is +# only imported once a user runs ``fluid mcp serve`` (lazy at the CLI). +"fluid_build/output_ports/mcp/server.py" = ["TID253"] +"fluid_build/output_ports/mcp/_transport.py" = ["TID253"] +# TID253: BigQuery validator needs google.cloud at import; loaded only by the +# GCP provider paths. +"fluid_build/providers/bigquery_validation.py" = ["TID253"] + +[tool.deptry] +known_first_party = ["fluid_build"] +# Extras that hold dev tooling, not runtime dependencies: their imports are +# expected in tests only. +optional_dependencies_dev_groups = ["dev", "docs", "test-emulators", "test-emulators-heavy"] +# Keeps deptry's defaults (they already exclude tests and .venv). +extend_exclude = ["scripts", "examples"] + +[tool.deptry.per_rule_ignores] +# h11 is a security floor for a transitive dependency; nothing imports it. +DEP002 = ["h11"] [tool.bandit] # B608: SQL query construction is expected — this is a pipeline/SQL code generator diff --git a/scripts/ci/bare_install_smoke.sh b/scripts/ci/bare_install_smoke.sh new file mode 100755 index 000000000..f31665995 --- /dev/null +++ b/scripts/ci/bare_install_smoke.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Smoke a BARE wheel install (no extras): the cold import cost of +# build_parser(), then the offline pipeline stages on an example contract. +# Called from ci.yml (build-check). Not from release.yml: its build job holds +# id-token and attestations write, and a tag's commit has already passed this. +# +# scripts/ci/bare_install_smoke.sh +# +# Run from a neutral directory (the script makes its own), so nothing resolves +# from the checkout. The contract is an AWS example: `generate iac` renders it +# offline, and nothing here needs a credential. +set -euo pipefail + +# Absolute, because the module-count probe below runs from /tmp. +VENV="$(cd "$1" && pwd)" +CONTRACT="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")" + +# Ceiling for `import fluid_build.cli; build_parser()` on a bare install. +# Measured 852 on Python 3.12 (2026-11-03); ceiling = measured + 2%, rounded +# up. Lower it when a slimming change drops the count. +BARE_MAX_MODULES=869 + +count=$(cd /tmp && "$VENV/bin/python" -I -c " +import sys; sys.argv = ['fluid', '--help'] +from fluid_build.cli import build_parser; build_parser() +print(len(sys.modules))") +echo "Bare install: build_parser() imports ${count} modules (ceiling ${BARE_MAX_MODULES})" +if [ "$count" -ge "$BARE_MAX_MODULES" ]; then + echo "::error::bare install imports ${count} modules on the build_parser() path (ceiling ${BARE_MAX_MODULES})" + exit 1 +fi + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +cd "$work" +FLUID="$VENV/bin/fluid" +"$FLUID" validate "$CONTRACT" +"$FLUID" plan "$CONTRACT" --out plan.json +"$FLUID" bundle "$CONTRACT" --format tgz --out bundle.tgz +"$FLUID" generate iac "$CONTRACT" --out iac +"$FLUID" doctor +echo "Bare-install smoke passed" diff --git a/scripts/ci/dep_budget.py b/scripts/ci/dep_budget.py new file mode 100644 index 000000000..675d783ec --- /dev/null +++ b/scripts/ci/dep_budget.py @@ -0,0 +1,157 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fail when the base install resolves to more packages than its ceiling. + +``pip install data-product-forge`` pulls a transitive tree, and nothing in the +repo noticed it growing. This resolves ``pyproject.toml`` (base dependencies, +no extras) for each supported target and compares the package count to a +committed ceiling. When a count goes over, the failure names the packages +added and removed against a committed baseline list for that target, so the +reviewer sees which dependency brought the rest. + + python scripts/ci/dep_budget.py # check every target + python scripts/ci/dep_budget.py --update # rewrite the baseline lists + +Borrowed, not built: resolution is ``uv pip compile`` (astral-sh/uv, MIT). +No off-the-shelf "dependency count budget" gate exists, so the comparison +stays this thin. + +Lowering a ceiling is a one-line edit to ``TARGETS`` plus ``--update`` to +refresh the baseline. The check is not run on every PR (see +``.github/workflows/dep-budget.yml``) because an upstream release can raise a +count with no change here. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +BASELINES = Path(__file__).resolve().parent / "dep_budget_baselines" + +# name -> (uv --python-platform, --python-version, ceiling). Measured with +# uv 0.11.23 on 2026-11-03. +TARGETS: dict[str, tuple[str, str, int]] = { + "linux-py312": ("linux", "3.12", 88), + "linux-py310": ("linux", "3.10", 86), + "windows-py312": ("windows", "3.12", 89), +} + +# ``name==version`` or ``name @ url``: uv writes one or the other per package. +_REQ = re.compile(r"^([A-Za-z0-9][A-Za-z0-9._-]*)\s*(?:==|@)") + + +def parse_packages(compiled: str) -> list[str]: + """Package names (PEP 503 normalised, sorted) from ``uv pip compile`` output. + + A line that is neither a comment nor a requirement fails loudly, so an + output format this does not understand cannot let a package through + uncounted. + """ + names = set() + for line in compiled.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + m = _REQ.match(line) + if not m: + raise ValueError(f"unrecognised line in uv pip compile output: {line!r}") + names.add(re.sub(r"[-_.]+", "-", m.group(1)).lower()) + return sorted(names) + + +def compile_target(platform: str, python: str) -> list[str]: + try: + out = subprocess.run( + [ + "uv", + "pip", + "compile", + "pyproject.toml", + "--python-version", + python, + "--python-platform", + platform, + "--no-header", + "--no-annotate", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ).stdout + except subprocess.CalledProcessError as exc: + # A resolver conflict or an index outage: show uv's own reason, which + # CalledProcessError's message leaves out. + raise SystemExit( + f"uv pip compile failed for {platform}/{python} (exit {exc.returncode}):\n" + f"{(exc.stderr or '').strip()}" + ) from exc + return parse_packages(out) + + +def check(name: str, packages: list[str], ceiling: int, baseline: list[str]) -> str | None: + """Return a failure message, or None when the count is within the ceiling.""" + if len(packages) <= ceiling: + return None + added, removed = sorted(set(packages) - set(baseline)), sorted(set(baseline) - set(packages)) + return ( + f"{name}: base install resolves to {len(packages)} packages (ceiling {ceiling}).\n" + f" added vs baseline: {', '.join(added) or '-'}\n" + f" removed vs baseline: {', '.join(removed) or '-'}\n" + " Over the ceiling: a new dependency, or an upstream release that grew its own " + "tree. Drop it, or move it to an extra.\n" + " If the growth is intended, raise the ceiling in scripts/ci/dep_budget.py " + "and run --update." + ) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--update", action="store_true", help="rewrite the baseline lists") + args = ap.parse_args(argv) + + failures = [] + for name, (platform, python, ceiling) in TARGETS.items(): + packages = compile_target(platform, python) + path = BASELINES / f"{name}.txt" + # Compare against the committed baseline before --update rewrites it, + # so an over-ceiling failure still names what changed. + baseline = path.read_text().split() if path.exists() else [] + print(f"{name}: {len(packages)} packages (ceiling {ceiling})") + msg = check(name, packages, ceiling, baseline) + if msg: + failures.append(msg) + elif len(packages) < ceiling: + # Ratchet: a ceiling with room to spare would let the count creep back. + print( + f"::notice::{name} is under its ceiling: lower it to {len(packages)} " + "in scripts/ci/dep_budget.py and run --update" + ) + if args.update: + BASELINES.mkdir(exist_ok=True) + path.write_text("".join(f"{p}\n" for p in packages)) + if failures: + print("\n" + "\n".join(failures), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/dep_budget_baselines/linux-py310.txt b/scripts/ci/dep_budget_baselines/linux-py310.txt new file mode 100644 index 000000000..c87d843b2 --- /dev/null +++ b/scripts/ci/dep_budget_baselines/linux-py310.txt @@ -0,0 +1,86 @@ +aiohappyeyeballs +aiohttp +aiosignal +annotated-types +anyio +async-timeout +attrs +backports-tarfile +certifi +cffi +charset-normalizer +click +cryptography +distro +exceptiongroup +fastuuid +filelock +frozenlist +fsspec +h11 +hf-xet +httpcore +httpcore2 +httpx +httpx2 +huggingface-hub +idna +importlib-metadata +jaraco-classes +jaraco-context +jaraco-functools +jeepney +jinja2 +jiter +jsonschema +jsonschema-specifications +keyring +litellm +markdown-it-py +markupsafe +mcp +mcp-types +mdurl +more-itertools +multidict +open-data-contract-standard +openai +openlineage-python +opentelemetry-api +packaging +propcache +pycparser +pydantic +pydantic-core +pydantic-settings +pygments +pyjwt +python-dateutil +python-dotenv +python-multipart +pyyaml +readchar +referencing +regex +requests +rfc8785 +rich +rpds-py +secretstorage +six +sniffio +sqlglot +sse-starlette +starlette +tiktoken +tokenizers +tomli +tqdm +truststore +typing-extensions +typing-inspection +tzdata +urllib3 +uvicorn +yarl +zipp diff --git a/scripts/ci/dep_budget_baselines/linux-py312.txt b/scripts/ci/dep_budget_baselines/linux-py312.txt new file mode 100644 index 000000000..63cc087e1 --- /dev/null +++ b/scripts/ci/dep_budget_baselines/linux-py312.txt @@ -0,0 +1,88 @@ +aiohappyeyeballs +aiohttp +aiosignal +annotated-types +anyio +attrs +boto3 +botocore +certifi +cffi +charset-normalizer +click +cryptography +distro +fastuuid +filelock +frozenlist +fsspec +h11 +h2 +hf-xet +hpack +httpcore +httpcore2 +httpx +httpx2 +huggingface-hub +hyperframe +idna +importlib-metadata +jaraco-classes +jaraco-context +jaraco-functools +jeepney +jinja2 +jiter +jmespath +jsonschema +jsonschema-specifications +keyring +litellm +markdown-it-py +markupsafe +mcp +mcp-types +mdurl +more-itertools +multidict +open-data-contract-standard +openai +openlineage-python +opentelemetry-api +packaging +propcache +pycparser +pydantic +pydantic-core +pydantic-settings +pygments +pyjwt +python-dateutil +python-dotenv +python-multipart +pyyaml +readchar +referencing +regex +requests +rfc8785 +rich +rpds-py +s3transfer +secretstorage +six +sniffio +sqlglot +sse-starlette +starlette +tiktoken +tokenizers +tqdm +truststore +typing-extensions +typing-inspection +urllib3 +uvicorn +yarl +zipp diff --git a/scripts/ci/dep_budget_baselines/windows-py312.txt b/scripts/ci/dep_budget_baselines/windows-py312.txt new file mode 100644 index 000000000..dc0e84975 --- /dev/null +++ b/scripts/ci/dep_budget_baselines/windows-py312.txt @@ -0,0 +1,89 @@ +aiohappyeyeballs +aiohttp +aiosignal +annotated-types +anyio +attrs +boto3 +botocore +certifi +cffi +charset-normalizer +click +colorama +cryptography +distro +fastuuid +filelock +frozenlist +fsspec +h11 +h2 +hf-xet +hpack +httpcore +httpcore2 +httpx +httpx2 +huggingface-hub +hyperframe +idna +importlib-metadata +jaraco-classes +jaraco-context +jaraco-functools +jinja2 +jiter +jmespath +jsonschema +jsonschema-specifications +keyring +litellm +markdown-it-py +markupsafe +mcp +mcp-types +mdurl +more-itertools +multidict +open-data-contract-standard +openai +openlineage-python +opentelemetry-api +packaging +propcache +pycparser +pydantic +pydantic-core +pydantic-settings +pygments +pyjwt +python-dateutil +python-dotenv +python-multipart +pywin32 +pywin32-ctypes +pyyaml +readchar +referencing +regex +requests +rfc8785 +rich +rpds-py +s3transfer +six +sniffio +sqlglot +sse-starlette +starlette +tiktoken +tokenizers +tqdm +truststore +typing-extensions +typing-inspection +urllib3 +uvicorn +yarl +zipp diff --git a/tests/perf/test_startup_budget.py b/tests/perf/test_startup_budget.py index dd546d50c..62ff1f908 100644 --- a/tests/perf/test_startup_budget.py +++ b/tests/perf/test_startup_budget.py @@ -80,7 +80,16 @@ # ``copilot.schemas.stage_outputs`` / ``modeling_techniques``). Deferring that # one is a follow-up outside this change's blast radius, which is why # ``pydantic`` is deliberately NOT added to ``FORBIDDEN_ON_HELP`` here. -MAX_MODULES = 1100 +# +# Ratchet: measured 852 on 3.12 / 866 on 3.14 / 840 on 3.10 (2026-11-03, a +# ``.[dev,local]`` install, the same one every ``Perf budgets`` lane uses); +# the ceiling is the 3.12 count + 5%, rounded up. Lower it whenever a slimming +# PR drops the count; the test prints the measured number so CI logs show it. +MAX_MODULES = 895 +# ``fluid_build.providers.discover_providers()`` imports every built-in +# provider module. Measured 872 on 3.12 / 885 on 3.14 / 696 on 3.10 in a +# fresh process (2026-11-03); ceiling is the 3.12 count + 5%, rounded up. +MAX_DISCOVERY_MODULES = 916 MAX_HELP_WALL_SECONDS = 2.0 # Heavy SDKs that must NOT load on the --help / parser-build path. ``mcp`` is @@ -160,8 +169,17 @@ def _probe(): return count, loaded -def test_help_module_count_under_budget(): +_DISCOVER_PROBE = ( + "import sys;" + "from fluid_build.providers import discover_providers; discover_providers();" + "print(len(sys.modules))" +) + + +def test_help_module_count_under_budget(capsys): count, _ = _probe() + with capsys.disabled(): + print(f"\n[startup-budget] build_parser(): {count} modules (budget {MAX_MODULES})") assert count < MAX_MODULES, ( f"`fluid --help` imports {count} modules (budget {MAX_MODULES}). " "A new top-level import crept onto the parser-build path — defer it " @@ -169,6 +187,28 @@ def test_help_module_count_under_budget(): ) +def test_provider_discovery_module_count_under_budget(capsys): + """``discover_providers()`` in a cold process. No banned-module assertion yet.""" + count = int( + subprocess.run( + [sys.executable, "-c", _DISCOVER_PROBE], + capture_output=True, + text=True, + check=True, + env=_clean_env(), + ).stdout.splitlines()[-1] + ) + with capsys.disabled(): + print( + f"\n[startup-budget] discover_providers(): {count} modules " + f"(budget {MAX_DISCOVERY_MODULES})" + ) + assert count < MAX_DISCOVERY_MODULES, ( + f"`discover_providers()` imports {count} modules (budget {MAX_DISCOVERY_MODULES}). " + "A provider module (or something it imports at module scope) got heavier." + ) + + def test_heavy_sdks_not_loaded_on_help(): _, loaded = _probe() assert not loaded, ( diff --git a/tests/test_ci_dep_budget.py b/tests/test_ci_dep_budget.py new file mode 100644 index 000000000..ee3e3dd1b --- /dev/null +++ b/tests/test_ci_dep_budget.py @@ -0,0 +1,89 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for ``scripts/ci/dep_budget.py`` (pure parts; no ``uv`` is run).""" + +from __future__ import annotations + +import importlib.util +import subprocess +from pathlib import Path + +import pytest + +_SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "ci" / "dep_budget.py" +_spec = importlib.util.spec_from_file_location("dep_budget", _SCRIPT) +assert _spec and _spec.loader +budget = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(budget) + + +def test_parse_packages_normalises_and_ignores_markers_and_comments() -> None: + compiled = "# a comment\nPyYAML==6.0\ntyping_extensions==4.0 ; python_version < '3.11'\n\n" + assert budget.parse_packages(compiled) == ["pyyaml", "typing-extensions"] + + +def test_check_passes_at_or_under_ceiling() -> None: + assert budget.check("t", ["a", "b"], 2, ["a", "b"]) is None + + +def test_check_names_added_and_removed_when_over_ceiling() -> None: + msg = budget.check("t", ["a", "c", "d"], 2, ["a", "b"]) + assert msg and "3 packages (ceiling 2)" in msg + assert "added vs baseline: c, d" in msg + assert "removed vs baseline: b" in msg + + +def test_every_target_has_a_committed_baseline_within_its_ceiling() -> None: + for name, (_, _, ceiling) in budget.TARGETS.items(): + listed = (budget.BASELINES / f"{name}.txt").read_text().split() + assert 0 < len(listed) <= ceiling, name + + +def test_parse_packages_counts_url_requirements() -> None: + compiled = "click @ https://example.invalid/click.whl\nfoo @ git+https://x/y\nbar==1\n" + assert budget.parse_packages(compiled) == ["bar", "click", "foo"] + + +def test_parse_packages_rejects_a_line_it_does_not_understand() -> None: + with pytest.raises(ValueError, match="unrecognised line"): + budget.parse_packages("bar==1\n-e file:///somewhere\n") + + +def test_a_uv_failure_shows_uvs_own_reason(monkeypatch) -> None: + def fail(*_a, **_k): + raise subprocess.CalledProcessError(1, ["uv"], stderr="error: No solution found") + + monkeypatch.setattr(budget.subprocess, "run", fail) + with pytest.raises(SystemExit, match="No solution found"): + budget.compile_target("linux", "3.12") + + +def test_update_reports_the_diff_against_the_old_baseline(monkeypatch, tmp_path, capsys) -> None: + (tmp_path / "t.txt").write_text("a\nb\n") + monkeypatch.setattr(budget, "BASELINES", tmp_path) + monkeypatch.setattr(budget, "TARGETS", {"t": ("linux", "3.12", 2)}) + monkeypatch.setattr(budget, "compile_target", lambda *_: ["a", "c", "d"]) + assert budget.main(["--update"]) == 1 + assert "added vs baseline: c, d" in capsys.readouterr().err + assert (tmp_path / "t.txt").read_text().split() == ["a", "c", "d"] + + +def test_a_count_under_the_ceiling_asks_for_a_ratchet(monkeypatch, tmp_path, capsys) -> None: + (tmp_path / "t.txt").write_text("a\n") + monkeypatch.setattr(budget, "BASELINES", tmp_path) + monkeypatch.setattr(budget, "TARGETS", {"t": ("linux", "3.12", 5)}) + monkeypatch.setattr(budget, "compile_target", lambda *_: ["a"]) + assert budget.main([]) == 0 + assert "lower it to 1" in capsys.readouterr().out From 8053eb711c37525d1375106daa680bc19350e963 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:58:39 +0100 Subject: [PATCH 10/26] fix: install hints name data-product-forge, not fluid-build (#790) * fix: install hints name data-product-forge, not fluid-build Doctor, the copilot, the MCP, ODCS and DuckDB errors, the generated Jenkinsfile and the agent docs told users to `pip install fluid-build[...]`. The distribution is `data-product-forge`; every hint now names it with the right extra. - Add an empty `mcp` extra so `data-product-forge[mcp]` resolves today; it is filled when mcp leaves the base install. - tests/test_no_wrong_package_name.py fails on any `fluid-build` not followed by `.` (domains are a separate fence). The allowlist matches a line, not a file (only OTel's service.namespace), and a coverage guard keeps the scan honest. Pattern borrowed from tests/test_no_reserved_tld_defaults.py. * test: catch every shape of a wrong install hint, and missing extras Follow-up from the security review: - An extras-exist check: every `data-product-forge[...]` in the scanned files must name extras that pyproject declares. It failed first on two existing hints. tracing.py now names the OTel packages it imports (opentelemetry-sdk, opentelemetry-exporter-otlp-proto-http), and the CONTRIBUTING.md example names `atlasclient`. Neither extra existed, so pip installed the base package and the feature still failed. - A sentence-final "fluid-build." no longer passes as a domain. - Install commands (pip, pip3, pipx, uv pip) fail on any fluid[-_.]build spelling, since PyPI normalises them to one name. The import package `fluid_build` stays legal everywhere else. - The scan now covers the non-.py files shipped in the wheel and CONTRIBUTING.md, and asserts that it read them. - odcs.py quotes its extra, which zsh rejected unquoted. * fix: the vowl hint keeps its extra, and the secrets baseline follows From the verifier: - `fluid odcs validate` printed the vowl hint through Rich markup, which read `[odcs-strict]` as a tag and dropped it. Users were told to install the base package, which has no vowl. It now prints with markup=False, as generate_ci already does. A test drives the real cprint. - tests/test_litellm_backend.py moved a baselined fake key from line 566 to 607. CI's detect-secrets-hook on changed files would exit 3, so .secrets.baseline is refreshed. The diff is that line number and generated_at only; no secret entry was added. --- .secrets.baseline | 2 +- AGENTS.md | 2 +- CHANGELOG.md | 5 + CLAUDE.md | 2 +- CONTRIBUTING.md | 5 +- fluid_build/cli/_attestation.py | 4 +- fluid_build/cli/dbt_mcp.py | 2 +- fluid_build/cli/doctor.py | 2 +- fluid_build/cli/forge_copilot_agent_loop.py | 2 +- fluid_build/cli/forge_copilot_memory.py | 2 +- fluid_build/cli/forge_db_tools.py | 2 +- fluid_build/cli/hosted_mcp.py | 2 +- .../cli/market_catalogs/mcp_catalog.py | 2 +- fluid_build/cli/odcs.py | 8 +- fluid_build/cli/scaffold_ci.py | 4 +- fluid_build/copilot/store/backends/sqlite.py | 2 +- fluid_build/llm/litellm_backend.py | 2 +- fluid_build/observability/tracing.py | 5 +- fluid_build/providers/odcs/validation.py | 2 +- pyproject.toml | 7 +- tests/test_litellm_backend.py | 5 +- tests/test_no_wrong_package_name.py | 210 ++++++++++++++++++ tests/test_odcs.py | 13 ++ 23 files changed, 265 insertions(+), 27 deletions(-) create mode 100644 tests/test_no_wrong_package_name.py diff --git a/.secrets.baseline b/.secrets.baseline index d2bd5129f..f0631d42f 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1904,7 +1904,7 @@ "filename": "tests/test_litellm_backend.py", "hashed_secret": "a9329abf28aa131139e12add040b3889ea98d12d", "is_verified": false, - "line_number": 566 + "line_number": 607 } ], "tests/test_litellm_router.py": [ diff --git a/AGENTS.md b/AGENTS.md index 9dbdc7640..ac84efe8e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -371,7 +371,7 @@ Cost is visible before it's spent: every run renders `.fluid/agents//{co Opt into the **LiteLLM unified backend** for one API across OpenAI / Anthropic / Gemini / Bedrock / Azure / Vertex / Groq / Cohere / Mistral / Ollama: ```bash -pip install "fluid-build[litellm]" +pip install "data-product-forge[litellm]" export FLUID_LLM_BACKEND=litellm fluid forge --llm-provider gemini --llm-model gemini-2.5-flash ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 5afdd08dd..c2f809e7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -147,6 +147,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 calls its "v1" headings an internal milestone. A test fails if it announces a 1.x release while the package is below 1.0. The package description named GCP, Snowflake and Local and now names AWS and Databricks too. +- **Install hints named `fluid-build` instead of `data-product-forge`.** Doctor, the copilot, + the MCP, ODCS and DuckDB errors, the generated Jenkinsfile and the docs told people to + `pip install fluid-build[...]`; they now name `data-product-forge` with the right extra, and a + ratchet test keeps the wrong name out. `data-product-forge[mcp]` resolves as an empty alias. + `fluid-build` on PyPI is now only an alias that installs `data-product-forge`. - **A contract's governance labels were dropped silently on Databricks.** The Databricks emitter never read `labels`, so `data-classification: confidential`, the `gdpr-article`, and the sovereignty jurisdiction, regulation and residency reached AWS (tags, Glue parameters) and GCP diff --git a/CLAUDE.md b/CLAUDE.md index 026349631..3ec3c630e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -129,7 +129,7 @@ A multi-phase push landed comprehensive UX + architecture upgrades on `feat/sour - **Self-healing repair loop.** `forge_copilot_runtime.py` runs the JSON-schema validator on every emitted contract and prepends path-specific errors to the next attempt's repair feedback. Function: `forge_copilot_corrective_feedback.build_schema_validation_message`. - **`fluid stats`.** Aggregates `.fluid/agents/*/cost.json` across runs. `--by provider/type/engine`, `--since `, `--json`. - **Forge templates emit v0.7.3 directly.** Each of `forge/templates/{starter,analytics,etl_pipeline,ml_pipeline,streaming}.py` now uses the shared spec-driven builder at `forge/templates/_v073_builder.py`. The legacy v0.5-vintage methods are preserved as `_legacy_generate_contract_unused` for reference. The runtime coercion layer at `forge_modes.py::_coerce_template_contract_to_v073` is now a no-op for v0.7.3 contracts (fast-path) and remains as a safety net for any legacy out-of-tree template. -- **LiteLLM unified backend.** `cli/forge_copilot_llm_litellm.py` ships an opt-in `LiteLLMProvider` that subclasses `LlmProvider` and routes every provider through one API. Enable via `FLUID_LLM_BACKEND=litellm` + `pip install 'fluid-build[litellm]'`. `RunCostTracker.record_call` now accepts `usd_override` (used by litellm to feed accurate per-call cost into the cost summary). Dispatched in `forge_copilot_llm_providers.py::get_llm_provider` + short-circuits in `call_llm` and `call_llm_streaming`. +- **LiteLLM unified backend.** `cli/forge_copilot_llm_litellm.py` ships an opt-in `LiteLLMProvider` that subclasses `LlmProvider` and routes every provider through one API. Enable via `FLUID_LLM_BACKEND=litellm` + `pip install 'data-product-forge[litellm]'`. `RunCostTracker.record_call` now accepts `usd_override` (used by litellm to feed accurate per-call cost into the cost summary). Dispatched in `forge_copilot_llm_providers.py::get_llm_provider` + short-circuits in `call_llm` and `call_llm_streaming`. - **UX telemetry.** `cli/_ux_telemetry.py` captures `time_to_first_panel_ms`, `questions_asked`, `inferences_used`, `picker_choice`, `mode`, `preview_accepted`, `schema_repair_attempts`. Emitted onto the `forge.invocation` OTel span when an exporter is configured. - **Mode dispatch fix.** Picking `template` now routes to `forge_modes.run_template_mode` (no AI). Picking `blank` goes to `_run_blank_mode`. Picking `refine`/`from_product` flows through the AI runtime with the right context flags. The `--template` flag is an argparse alias of `--scaffold`. - **Datamesh-manager Silver↔Gold archetype swap fix.** `providers/datamesh_manager/datamesh_manager.py` had `Silver→consumer-aligned` and `Gold→aggregate` reversed. Now correctly reads `metadata.productType` first and falls back to the canonical layer mapping (Bronze→source-aligned / Silver→aggregate / Gold→consumer-aligned). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c60db01bf..751c4c391 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -372,9 +372,8 @@ class AtlasCatalogAdapter(CatalogAdapter): import atlasclient # type: ignore[import-not-found] except ImportError as exc: raise CatalogConfigError( - "atlasclient missing. Install with: " - 'pip install "data-product-forge[atlas]"', - suggestions=['pip install "data-product-forge[atlas]"'], + "atlasclient missing. Install with: pip install atlasclient", + suggestions=["pip install atlasclient"], ) from exc self._client = atlasclient.Atlas(...) return self._client diff --git a/fluid_build/cli/_attestation.py b/fluid_build/cli/_attestation.py index dc8bcc0cf..58ea6bcdc 100644 --- a/fluid_build/cli/_attestation.py +++ b/fluid_build/cli/_attestation.py @@ -73,9 +73,9 @@ # SLSA Provenance predicate type. Stable for SLSA v1.0; bumping SLSA # version would change this URI and the internal structure. _SLSA_PROVENANCE_PREDICATE_TYPE = "https://slsa.dev/provenance/v1" -# Our build-type URI — identifies the builder (fluid-build bundle) +# Our build-type URI — identifies the builder (`fluid bundle`) # and its version. The version suffix lets verifiers pin to a -# specific fluid-build release's provenance schema. +# specific data-product-forge release's provenance schema. _FLUID_BUILD_TYPE_V1 = "https://fluid-build.io/bundle-provenance/v1" diff --git a/fluid_build/cli/dbt_mcp.py b/fluid_build/cli/dbt_mcp.py index 71bf9bea3..61e9e57df 100644 --- a/fluid_build/cli/dbt_mcp.py +++ b/fluid_build/cli/dbt_mcp.py @@ -188,7 +188,7 @@ async def _open_real_session(self) -> AsyncIterator[Any]: from mcp import ClientSessionGroup, StdioServerParameters except ImportError as exc: # pragma: no cover - mcp is a base dep raise RuntimeError( - "The dbt MCP delegate requires the 'mcp' SDK (ships with fluid-build)." + "The dbt MCP delegate requires the 'mcp' SDK (pip install 'data-product-forge[mcp]')." ) from exc params = StdioServerParameters( diff --git a/fluid_build/cli/doctor.py b/fluid_build/cli/doctor.py index 003bee39b..3417119ba 100644 --- a/fluid_build/cli/doctor.py +++ b/fluid_build/cli/doctor.py @@ -1203,7 +1203,7 @@ def _check_fluid_features() -> Tuple[bool, List[Dict[str, any]]]: "category": "ai", "status": "⚠️ Not installed (optional)", "ok": True, - "details": "Install with: pip install 'fluid-build[litellm]'", + "details": "Install with: pip install 'data-product-forge[litellm]'", } ) diff --git a/fluid_build/cli/forge_copilot_agent_loop.py b/fluid_build/cli/forge_copilot_agent_loop.py index 8a079a762..1da336c00 100644 --- a/fluid_build/cli/forge_copilot_agent_loop.py +++ b/fluid_build/cli/forge_copilot_agent_loop.py @@ -686,7 +686,7 @@ def _call_llm_with_tools( raise CopilotGenerationError( "copilot_agent_loop_litellm_missing", "The agent loop needs litellm for tool use but it is not installed.", - suggestions=["pip install 'fluid-build[litellm]'"], + suggestions=["pip install 'data-product-forge[litellm]'"], ) from exc except Exception as exc: # noqa: BLE001 — translate to the loop's error type raise CopilotGenerationError( diff --git a/fluid_build/cli/forge_copilot_memory.py b/fluid_build/cli/forge_copilot_memory.py index d58ecb38a..8d4eef8a0 100644 --- a/fluid_build/cli/forge_copilot_memory.py +++ b/fluid_build/cli/forge_copilot_memory.py @@ -404,7 +404,7 @@ def _coerce_memory_document(raw: Any, *, project_root: Path) -> CopilotProjectMe if schema_version > MEMORY_SCHEMA_VERSION: raise ValueError( f"unsupported schema_version {schema_version}; expected {MEMORY_SCHEMA_VERSION}. " - "Upgrade fluid-build to read this memory file." + "Upgrade data-product-forge to read this memory file." ) if schema_version < MEMORY_SCHEMA_VERSION: # Future: add migration logic here (e.g. v0 -> v1 transforms). diff --git a/fluid_build/cli/forge_db_tools.py b/fluid_build/cli/forge_db_tools.py index 52bec375c..f672f1f93 100644 --- a/fluid_build/cli/forge_db_tools.py +++ b/fluid_build/cli/forge_db_tools.py @@ -346,7 +346,7 @@ def _fetch_sample_rows(arguments: Dict[str, Any], env: Mapping[str, str]) -> Dic except ImportError: return { "error": "DuckDbNotInstalled", - "message": "fetch_sample_rows requires duckdb (pip install 'fluid-build[local]').", + "message": "fetch_sample_rows requires duckdb (pip install 'data-product-forge[local]').", } from fluid_build.providers._duckdb_sandbox import DuckDBAllowlist, secure_duckdb_connect diff --git a/fluid_build/cli/hosted_mcp.py b/fluid_build/cli/hosted_mcp.py index adea45b1b..f3344136d 100644 --- a/fluid_build/cli/hosted_mcp.py +++ b/fluid_build/cli/hosted_mcp.py @@ -334,7 +334,7 @@ async def _open_real_session(self) -> AsyncIterator[Any]: from mcp import ClientSessionGroup, StdioServerParameters except ImportError as exc: # pragma: no cover - mcp is a base dep raise RuntimeError( - "Hosted-MCP delegation requires the 'mcp' SDK (ships with fluid-build)." + "Hosted-MCP delegation requires the 'mcp' SDK (pip install 'data-product-forge[mcp]')." ) from exc params = StdioServerParameters( diff --git a/fluid_build/cli/market_catalogs/mcp_catalog.py b/fluid_build/cli/market_catalogs/mcp_catalog.py index d303d8c8f..2957775fa 100644 --- a/fluid_build/cli/market_catalogs/mcp_catalog.py +++ b/fluid_build/cli/market_catalogs/mcp_catalog.py @@ -505,7 +505,7 @@ async def _open_real_session(self) -> AsyncIterator[Any]: from mcp import ClientSessionGroup except ImportError as exc: # pragma: no cover - mcp is a core dep raise RuntimeError( - "MCP discovery requires the 'mcp' SDK (pip install 'fluid-build')." + "MCP discovery requires the 'mcp' SDK (pip install 'data-product-forge[mcp]')." ) from exc params = self._build_server_params() diff --git a/fluid_build/cli/odcs.py b/fluid_build/cli/odcs.py index 6ffd83956..b277d913e 100644 --- a/fluid_build/cli/odcs.py +++ b/fluid_build/cli/odcs.py @@ -369,7 +369,7 @@ def register(subparsers) -> None: help="Validate ODCS contract file", description=( "Validate an ODCS YAML/JSON file. Default: jsonschema first-pass + " - "vowl second-pass (when installed via `pip install fluid-build[odcs-strict]`). " + "vowl second-pass (when installed via `pip install 'data-product-forge[odcs-strict]'`). " "All violations are collected — exit code is 1 if any check fails." ), ) @@ -583,7 +583,11 @@ def _run_odcs_validate(args): else: if diag is None: report["passes"]["vowl"] = {"ok": True, "skipped": "vowl not installed"} - cprint("○ vowl: not installed (pip install 'fluid-build[odcs-strict]')") + # markup=False: Rich would read ``[odcs-strict]`` as a tag and drop it. + cprint( + "○ vowl: not installed (pip install 'data-product-forge[odcs-strict]')", + markup=False, + ) else: report["passes"]["vowl"] = {"ok": True, "diagnostics": diag} cprint( diff --git a/fluid_build/cli/scaffold_ci.py b/fluid_build/cli/scaffold_ci.py index d0cca526a..68c174d59 100644 --- a/fluid_build/cli/scaffold_ci.py +++ b/fluid_build/cli/scaffold_ci.py @@ -287,8 +287,8 @@ def register(subparsers: argparse._SubParsersAction): stages { stage('Setup') { steps { - // Ensure Python + fluid-build are available on the agent. - // If your agent image pre-installs fluid-build, this step + // Ensure Python + data-product-forge are available on the agent. + // If your agent image pre-installs data-product-forge, this step // is a no-op. sh ''' python3 -m pip install --user --quiet data-product-forge diff --git a/fluid_build/copilot/store/backends/sqlite.py b/fluid_build/copilot/store/backends/sqlite.py index abdb7f1e1..d559835a2 100644 --- a/fluid_build/copilot/store/backends/sqlite.py +++ b/fluid_build/copilot/store/backends/sqlite.py @@ -100,7 +100,7 @@ def _init_db(self) -> None: _log.warning( "Sqlite store at %s reports schema version %d but this " "CLI only knows about version %d. Proceeding read-only " - "is recommended; upgrade fluid-build before writing.", + "is recommended; upgrade data-product-forge before writing.", self.path, current, _SCHEMA_VERSION, diff --git a/fluid_build/llm/litellm_backend.py b/fluid_build/llm/litellm_backend.py index 24ce7cd11..9b61d6508 100644 --- a/fluid_build/llm/litellm_backend.py +++ b/fluid_build/llm/litellm_backend.py @@ -95,7 +95,7 @@ def _get_litellm() -> Any: "copilot_litellm_unavailable", "litellm is not installed (this is a hard dependency).", suggestions=[ - "pip install --upgrade fluid-build", + "pip install 'data-product-forge[litellm]'", "Or install litellm directly: pip install litellm", ], ) from exc diff --git a/fluid_build/observability/tracing.py b/fluid_build/observability/tracing.py index 3e133d6e6..c1bf100f5 100644 --- a/fluid_build/observability/tracing.py +++ b/fluid_build/observability/tracing.py @@ -33,8 +33,9 @@ ``OTEL_EXPORTER_OTLP_ENDPOINT`` is unset, :func:`traced_stage` returns the wrapped function unchanged — the decorator is a no-op. This lets teams that don't want OTEL opt out by simply not setting the env var; -teams that want it install via ``data-product-forge[observability]`` -and point the env var at their collector. +teams that want it run ``pip install opentelemetry-sdk +opentelemetry-exporter-otlp-proto-http`` and point the env var at their +collector. **Security note:** span attributes are filtered through the existing :class:`fluid_build.observability.secret_redactor.SecretRedactingFilter` diff --git a/fluid_build/providers/odcs/validation.py b/fluid_build/providers/odcs/validation.py index c75ef4b94..7cb2260db 100644 --- a/fluid_build/providers/odcs/validation.py +++ b/fluid_build/providers/odcs/validation.py @@ -100,7 +100,7 @@ def validate_via_vowl(odcs: Mapping[str, Any]) -> Optional[Dict[str, Any]]: Why: ``jsonschema`` alone misses format-level (uuid/email/...) and unevaluated-properties enforcement; vowl is the Bitol-vendor-listed - validator that catches them. Pip-extra: ``fluid-build[odcs-strict]``. + validator that catches them. Pip-extra: ``data-product-forge[odcs-strict]``. Returns ``None`` when vowl isn't installed (so callers can skip cleanly). Returns ``{api_version, schemas, total_checks}`` on diff --git a/pyproject.toml b/pyproject.toml index 452110404..55312b7fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -349,10 +349,15 @@ postgres = [ # LiteLLM is now a core dependency (above), not an extra. The # ``[litellm]`` alias is kept empty for back-compat with anyone who -# scripted ``pip install fluid-build[litellm]`` during the opt-in +# scripted ``pip install data-product-forge[litellm]`` during the opt-in # preview window. litellm = [] +# mcp is in the base install today. This empty alias lets +# ``pip install 'data-product-forge[mcp]'`` resolve now; it gets filled +# when mcp leaves the base install. +mcp = [] + # V1.5 metadata-source catalog adapters. # # Each ``[]`` extra installs ONE catalog adapter's SDK. The diff --git a/tests/test_litellm_backend.py b/tests/test_litellm_backend.py index cc442f064..e89b09658 100644 --- a/tests/test_litellm_backend.py +++ b/tests/test_litellm_backend.py @@ -477,8 +477,9 @@ def _bad_import(name, *args, **kwargs): assert excinfo.value.event == "copilot_litellm_unavailable" suggestions = excinfo.value.suggestions assert any("pip install" in s for s in suggestions) - # The hint points at re-installing fluid-build (litellm is hard dep). - assert any("fluid-build" in s or "litellm" in s for s in suggestions) + # The hint names the real PyPI distribution, never the `fluid-build` alias. + assert any("data-product-forge[litellm]" in s for s in suggestions) + assert not any("fluid-build" in s for s in suggestions) # --------------------------------------------------------------------------- diff --git a/tests/test_no_wrong_package_name.py b/tests/test_no_wrong_package_name.py new file mode 100644 index 000000000..adb68ba81 --- /dev/null +++ b/tests/test_no_wrong_package_name.py @@ -0,0 +1,210 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""No shipped string may name `fluid-build` as the package to install. + +This project's PyPI distribution is `data-product-forge`. `fluid-build` exists +on PyPI only as an alias the maintainers registered so old releases stay safe. +Install hints, upgrade advice and doc snippets that said `pip install +fluid-build[...]` sent readers to a package that is not this one. + +SCOPE: any `fluid-build` NOT followed by `.` and a letter fails. A +`fluid-build.dev` is a domain, a separate concern covered by the reserved-host +fence; a sentence-final "fluid-build." is not a domain and still fails. Text is +scanned, not the AST, because the offenders were log strings, comments, +docstrings and Markdown alike. Three more rules ride along: + +* an install command (`pip install`, `pip3 install`, `uv pip install`, + `pipx install`) may not name `fluid[-_.]build` at all. PyPI normalises `_` and + `.` to `-`, so `pip install fluid_build` is the same wrong package. The import + package `fluid_build` stays legal everywhere else. +* every `data-product-forge[extra]` must name an extra that exists in + pyproject's `[project.optional-dependencies]`. +* the scan covers the non-.py files that ship in the wheel under `fluid_build/` + (suffixes from MANIFEST.in and the package-data config) and CONTRIBUTING.md. + `tests/`, `docs/` and the root Jenkinsfile are not scanned. + +ALLOWLIST: an identity that must stay as it is, matched by its line (not the +whole file), with the reason beside it. +`tests/` is not scanned. +""" + +from __future__ import annotations + +import pathlib +import re + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - Python < 3.11 + import tomli as tomllib # type: ignore[no-redef] + +ROOT = pathlib.Path(__file__).resolve().parents[1] +PACKAGE = ROOT / "fluid_build" + +WRONG_NAME = re.compile(r"fluid-build(?!\.[A-Za-z])") + +# An install command, then the rest of that command (cut at a shell operator or +# a comment). Any token in the rest that is `fluid[-_.]build` is the wrong dist. +INSTALL_CMD = re.compile(r"(?:\bpip3?|\bpipx|\buv\s+pip)\s+install\b(?P[^&;|#]*)") +WRONG_DIST_TOKEN = re.compile(r"(?i)fluid[-_.]build") + +# `data-product-forge[a,b]`; the `<` of a placeholder like `[]` never matches. +EXTRAS_REF = re.compile(r"data-product-forge\[([A-Za-z0-9_.,\- ]+)\]") + +# Non-.py suffixes that ship under fluid_build/ (MANIFEST.in's recursive-include +# plus the package-data globs in pyproject.toml), and the directories shipped whole. +ASSET_SUFFIXES = { + ".yaml", ".json", ".j2", ".html", ".csv", ".sql", ".md", ".example", ".hcl", +} # fmt: skip +ASSET_DIRS = {"templates", "assets", "static"} + +# (path relative to the repo root, fragment of the allowed line) -> reason. An entry +# must still occur, or the coverage guard below fails, so a stale entry cannot +# linger; any other `fluid-build` in the same file still fails. +ALLOWLIST = { + ("fluid_build/observability/tracing.py", '"service.namespace": "fluid-build"'): ( + "OTel `service.namespace` is telemetry identity, not an install hint" + ), +} + + +def _allowed(rel: str, text: str) -> bool: + return any(rel == path and fragment in text for path, fragment in ALLOWLIST) + + +def _is_shipped_asset(path: pathlib.Path) -> bool: + rel = path.relative_to(PACKAGE) + if path.suffix in ASSET_SUFFIXES: + return True + return rel.parts[0] in ASSET_DIRS or (rel.parts[:2] == ("providers", "templates")) + + +def _scanned_files() -> list[pathlib.Path]: + files = [] + for p in PACKAGE.rglob("*"): + if not p.is_file() or "__pycache__" in p.parts: + continue + if p.suffix == ".py" or _is_shipped_asset(p): + files.append(p) + files += [ + ROOT / n + for n in ("AGENTS.md", "CLAUDE.md", "README.md", "CONTRIBUTING.md", "pyproject.toml") + ] + return sorted(files) + + +def _lines(path: pathlib.Path): + try: + text = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + return + yield from enumerate(text.splitlines(), 1) + + +def _wrong_name(line: str) -> bool: + return bool(WRONG_NAME.search(line)) + + +def _wrong_install(line: str) -> bool: + return any(WRONG_DIST_TOKEN.search(m.group("rest")) for m in INSTALL_CMD.finditer(line)) + + +def _occurrences() -> list[tuple[str, int, str]]: + out = [] + for path in _scanned_files(): + rel = path.relative_to(ROOT).as_posix() + for lineno, line in _lines(path): + if _wrong_name(line) or _wrong_install(line): + out.append((rel, lineno, line.strip())) + return out + + +def _declared_extras() -> set[str]: + data = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + return {_norm(k) for k in data["project"]["optional-dependencies"]} + + +def _norm(name: str) -> str: + return re.sub(r"[-_.]+", "-", name.strip().lower()) + + +def _unknown_extras() -> list[str]: + declared = _declared_extras() + out = [] + for path in _scanned_files(): + rel = path.relative_to(ROOT).as_posix() + for lineno, line in _lines(path): + for m in EXTRAS_REF.finditer(line): + for extra in m.group(1).split(","): + if _norm(extra) not in declared: + out.append(f"{rel}:{lineno} data-product-forge[{extra.strip()}]") + return out + + +def test_the_scan_reaches_the_tree(): + """A scanner that found nothing would make the tests below vacuously green.""" + files = _scanned_files() + assert len(files) > 500, f"only found {len(files)} files to scan" + assert any( + p.suffix != ".py" and PACKAGE in p.parents for p in files + ), "no non-.py asset under fluid_build/ was scanned" + assert ROOT / "CONTRIBUTING.md" in files + occurrences = _occurrences() + for path, fragment in ALLOWLIST: + assert any( + rel == path and fragment in text for rel, _, text in occurrences + ), f"allowlisted occurrence {fragment!r} in {path} not seen" + assert {"litellm", "local", "odcs-strict", "mcp"} <= _declared_extras() + + +def test_no_shipped_string_names_the_wrong_package(): + offenders = [ + f"{rel}:{lineno} {text}" for rel, lineno, text in _occurrences() if not _allowed(rel, text) + ] + assert offenders == [], ( + "the PyPI distribution is `data-product-forge`, not `fluid-build` " + "(or `fluid_build`, which PyPI treats as the same name); " + "install hints must name it:\n " + "\n ".join(offenders) + ) + + +def test_every_named_extra_exists(): + """`pip install 'data-product-forge[observability]'` against a missing extra + installs the bare package and only warns, so the hint silently does nothing.""" + unknown = _unknown_extras() + assert unknown == [], ( + "these name an extra that pyproject's [project.optional-dependencies] does " + "not declare:\n " + "\n ".join(unknown) + ) + + +def test_the_rules_catch_what_they_should_and_spare_what_they_should_not(): + for bad in ( + "pip install fluid-build", + "pip install 'fluid-build[litellm]'", + "uv pip install fluid_build", + "pipx install fluid.build", + "pip3 install --upgrade Fluid-Build", + ): + assert _wrong_install(bad), f"missed {bad!r}" + for ok in ( + "pip install 'data-product-forge[mcp]'", + "python -m fluid_build.cli --help", + "pip install foo && python -m fluid_build.cli", + 'User-Agent: "FLUID-Build-Tool/1.0"', + ): + assert not _wrong_install(ok), f"false positive on {ok!r}" + assert _wrong_name("Upgrade fluid-build.") + assert not _wrong_name("see https://fluid-build.dev/docs") diff --git a/tests/test_odcs.py b/tests/test_odcs.py index ce8e1259b..6f25d87f4 100644 --- a/tests/test_odcs.py +++ b/tests/test_odcs.py @@ -548,6 +548,19 @@ def test_json_file_parsed_correctly(self, tmp_path): code = _run_odcs_validate(self._args(odcs_file)) assert code == 0 + def test_vowl_missing_hint_keeps_its_extra(self, tmp_path, capsys): + # cprint renders Rich markup, which swallowed ``[odcs-strict]`` and told + # users to install the base package, which does not bring vowl. + odcs_file = tmp_path / "c.yaml" + odcs_file.write_text(yaml.dump(_odcs_contract())) + with ( + patch("fluid_build.providers.odcs.validation.collect_errors", return_value=[]), + patch("fluid_build.providers.odcs.validation.validate_via_vowl", return_value=None), + ): + code = _run_odcs_validate(self._args(odcs_file, vowl=True)) + assert code == 0 + assert "pip install 'data-product-forge[odcs-strict]'" in capsys.readouterr().out + # --------------------------------------------------------------------------- # _run_odcs_info (argparse helper) From e59e325c93878a6b90554f2076dd0ef36ecf9a9b Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:12:15 +0100 Subject: [PATCH 11/26] fix(diff): fluid diff --baseline sees the columns of a 0.7.x contract (#789) fluid diff --baseline read an expose's columns from exposes[].schema, but a 0.7.x contract keeps them in exposes[].contract.schema, so a removed column, a type change or a new NOT NULL column reported "No changes detected.", --fail-on-breaking exited 0, and the fluid forge --refine preview said "No changes vs the existing contract." Columns now come from contract.schema when schema is absent, and a column is NOT NULL when required is true or nullable is false. * fix(diff): fluid diff --baseline sees the columns of a 0.7.x contract * fix(diff): a column is NOT NULL when required or nullable: false says so * test(diff): reuse the shared 0.7.6 fixture and pin which spelling names a nullability finding --- CHANGELOG.md | 15 ++ fluid_build/api/changelog_rules.py | 77 ++++++- tests/fragments/test_one_loader_parity.py | 21 +- tests/test_diff_changelog.py | 266 ++++++++++++++++++++++ tests/test_refine_diff.py | 29 +++ 5 files changed, 379 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2f809e7e..1a96a1fcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -132,6 +132,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`fluid diff --baseline` reported nothing for column changes in a 0.7.x contract.** It read + columns from `exposes[].schema`, but a 0.7.x contract keeps them in `exposes[].contract.schema` + (as does every `fluid init` template), so a removed column, a type change or a new NOT NULL + column was missed, `--fail-on-breaking` exited 0, and the `fluid forge --refine` preview said + "No changes vs the existing contract." It now reads `contract.schema` when `schema` is absent, + and a column is NOT NULL when `required` is true or `nullable` is `false`, for an added column + and for a change in nullability (a flat `schema` column that says only `required: true` was + nullable before). Contracts that used to diff clean can now report breaking changes (a removed + column, a type change that is not a widening, an added `required: true` column), so a pipeline + that runs `fluid diff --baseline --fail-on-breaking` can start exiting 1. A finding for a 0.7.x + column is reported at `exposes[N].contract.schema.`, and a nullability one at its + `.required`. Still not compared in a 0.7.x contract: quality-rule severity and a column's + `sensitivity`. A 0.7.x primary key is not recognised either, so a key column that becomes + nullable is reported as non-breaking. + - **A failed `fluid apply` on OpenTofu printed raw JSON and no way forward.** The runner uses `-json`, so the error was the last 800 characters of diagnostic lines. It now reads the diagnostic (the summary, the resource address and the provider's detail) and says that OpenTofu diff --git a/fluid_build/api/changelog_rules.py b/fluid_build/api/changelog_rules.py index 280262870..7cd6e622e 100644 --- a/fluid_build/api/changelog_rules.py +++ b/fluid_build/api/changelog_rules.py @@ -175,6 +175,52 @@ def _columns_by_name(columns: Sequence[Mapping[str, Any]]) -> dict[str, Mapping[ return out +def _expose_schema(expose: Mapping[str, Any]) -> Tuple[Sequence[Any], Optional[str]]: + """Return an expose's columns and where, below the expose, they were found. + + Pre-0.7 contracts keep their columns at ``schema``; 0.7.x contracts keep + them at ``contract.schema``. The flat ``schema`` is read when it is + present and ``contract.schema`` only when it is absent, so a contract + that carries both is read as it was before 0.7.x. The location is + ``None`` when the expose carries no schema at all. + """ + columns = expose.get("schema") + if columns is not None: + return columns or [], "schema" + contract = expose.get("contract") + columns = contract.get("schema") if isinstance(contract, Mapping) else None + if columns is not None: + return columns or [], "contract.schema" + return [], None + + +def _is_nullable(column: Mapping[str, Any]) -> bool: + """Whether *column* admits NULL. + + Pre-0.7 columns spell NOT NULL ``nullable: false``. A 0.7.x column has no + ``nullable`` key and spells it ``required: true``. A column is NOT NULL + when either says so: the IaC emitters read ``required`` alone, by + truthiness, so a column that declares both and contradicts itself must not + be read as nullable. + """ + return column.get("nullable") is not False and not column.get("required") + + +def _nullability_key(old: Mapping[str, Any], new: Mapping[str, Any]) -> str: + """The key a nullability change is reported at: the one the contract uses. + + A 0.7.x column says ``required``, a pre-0.7 one ``nullable``. The new + column decides; the old one does when the new says neither. A column that + carries both is reported at ``nullable``. + """ + for column in (new, old): + if "nullable" in column: + return "nullable" + if "required" in column: + return "required" + return "nullable" + + def _primary_key_set(expose: Mapping[str, Any]) -> frozenset[str]: """Extract the set of primary-key column names from an expose dict. @@ -187,7 +233,7 @@ def _primary_key_set(expose: Mapping[str, Any]) -> frozenset[str]: for n in expose_pk: if isinstance(n, str): pk.add(n) - for col in expose.get("schema", []) or []: + for col in _expose_schema(expose)[0]: if isinstance(col, Mapping) and col.get("primaryKey") is True: name = col.get("name") if isinstance(name, str): @@ -201,13 +247,19 @@ def diff_columns( expose_id: str, expose_idx: int, ) -> List[Change]: - """Classify per-column changes between two versions of the same expose.""" - base_cols = baseline_expose.get("schema", []) or [] - new_cols = new_expose.get("schema", []) or [] + """Classify per-column changes between two versions of the same expose. + + Columns are read from ``schema`` or, in a 0.7.x contract, from + ``contract.schema`` (see :func:`_expose_schema`). + """ + base_cols, base_at = _expose_schema(baseline_expose) + new_cols, new_at = _expose_schema(new_expose) base_by_name = _columns_by_name(base_cols) new_by_name = _columns_by_name(new_cols) pk = _primary_key_set(new_expose) | _primary_key_set(baseline_expose) - path_prefix = f"exposes[{expose_idx}].schema" + # Paths follow the layout the new contract uses (the baseline's, when the + # new one gave the expose no schema). + path_prefix = f"exposes[{expose_idx}].{new_at or base_at or 'schema'}" changes: List[Change] = [] @@ -228,7 +280,7 @@ def diff_columns( # (downstream pipelines may insert rows that fail the NOT NULL check). for col_name in new_by_name.keys() - base_by_name.keys(): col = new_by_name[col_name] - is_nullable = col.get("nullable", True) is not False + is_nullable = _is_nullable(col) severity = "non_breaking" if is_nullable else "breaking" description = ( f"column '{col_name}' added to expose '{expose_id}'" @@ -326,14 +378,15 @@ def diff_columns( ) # Nullability changes. - old_null = old.get("nullable", True) is not False - new_null = new.get("nullable", True) is not False + old_null = _is_nullable(old) + new_null = _is_nullable(new) if old_null != new_null: + key = _nullability_key(old, new) if new_null and col_name in pk: # PK column became nullable — breaking. changes.append( Change( - path=f"{path_prefix}.{col_name}.nullable", + path=f"{path_prefix}.{col_name}.{key}", kind="primary_key_nullable", severity="breaking", description=( @@ -348,7 +401,7 @@ def diff_columns( # NOT NULL added to a previously-nullable column — breaking. changes.append( Change( - path=f"{path_prefix}.{col_name}.nullable", + path=f"{path_prefix}.{col_name}.{key}", kind="column_nullable_tightened", severity="breaking", description=( @@ -363,7 +416,7 @@ def diff_columns( # Loosening to nullable on a non-PK — non-breaking. changes.append( Change( - path=f"{path_prefix}.{col_name}.nullable", + path=f"{path_prefix}.{col_name}.{key}", kind="column_nullable_loosened", severity="non_breaking", description=f"column '{col_name}' is now nullable", @@ -688,7 +741,7 @@ def _diff_nested_fields( for fname in new_by_name.keys() - base_by_name.keys(): field = new_by_name[fname] - is_nullable = field.get("nullable", True) is not False + is_nullable = _is_nullable(field) severity = "non_breaking" if is_nullable else "breaking" changes.append( Change( diff --git a/tests/fragments/test_one_loader_parity.py b/tests/fragments/test_one_loader_parity.py index df4a305ed..3340fa5b7 100644 --- a/tests/fragments/test_one_loader_parity.py +++ b/tests/fragments/test_one_loader_parity.py @@ -1121,8 +1121,7 @@ def test_g12_the_checks_look_for_the_prod_file( def test_g12_a_contract_tests_baseline_is_the_flat_twins_and_sees_a_column_removed_in_a_fragment( fx: Path, flat: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch ) -> None: - """``contract-tests`` reads ``exposes[].contract.schema``: the check that does see 0.7.x - columns (``diff`` does not: G-13).""" + """``contract-tests`` reads ``exposes[].contract.schema``, as ``diff`` does (G-13).""" baselines = {} for name, contract in (("fx", fx), ("flat", flat)): argv = ["contract-tests", "contract.fluid.yaml", "--env", "prod"] @@ -1172,12 +1171,6 @@ def test_g13_an_expose_the_baseline_has_and_the_ref_contract_lost_is_a_breaking_ assert "expose 'out' removed" not in text and "added" not in text -@pytest.mark.xfail( - strict=True, - reason="BUG G13-diff-columns: fluid diff --baseline reads exposes[].schema, but a 0.7.x " - "contract keeps its columns in exposes[].contract.schema, so a removed column is " - "'No changes detected' (a flat contract too)", -) def test_g13_a_column_removed_inside_the_schema_fragment_is_detected( fx: Path, flat: Path, capsys: pytest.CaptureFixture ) -> None: @@ -1187,19 +1180,13 @@ def test_g13_a_column_removed_inside_the_schema_fragment_is_detected( rc, text = cli(["diff", fx, "--baseline", flat, "--fail-on-breaking"], capsys) assert rc == 1, text - assert "column 'label' removed" in text + assert _squash("column 'label' removed") in _squash(text) -@pytest.mark.xfail( - strict=True, - reason="BUG G13-diff-columns: fluid diff --baseline reads exposes[].schema, but a 0.7.x " - "contract keeps its columns in exposes[].contract.schema, so a removed column is " - "'No changes detected' (flat contract too)", -) def test_g13_a_column_removed_from_a_flat_contract_is_detected_too( tmp_path: Path, flat: Path, capsys: pytest.CaptureFixture ) -> None: - """The same removal in the flat twin: the gap is in ``diff``, not in the refs.""" + """The same removal in the flat twin: ``diff`` finds it whether or not the schema is a ref.""" document = _read(flat) document["exposes"][0]["contract"]["schema"] = document["exposes"][0]["contract"]["schema"][:1] changed = _write(tmp_path / "changed" / "contract.fluid.yaml", _dump(document)) @@ -1207,7 +1194,7 @@ def test_g13_a_column_removed_from_a_flat_contract_is_detected_too( rc, text = cli(["diff", changed, "--baseline", flat, "--fail-on-breaking"], capsys) assert rc == 1, text - assert "column 'label' removed" in text + assert _squash("column 'label' removed") in _squash(text) # ── G-14 · the exports ───────────────────────────────────────────────────── diff --git a/tests/test_diff_changelog.py b/tests/test_diff_changelog.py index 4823e0f8a..56ab7b21a 100644 --- a/tests/test_diff_changelog.py +++ b/tests/test_diff_changelog.py @@ -38,6 +38,7 @@ diff_sovereignty, ) from fluid_build.api.changelog_types import Change, ChangelogReport +from tests.test_schema_076_preview_additions import _base_contract FIXTURES = Path(__file__).parent / "fixtures" / "changelog" @@ -642,6 +643,271 @@ def test_pii_removed_is_info_signal(): assert pii[0].severity == "info" +# --------------------------------------------------------------------------- +# 0.7.x contracts keep their columns under exposes[].contract.schema +# --------------------------------------------------------------------------- + + +def _contract_076(columns): + """The valid 0.7.6 contract the schema tests share, with ``columns`` on its one expose. + + 0.7.6 closes ``$defs.column``: it has no ``nullable``, ``primaryKey`` or + ``pii`` key, and NOT NULL is spelled ``required: true``. + """ + contract = _base_contract("0.7.6") + contract["exposes"][0]["contract"]["schema"] = columns + return contract + + +_ID_COLUMN = {"name": "id", "type": "string", "required": True} + + +def test_076_fixture_validates_against_the_bundled_schema(): + """The shape every test below diffs: if 0.7.6 stops accepting it they prove nothing.""" + from fluid_build.schema_manager import FluidSchemaManager + + columns = [_ID_COLUMN, {"name": "note", "type": "string", "required": False}] + result = FluidSchemaManager().validate_contract( + _contract_076(columns), schema_version="0.7.6", offline_only=True + ) + assert result.is_valid, result.errors + + +def test_076_identical_contracts_have_no_changes(): + assert compare_contracts(_contract_076([_ID_COLUMN]), _contract_076([_ID_COLUMN])).total == 0 + + +@pytest.mark.parametrize( + "column, severity", + [ + pytest.param({"name": "note", "type": "string"}, "non_breaking", id="plain"), + pytest.param( + {"name": "note", "type": "string", "required": False}, + "non_breaking", + id="required-false", + ), + pytest.param( + {"name": "note", "type": "string", "required": True}, "breaking", id="required-true" + ), + ], +) +def test_076_added_column_severity_follows_required(column, severity): + report = compare_contracts(_contract_076([_ID_COLUMN]), _contract_076([_ID_COLUMN, column])) + + assert report.total == 1 + (change,) = getattr(report, severity) + assert change.kind == "column_added" + assert change.path == "exposes[0].contract.schema.note" + assert ("NOT NULL" in change.description) == (severity == "breaking") + + +def test_076_column_removed_is_breaking(): + old = _contract_076([_ID_COLUMN, {"name": "note", "type": "string"}]) + + report = compare_contracts(old, _contract_076([_ID_COLUMN])) + + assert report.total == 1 + (change,) = report.breaking + assert change.kind == "column_removed" + assert change.path == "exposes[0].contract.schema.note" + + +@pytest.mark.parametrize( + "old_type, new_type, severity, kind", + [ + ("string", "int", "breaking", "column_type_changed"), + ("bigint", "int", "breaking", "column_type_changed"), + ("int", "bigint", "non_breaking", "column_type_widened"), + ("decimal(10,2)", "decimal(20,2)", "non_breaking", "column_type_widened"), + ], +) +def test_076_column_type_change_is_classified(old_type, new_type, severity, kind): + old = _contract_076([{"name": "id", "type": old_type}]) + new = _contract_076([{"name": "id", "type": new_type}]) + + report = compare_contracts(old, new) + + assert report.total == 1 + (change,) = getattr(report, severity) + assert change.kind == kind + assert change.path == "exposes[0].contract.schema.id.type" + assert (change.before, change.after) == (old_type.upper(), new_type.upper()) + + +@pytest.mark.parametrize( + "old_required, new_required, severity, kind", + [ + (False, True, "breaking", "column_nullable_tightened"), + (True, False, "non_breaking", "column_nullable_loosened"), + ], +) +def test_076_required_flip_is_a_nullability_change(old_required, new_required, severity, kind): + old = _contract_076([{"name": "id", "type": "string", "required": old_required}]) + new = _contract_076([{"name": "id", "type": "string", "required": new_required}]) + + report = compare_contracts(old, new) + + assert report.total == 1 + (change,) = getattr(report, severity) + assert change.kind == kind + assert change.path == "exposes[0].contract.schema.id.required" + assert (change.before, change.after) == (not old_required, not new_required) + + +@pytest.mark.parametrize( + "column, severity", + [ + pytest.param({}, "non_breaking", id="neither"), + pytest.param({"nullable": True}, "non_breaking", id="nullable-true"), + pytest.param({"nullable": False}, "breaking", id="nullable-false"), + pytest.param({"required": False}, "non_breaking", id="required-false"), + pytest.param({"required": True}, "breaking", id="required-true"), + pytest.param({"required": 1}, "breaking", id="required-truthy"), + pytest.param( + {"required": True, "nullable": True}, "breaking", id="required-beats-nullable" + ), + pytest.param( + {"required": False, "nullable": False}, "breaking", id="nullable-false-stands" + ), + ], +) +def test_a_column_is_not_null_when_required_or_nullable_false_says_so(column, severity): + """The emitters read ``required`` alone, so a column that declares both and contradicts + itself is read as the stricter of the two, never as nullable.""" + old = {"id": "x", "schema": []} + new = {"id": "x", "schema": [{"name": "n", "type": "INT", **column}]} + + changes = diff_columns(old, new, "x", 0) + + assert [(c.kind, c.severity) for c in changes] == [("column_added", severity)] + + +@pytest.mark.parametrize( + "old_column, new_column, primary_key, kind, key", + [ + ({"nullable": True}, {"nullable": False}, False, "column_nullable_tightened", "nullable"), + ({"nullable": False}, {"nullable": True}, False, "column_nullable_loosened", "nullable"), + ({}, {"required": True}, False, "column_nullable_tightened", "required"), + ({"required": True}, {}, False, "column_nullable_loosened", "required"), + # Spelled differently on the two sides: the new column decides. + ({"nullable": False}, {"required": False}, False, "column_nullable_loosened", "required"), + ({"required": True}, {"nullable": True}, False, "column_nullable_loosened", "nullable"), + ({"nullable": False}, {"nullable": True}, True, "primary_key_nullable", "nullable"), + ({"required": True}, {"required": False}, True, "primary_key_nullable", "required"), + ], +) +def test_a_nullability_change_is_reported_at_the_key_the_contract_uses( + old_column, new_column, primary_key, kind, key +): + def expose(column): + columns = [{"name": "n", "type": "INT", **column}] + return {"id": "x", "schema": columns, **({"primaryKey": ["n"]} if primary_key else {})} + + changes = diff_columns(expose(old_column), expose(new_column), "x", 0) + + assert [(c.kind, c.path) for c in changes] == [(kind, f"exposes[0].schema.n.{key}")] + + +def test_nested_field_added_required_is_breaking(): + old = { + "id": "x", + "schema": [ + {"name": "address", "type": "STRUCT", "fields": [{"name": "street", "type": "STRING"}]} + ], + } + new = { + "id": "x", + "schema": [ + { + "name": "address", + "type": "STRUCT", + "fields": [ + {"name": "street", "type": "STRING"}, + {"name": "country", "type": "STRING", "required": True}, + ], + } + ], + } + + changes = diff_columns(old, new, "x", 0) + + assert [(c.kind, c.severity) for c in changes] == [("nested_field_added", "breaking")] + + +def test_flat_schema_wins_when_an_expose_carries_both_layouts(): + a, b, c = ( + {"name": "a", "type": "STRING"}, + {"name": "b", "type": "STRING"}, + {"name": "c", "type": "STRING"}, + ) + old = {"id": "x", "schema": [a], "contract": {"schema": [a, b]}} + new = {"id": "x", "schema": [a, c], "contract": {"schema": [a, b]}} + + changes = diff_columns(old, new, "x", 0) + + assert [(ch.kind, ch.path) for ch in changes] == [("column_added", "exposes[0].schema.c")] + + +def _contract_flat(columns): + """``_contract_076``'s pre-0.7 twin: columns at ``exposes[].schema``, NOT NULL as ``nullable``.""" + contract = _contract_076([]) + contract["exposes"] = [{"id": "orders", "schema": columns}] + return contract + + +def test_moving_columns_from_schema_to_contract_schema_is_not_a_change(): + """A pre-0.7 baseline against its 0.7.x rewrite: ``nullable: false`` is ``required: true``.""" + old = _contract_flat([{"name": "id", "type": "STRING", "nullable": False}]) + new = _contract_076([{"name": "id", "type": "string", "required": True}]) + + assert compare_contracts(old, new).total == 0 + + +def test_a_column_dropped_while_moving_to_contract_schema_is_still_found(): + old = _contract_flat([{"name": "id", "type": "STRING"}, {"name": "note", "type": "STRING"}]) + new = _contract_076([{"name": "id", "type": "string"}]) + + report = compare_contracts(old, new) + + assert report.total == 1 + assert [(c.kind, c.path) for c in report.breaking] == [ + ("column_removed", "exposes[0].contract.schema.note") + ] + + +@pytest.mark.parametrize("fail_on_breaking, expected_rc", [(True, 1), (False, 0)]) +def test_cli_diff_baseline_sees_a_required_column_added_to_a_076_contract( + tmp_path, fail_on_breaking, expected_rc +): + """``--fail-on-breaking`` is the exit code CI gates on; it must see 0.7.x columns.""" + import yaml + + from fluid_build.cli import diff as diff_mod + + old = tmp_path / "old.fluid.yaml" + new = tmp_path / "new.fluid.yaml" + old.write_text(yaml.safe_dump(_contract_076([_ID_COLUMN]))) + added = {"name": "region", "type": "string", "required": True} + new.write_text(yaml.safe_dump(_contract_076([_ID_COLUMN, added]))) + out = tmp_path / "diff.json" + args = argparse.Namespace( + contract=str(new), + baseline=str(old), + env=None, + state=None, + out=str(out), + exit_on_drift=False, + fail_on_breaking=fail_on_breaking, + format="json", + cmd="diff", + ) + + rc = diff_mod.run(args, logging.getLogger("test.diff")) + + assert rc == expected_rc + assert [c["kind"] for c in json.loads(out.read_text())["breaking"]] == ["column_added"] + + # --------------------------------------------------------------------------- # OTel span attributes on the diff sub-mode # --------------------------------------------------------------------------- diff --git a/tests/test_refine_diff.py b/tests/test_refine_diff.py index 66164a2bb..644658ee5 100644 --- a/tests/test_refine_diff.py +++ b/tests/test_refine_diff.py @@ -40,6 +40,13 @@ def _contract(schema, *, name="Orders"): } +def _contract_076(columns, *, name="Orders"): + """The 0.7.x layout: columns under ``exposes[].contract.schema``, NOT NULL as ``required``.""" + contract = _contract([], name=name) + contract["exposes"] = [{"exposeId": "main", "kind": "table", "contract": {"schema": columns}}] + return contract + + @pytest.fixture() def record_console(): rich_console = pytest.importorskip("rich.console") @@ -97,3 +104,25 @@ def test_never_raises_on_garbage_input(record_console): """The presenter is advisory and must never crash the authoring flow.""" assert render_refine_diff("not-a-mapping", 12345, console=record_console) is False assert render_refine_diff({}, {}, console=record_console) is False + + +@pytest.mark.parametrize( + "extra_in, shown", + [ + pytest.param("new", ("amount", "not null"), id="required-column-added"), + pytest.param("old", ("amount", "column_removed"), id="column-removed"), + ], +) +def test_076_changes_are_rendered_not_swallowed(record_console, extra_in, shown): + """A refine that adds or drops a column of a 0.7.x contract must not say 'No changes'.""" + id_column = {"name": "id", "type": "string"} + amount = {"name": "amount", "type": "number", "required": True} + old = _contract_076([id_column] + ([amount] if extra_in == "old" else [])) + new = _contract_076([id_column] + ([amount] if extra_in == "new" else [])) + + rendered = render_refine_diff(old, new, console=record_console) + + out = record_console.export_text().lower() + assert rendered is True + assert "no changes" not in out + assert all(fragment in out for fragment in shown) From f3b258a4d6413a90948999d45ef9750152fe05b3 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:23:37 +0100 Subject: [PATCH 12/26] feat!: retire redshift and datamesh-manager as providers (#796) * feat!: retire redshift and datamesh-manager as providers `redshift` only forwarded plan and apply to `aws`. Its real job, the Redshift rollback DDL, is dispatched by snapshot name and never went through the registry. Data Mesh Manager is a catalog, already reached through `fluid publish --target datamesh-manager` and `fluid datamesh-manager`. `fluid providers` now lists aws, databricks, gcp, local and snowflake, plus third-party plugins. - datamesh_manager sets __fluid_no_autoregister__, as the exporters did in 0.10.0. RedshiftProvider stays as the rollback-only helper. - The error catalog and the user-facing provider lists name the real built-ins: no azure, and databricks added. AGENTS.md describes Data Mesh Manager as a catalog backend. BREAKING CHANGE: `--provider redshift` and `--provider datamesh-manager` (also through FLUID_PROVIDER) are no longer accepted. They exit 2 with a one-line fix: use `--provider aws`, or `fluid publish --target datamesh-manager`. The fix comes from one map, RETIRED_PROVIDERS, which the global flag check, build_provider and the plan gate share. A plugin that registers either name still wins. * fix: reserve the retired provider names, and serve a redshift platform with aws Fixes from the security review and the defect hunt: - Plugins cannot claim `redshift` or `datamesh-manager`. Entry points load before the built-ins, so a plugin could take a retired name and replace its hint. Discovery now skips such a registration before `ep.load()`, so the plugin's code never runs, and logs a WARNING (`provider_retired_name_reserved`) naming the distribution. An explicit `register_provider(...)` call in code still works. Pattern from llm/provider_plugins._reserved_builtin_names. - A contract that names `redshift` as a platform resolves to `aws` through RETIRED_SUCCESSORS (`builds[].execution.runtime.platform` still allows it). This covers resolve_provider_from_contract, apply._declared_platforms, policy-apply and export. So plan, apply and generate iac agree again, as when RedshiftProvider forwarded to AwsProvider. A `--provider redshift` the user types still gets the hint. - The hint no longer needs provider discovery, so it still shows when the registry fails to load or an allowlist hides every provider. - CHANGELOG: Redshift objects apply through `aws`. The rollback claim is dropped. * fix: retired-name helpers pass a non-string name through From the security re-review: resolve_retired_platform and _claims_retired_name raised AttributeError on a non-string name, which an LLM-drafted contract can carry. The previous commit passed such a value through unchanged. Both now return it untouched. * fix: the reserved-name warning names the plugin an operator must remove The provider_retired_name_reserved WARNING carried the plugin only in a log extra, which the CLI formatters drop. An operator saw the event but not which package to uninstall, as two reviewers found. The message now names the plugin and the retired name, repr'd so control characters are escaped. The evt and plugin extras stay for log parsers. * test: one import style per module in test_retired_providers Code-quality review: fluid_build.providers.datamesh_manager and fluid_build.cli were each imported with both `import` and `from ... import` in this file. The test still checks that DataMeshManagerProvider is exported, now through the module. --- AGENTS.md | 2 +- CHANGELOG.md | 12 + README.md | 18 +- fluid_build/_error_catalog.py | 11 +- fluid_build/cli/__init__.py | 29 +- fluid_build/cli/_common.py | 15 +- fluid_build/cli/apply.py | 7 +- fluid_build/cli/config.py | 2 +- fluid_build/cli/contract_validation.py | 2 +- fluid_build/cli/export.py | 4 +- fluid_build/cli/plan.py | 3 + fluid_build/cli/policy_apply.py | 5 +- fluid_build/cli/test.py | 6 +- fluid_build/providers/__init__.py | 76 ++++- fluid_build/providers/aws/__init__.py | 6 +- .../providers/aws/redshift_provider.py | 13 +- .../providers/datamesh_manager/__init__.py | 20 +- tests/cli/test_apply_simple_provider.py | 23 ++ tests/providers/test_retired_providers.py | 269 ++++++++++++++++++ tests/test_global_provider_flag.py | 12 +- tests/test_plugin_policy_governs_builtins.py | 16 +- 21 files changed, 481 insertions(+), 70 deletions(-) create mode 100644 tests/providers/test_retired_providers.py diff --git a/AGENTS.md b/AGENTS.md index ac84efe8e..185461a9a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -199,7 +199,7 @@ new extra installed). `uv` is dev-time only — not a runtime dep. - **Changing what `fluid plan` emits**: `plan.json` must carry both `planDigest` and `bundleDigest` (re-run `inject_digests` after any mutation to the dict). Actions must carry BOTH `op` and `action_type` fields — apply.py's provider dispatcher reads `op`, display/viz reads `action_type`. Dropping either silently breaks the pipeline. - **Generating a new CI system template**: extend `fluid_build/forge/core/pipeline_templates.py`. The 11-stage sequence + the `--install-mode {pypi,dev-source}` semantics must be preserved across systems. Jenkins is the reference implementation; GitHub Actions / GitLab / Azure / Bitbucket / CircleCI / Tekton port from it. - **Adding a new Path-B scheduler**: extend `fluid_build/providers//plan/schedule.py` with a `SchedulePlanner`-compatible interface. Stage 6 `fluid plan` invokes it; stage 7 apply executes the resulting actions alongside DDL. -- **Adding a new publish target / catalog backend**: drop a new module under `fluid_build/build_runners/catalog_registrars/.py` that defines a `CatalogRegistrar` (the sync HTTP-level shape — see `datahub.py` as the reference) and ends with a `register_catalog_backend(CatalogBackendSpec(...))` call. The spec declares the canonical name, optional aliases, the registrar factory, and a flat `env_vars` mapping (config key → tuple of accepted env vars). That single file makes the backend visible on both publish surfaces — `fluid publish --target ` and contract `properties.catalog.register: []` — and applies the env-var overrides via `config_manager._apply_catalog_env_overrides`. No edits to `providers/catalogs/__init__.py`, `_catalog.py`, or `publish.py` are required. The two legacy native async providers (`fluid-command-center`, `datamesh-manager`) keep custom config shapes and live as hand-wired `BaseCatalogProvider` subclasses under `providers/catalogs/`; new backends should prefer the registrar shape unless they truly need the async-native surface. +- **Adding a new publish target / catalog backend**: drop a new module under `fluid_build/build_runners/catalog_registrars/.py` that defines a `CatalogRegistrar` (the sync HTTP-level shape — see `datahub.py` as the reference) and ends with a `register_catalog_backend(CatalogBackendSpec(...))` call. The spec declares the canonical name, optional aliases, the registrar factory, and a flat `env_vars` mapping (config key → tuple of accepted env vars). That single file makes the backend visible on both publish surfaces — `fluid publish --target ` and contract `properties.catalog.register: []` — and applies the env-var overrides via `config_manager._apply_catalog_env_overrides`. No edits to `providers/catalogs/__init__.py`, `_catalog.py`, or `publish.py` are required. Command Center (`fluid-command-center`) keeps a custom config shape as a hand-wired `BaseCatalogProvider` subclass under `providers/catalogs/`; Data Mesh Manager is a catalog backend (`build_runners/catalog_registrars/datamesh_manager.py`) reached with `fluid publish --target datamesh-manager`, plus the `fluid datamesh-manager` command, and is not a provider; new backends should prefer the registrar shape unless they truly need the async-native surface. - **Adding a copilot tool that takes a path**: accept `workspace_root: Optional[Path]` as a kwarg from `dispatch_tool_call`; confine the LLM-supplied path with `resolved.relative_to(workspace_root)` and apply a suffix allow-list + size cap. Reference implementation: `cli/forge_copilot_tools.py::_dispatch_read_sample_schema`. - **Emitting new SQL DDL**: identifiers must go through `fluid_build/providers/_sql_safety.py::validate_ident`; string literals through `quote_string_literal`. Inline `.replace("'", "''")` is considered a regression — it diverges from the central helper. - **Adding a secret pattern to logs**: extend `fluid_build/providers/snowflake/util/logging.py::SENSITIVE_PATTERNS`/`SENSITIVE_KEYS` AND mirror the addition into `fluid_build/observability/secret_redactor.py` to keep the two redactor layers symmetric. diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a96a1fcb..245f37248 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -217,6 +217,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 passes when the region matches the declaration, fails with the fix when it does not, and is skipped when the contract declares none. +### Removed + +- **`redshift` and `datamesh-manager` are no longer providers.** Neither deployed anything as a + provider: `redshift` only forwarded to `aws`, and Data Mesh Manager is a catalog. `fluid providers` now lists `aws`, `databricks`, `gcp`, `local` and `snowflake`, plus any + third-party plugin. Redshift objects apply through the `aws` provider, and Data Mesh Manager + publishing is unchanged (`fluid publish --target datamesh-manager`, `fluid datamesh-manager`). + **Breaking (CLI surface):** `--provider redshift` and `--provider datamesh-manager` (and + `FLUID_PROVIDER`) are no longer accepted; they exit 2 with the replacement + (`--provider aws`, or `fluid publish --target datamesh-manager`) instead of a bare "unknown + provider". A contract's `redshift` build runtime or expose platform is served by `aws`, and plugins + cannot claim either retired name. + ### Security - **`validate_ident` let a name with a trailing newline through.** Python's `$` also matches just diff --git a/README.md b/README.md index 7b2156aac..cff198fd5 100644 --- a/README.md +++ b/README.md @@ -286,19 +286,15 @@ something is a deployment target: `--provider` also accepts `confluent`, for Kafka-side resources, and `auto`, which is the default and reads the target out of the contract's binding. -**`fluid providers` prints a longer list, and it is a different thing.** It is -the registry of everything that registers as a provider plugin, which includes -publishers and a partly-landed emitter: +**`fluid providers` prints these providers and any third-party plugins.** Two +names that older releases listed are no longer providers (0.27.0): - `datamesh_manager` is a **publish** target, reached by - [`fluid publish`](https://agenticstiger.github.io/forge_docs/cli/datamesh-manager.html). - It receives a contract's metadata; it never provisions infrastructure. -- `redshift` has an emitter in the tree but is **not selectable with - `--provider`** — `fluid generate iac --provider redshift` exits with - `invalid choice`. Treat it as unfinished rather than as a fourth cloud. - -Neither belongs in a list of places a data product can be deployed, and this -README previously listed both as though they were. + [`fluid publish --target datamesh-manager`](https://agenticstiger.github.io/forge_docs/cli/datamesh-manager.html) + or `fluid datamesh-manager`. It receives a contract's metadata; it never + provisions infrastructure. `--provider datamesh-manager` exits 2 and says so. +- `redshift` is part of the `aws` provider. `--provider redshift` exits 2 and + says to use `--provider aws`. That column is the set each emitter is capable of. Which resources a particular contract actually produces is visible before anything is applied: `fluid generate iac ` writes the diff --git a/fluid_build/_error_catalog.py b/fluid_build/_error_catalog.py index 30c31226f..db85b031a 100644 --- a/fluid_build/_error_catalog.py +++ b/fluid_build/_error_catalog.py @@ -74,7 +74,7 @@ def slug_for(event: str) -> str: # ── providers ────────────────────────────────────────────────────────── "provider_not_specified": ( [ - "Pass a provider: --provider local|gcp|snowflake|aws|azure", + "Pass a provider: --provider aws|databricks|gcp|local|snowflake", "Or set the FLUID_PROVIDER environment variable", "Run 'fluid providers' to list the available providers", ], @@ -91,7 +91,14 @@ def slug_for(event: str) -> str: "provider_unknown": ( [ "Run 'fluid providers' to see the available providers", - "Valid built-ins: local, gcp, snowflake, aws, azure", + "Valid built-ins: aws, databricks, gcp, local, snowflake", + ], + "providers", + ), + "provider_retired": ( + [ + "Run 'fluid providers' to see the available providers", + "CHANGELOG.md lists it under Removed, with its replacement", ], "providers", ), diff --git a/fluid_build/cli/__init__.py b/fluid_build/cli/__init__.py index 6e4c65356..be42d9d18 100644 --- a/fluid_build/cli/__init__.py +++ b/fluid_build/cli/__init__.py @@ -152,9 +152,10 @@ def build_parser() -> argparse.ArgumentParser: # Deliberately NO argparse ``choices``. The provider set is a registry # (built-ins + ``fluid_build.providers`` entry-point plugins), and the # hardcoded enum disagreed with it in both directions: it rejected - # ``redshift`` and ``datamesh_manager``, which ``fluid providers`` - # lists, and accepted ``azure``, which is not registered at all — and a - # pip-installed provider plugin could never be passed. Enumerating the + # ``redshift`` and ``datamesh_manager`` (then registered; since retired, + # see ``RETIRED_PROVIDERS``), and accepted ``azure``, which is not + # registered at all — and a pip-installed provider plugin could never be + # passed. Enumerating the # registry here is not an option: discovery imports every provider # module, and ``build_parser()`` is the ``fluid --help`` cold path. # The value is validated against the live registry in @@ -344,12 +345,19 @@ def main(argv: Optional[List[str]] = None) -> int: # `--provider` / `FLUID_PROVIDER` keeps its current startup cost # (tests/perf/test_startup_budget.py). requested_provider = getattr(args, "provider", None) + # A retired name needs no discovery: the answer is the same whatever the + # registry holds, and a registry that failed to load must not hide it. + from fluid_build.providers import retired_provider_hint + + retired = bool(requested_provider) and retired_provider_hint(requested_provider) is not None known_providers = ( _known_provider_names() - if requested_provider and requested_provider != _PROVIDER_AUTO + if requested_provider and requested_provider != _PROVIDER_AUTO and not retired else [] ) _validate_global_args(args, cli.logger, known_providers=known_providers) + if retired: + return 2 if requested_provider and known_providers and requested_provider not in known_providers: return 2 @@ -750,8 +758,17 @@ def _validate_global_args( # would drag provider discovery onto the `--help` path. Checking it # here means the accepted values are exactly what `fluid providers` # lists, plugins included. - known = _known_provider_names() if known_providers is None else known_providers - if known and args.provider not in known and args.provider != _PROVIDER_AUTO: + from fluid_build.providers import retired_provider_hint + + retired_hint = retired_provider_hint(args.provider) + known = ( + [] + if retired_hint + else (_known_provider_names() if known_providers is None else known_providers) + ) + if retired_hint: + issues.append(retired_hint) + elif known and args.provider not in known and args.provider != _PROVIDER_AUTO: issues.append( f"Unknown provider '{args.provider}' — installed providers: " f"{', '.join(known)} (see `fluid providers`)" diff --git a/fluid_build/cli/_common.py b/fluid_build/cli/_common.py index acdd84537..c652de397 100644 --- a/fluid_build/cli/_common.py +++ b/fluid_build/cli/_common.py @@ -211,23 +211,28 @@ def resolve_provider_from_contract(contract: Dict[str, Any]) -> tuple[str, Dict[ falls back to top-level binding.platform (Snowflake-style), then builds[].execution.runtime.platform. + A retired provider name used as a platform (``redshift``) resolves to its + successor provider (``aws``). + Returns: (provider_name, location_dict) where location may contain project, region, dataset, bucket, etc. """ + from fluid_build.providers import resolve_retired_platform + for exp in contract.get("exposes", []): binding = exp.get("binding", {}) platform = binding.get("platform", "") if platform: - return platform, binding.get("location", {}) + return resolve_retired_platform(platform), binding.get("location", {}) # Top-level binding (Snowflake-style contracts) top_binding = contract.get("binding", {}) if isinstance(top_binding, dict) and top_binding.get("platform"): - return top_binding["platform"], top_binding.get("location", {}) + return resolve_retired_platform(top_binding["platform"]), top_binding.get("location", {}) for build in contract.get("builds", []): platform = (build.get("execution") or {}).get("runtime", {}).get("platform", "") if platform: - return platform, {} + return resolve_retired_platform(platform), {} return "", {} @@ -251,7 +256,11 @@ def build_provider( # blocked provider as "unknown" sends the operator hunting for a # typo or a missing extra. from fluid_build.plugin_manager import is_allowed + from fluid_build.providers import retired_provider_hint + hint = retired_provider_hint(name) + if hint: + raise CLIError(2, "provider_retired", {"error": hint}) if not is_allowed(name): raise CLIError(2, "provider_blocked_by_policy", {"requested": name}) raise CLIError( diff --git a/fluid_build/cli/apply.py b/fluid_build/cli/apply.py index 088551113..a4c8d45da 100644 --- a/fluid_build/cli/apply.py +++ b/fluid_build/cli/apply.py @@ -1208,7 +1208,10 @@ def _declared_platforms(contract: Dict[str, Any]) -> Tuple[List[_Declared], List ) def _key(platform: Any) -> str: - return str(platform).strip().lower().replace("-", "_") + from fluid_build.providers import resolve_retired_platform + + # ``redshift`` (retired as a provider) is served by its successor, ``aws``. + return resolve_retired_platform(str(platform).strip().lower().replace("-", "_")) builds = [b for b in contract.get("builds") or [] if isinstance(b, dict)] lands: List[_Declared] = [] @@ -1234,7 +1237,7 @@ def _simple_apply_provider(contract: Dict[str, Any], logger: logging.Logger) -> A ``binding.platform`` says where an expose lands and a build's ``execution.runtime.platform`` where it runs; neither is a provider name, - though ``local``, ``gcp``, ``aws``, ``snowflake`` and ``redshift`` are also + though ``local``, ``gcp``, ``aws`` and ``snowflake`` are also providers. Read as one, a Kafka binding or a Flink build's ``runtime.platform: kubernetes`` ended the run as ``provider_unknown``. diff --git a/fluid_build/cli/config.py b/fluid_build/cli/config.py index e9f9150ca..7abc3644d 100644 --- a/fluid_build/cli/config.py +++ b/fluid_build/cli/config.py @@ -457,7 +457,7 @@ def create_production_documentation() -> str: ### Required Environment Variables ```bash # Infrastructure (Required) -export FLUID_PROVIDER="gcp" # or aws, azure, snowflake +export FLUID_PROVIDER="gcp" # or aws, databricks, snowflake export FLUID_PROJECT="your-project-id" # Cloud project identifier export FLUID_REGION="us-central1" # Cloud region diff --git a/fluid_build/cli/contract_validation.py b/fluid_build/cli/contract_validation.py index 6c7c63bc9..85a601350 100644 --- a/fluid_build/cli/contract_validation.py +++ b/fluid_build/cli/contract_validation.py @@ -111,7 +111,7 @@ def _dist_installed(dist_name: str) -> bool: ), "azure": ( "Provider 'azure' validation not yet implemented", - "Supported providers: gcp, snowflake, aws, local. Others coming soon.", + "Supported providers: aws, databricks, gcp, local, snowflake. Others coming soon.", ), } diff --git a/fluid_build/cli/export.py b/fluid_build/cli/export.py index f5adc3d63..fe7bae0b6 100644 --- a/fluid_build/cli/export.py +++ b/fluid_build/cli/export.py @@ -132,7 +132,9 @@ def run(args, logger: logging.Logger) -> int: provider_name = args.provider if not provider_name: # Auto-detect from contract - provider_name = contract.get("provider", "aws") + from fluid_build.providers import resolve_retired_platform + + provider_name = resolve_retired_platform(contract.get("provider", "aws")) if args.verbose: info(logger, f"Auto-detected provider: {provider_name}") diff --git a/fluid_build/cli/plan.py b/fluid_build/cli/plan.py index 9f011c664..3faab1f48 100644 --- a/fluid_build/cli/plan.py +++ b/fluid_build/cli/plan.py @@ -682,6 +682,9 @@ def _gate_provider_flag(requested: Optional[str], logger: logging.Logger) -> Non registry.discover_providers(logger) name = requested.strip().lower().replace("-", "_") if name not in registry.PROVIDERS: + hint = registry.retired_provider_hint(name) + if hint: + raise CLIError(2, "provider_retired", {"error": hint}) raise CLIError( 2, "provider_unknown", {"requested": name, "available": sorted(registry.PROVIDERS)} ) diff --git a/fluid_build/cli/policy_apply.py b/fluid_build/cli/policy_apply.py index 97d21174a..f54b91eeb 100644 --- a/fluid_build/cli/policy_apply.py +++ b/fluid_build/cli/policy_apply.py @@ -70,7 +70,10 @@ def _resolve_from_bindings(data: dict) -> tuple[str, str]: project = b.get("project", "") if provider and project: break - return provider, project + from fluid_build.providers import resolve_retired_platform + + # A platform the contract named, so a retired one resolves to its successor. + return resolve_retired_platform(provider) if provider else provider, project @_traced_stage("policy_apply") diff --git a/fluid_build/cli/test.py b/fluid_build/cli/test.py index 204690fda..3da19ba4d 100644 --- a/fluid_build/cli/test.py +++ b/fluid_build/cli/test.py @@ -103,7 +103,7 @@ def register(subparsers: argparse._SubParsersAction) -> None: - Metadata and governance completeness - Optional drift detection against historical results - Supports all FLUID providers: gcp, snowflake, aws, local. + Supports all FLUID providers: aws, databricks, gcp, local, snowflake. """, epilog=""" Examples: @@ -139,7 +139,9 @@ def register(subparsers: argparse._SubParsersAction) -> None: # --- optional --- p.add_argument("--env", help="Environment overlay (dev/test/prod)") - p.add_argument("--provider", help="Override provider platform (gcp, snowflake, aws, local)") + p.add_argument( + "--provider", help="Override provider platform (aws, databricks, gcp, local, snowflake)" + ) p.add_argument("--project", help="Override project/account ID") p.add_argument("--region", help="Override region/location") diff --git a/fluid_build/providers/__init__.py b/fluid_build/providers/__init__.py index 30c54eab2..242444bf3 100644 --- a/fluid_build/providers/__init__.py +++ b/fluid_build/providers/__init__.py @@ -105,6 +105,68 @@ def _normalize_name(name: str) -> str: return name.strip().lower().replace("-", "_") +#: Names that are no longer providers -> what to do instead. Not +#: registry entries: a user who typed one gets this line (exit 2) rather than a +#: bare "unknown provider". Keys are :func:`_normalize_name` spellings. +RETIRED_PROVIDERS: Dict[str, str] = { + "redshift": "Redshift is part of the aws provider; use --provider aws", + "datamesh_manager": ( + "Data Mesh Manager is a catalog, not a provider; use " + "`fluid publish --target datamesh-manager` (or `fluid datamesh-manager`)" + ), +} + + +#: The provider that now serves a retired name used as a *platform* in a contract +#: (``builds[].execution.runtime.platform: redshift`` still validates). Data Mesh +#: Manager is a catalog, not a platform, so it has none. +RETIRED_SUCCESSORS: Dict[str, str] = {"redshift": "aws"} + + +def resolve_retired_platform(name: str) -> str: + """The successor provider for a retired platform name; any other name unchanged.""" + if not isinstance(name, str): + return name + return RETIRED_SUCCESSORS.get(_normalize_name(name), name) + + +def _claims_retired_name(name: str, who: str, logger: Optional[logging.Logger]) -> bool: + """True (after a WARNING) when a plugin tries to register a retired provider name. + + A retired name stays reserved: an entry point able to claim ``redshift`` would + replace the retirement hint for ``--provider redshift`` and win the platform + lookup for contracts. Only discovery calls this; an explicit + ``register_provider(...)`` call in code stays allowed. + """ + if not isinstance(name, str) or _normalize_name(name) not in RETIRED_PROVIDERS: + return False + # The plugin goes into the message itself, repr'd so control characters are + # escaped: the CLI's formatters print the message only, and this line is how + # an operator learns which package to uninstall. + (logger or _log).warning( + "provider_retired_name_reserved: %r tried to register the retired provider %r; " + "it was skipped", + who, + _normalize_name(name), + extra={ + "evt": "provider_retired_name_reserved", + "_name": _normalize_name(name), + "plugin": who, + }, + ) + return True + + +def retired_provider_hint(name: Optional[str]) -> Optional[str]: + """One-line ``--provider``/``FLUID_PROVIDER`` error for a retired name, else ``None``.""" + if not name: + return None + use = RETIRED_PROVIDERS.get(_normalize_name(name)) + if use is None: + return None + return f"Provider '{name}' was retired: {use}" + + def _add_discovery_error(source: str, modname: str, exc: BaseException) -> None: # DISCOVERY_ERRORS is surfaced to users via registry_dump() / `fluid providers # --debug`, so it must NOT carry raw exception text or a full traceback — either @@ -466,7 +528,7 @@ def discover_providers(logger: Optional[logging.Logger] = None, *, force: bool = return # What the policy now allows is no longer blocked. A name it still - # blocks stays recorded: one registered at import time (redshift, by + # blocks stays recorded: one registered at import time (aws, by # the aws package) is not offered again when its module is cached. from fluid_build.plugin_manager import is_allowed @@ -541,6 +603,12 @@ def _discover_entrypoints(logger: Optional[logging.Logger]) -> None: ) POLICY_BLOCKED.add(_normalize_name(ep.name)) continue + from fluid_build.plugin_manager import entry_point_distribution + + if _claims_retired_name( + ep.name, entry_point_distribution(ep) or str(getattr(ep, "value", ep)), logger + ): + continue try: provider_cls = ep.load() register_provider( @@ -634,6 +702,8 @@ def _auto_register_from_module(mod, logger: Optional[logging.Logger]) -> None: providers_map = getattr(mod, "PROVIDERS", None) if isinstance(providers_map, dict) and providers_map: for name, prov in list(providers_map.items()): + if _claims_retired_name(name, getattr(mod, "__name__", ""), logger): + continue try: register_provider(name, prov, override=False, logger=logger) except Exception as exc: @@ -651,6 +721,8 @@ def _auto_register_from_module(mod, logger: Optional[logging.Logger]) -> None: name = getattr(mod, "NAME", None) prov = getattr(mod, "Provider", None) if isinstance(name, str) and prov is not None: + if _claims_retired_name(name, getattr(mod, "__name__", ""), logger): + return try: register_provider(name, prov, override=False, logger=logger) _safe_log( @@ -686,6 +758,8 @@ def _auto_register_from_module(mod, logger: Optional[logging.Logger]) -> None: inferred = _normalize_name(getattr(cls, "name", cls.__name__).replace("Provider", "")) if not _is_valid_name(inferred): inferred = _normalize_name(getattr(mod, "__name__", "provider").rsplit(".", 1)[-1]) + if _claims_retired_name(inferred, getattr(mod, "__name__", ""), logger): + return register_provider(inferred, cls, override=False, logger=logger) _safe_log( logger, diff --git a/fluid_build/providers/aws/__init__.py b/fluid_build/providers/aws/__init__.py index 852679bf5..70e79879c 100644 --- a/fluid_build/providers/aws/__init__.py +++ b/fluid_build/providers/aws/__init__.py @@ -19,7 +19,8 @@ S3 / Glue / Athena / Lambda / EventBridge / Redshift. * :class:`RedshiftProvider` — owns Redshift's rollback DDL (DROP + CTAS) which is incompatible with the rest of AWS's S3 prefix-copy - semantics; registered under the ``"redshift"`` provider name. + semantics. Not a registered provider (retired as one, see + ``providers.RETIRED_PROVIDERS``); the rollback dispatch imports it directly. The legacy ``aws.py::AWSProvider`` (1601 LOC, registered as ``LegacyAWSProvider``) was deleted — its rollback methods migrated @@ -33,8 +34,5 @@ from .redshift_provider import RedshiftProvider register_provider("aws", AwsProvider) -# Redshift dispatches by its own name so the rollback writer can route -# DROP+CTAS DDL emission to a class with the right semantics. -register_provider("redshift", RedshiftProvider) __all__ = ["AwsProvider", "RedshiftProvider"] diff --git a/fluid_build/providers/aws/redshift_provider.py b/fluid_build/providers/aws/redshift_provider.py index 55595dc84..089de31d4 100644 --- a/fluid_build/providers/aws/redshift_provider.py +++ b/fluid_build/providers/aws/redshift_provider.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Standalone Redshift provider for rollback / DDL emission. +"""Standalone Redshift helper for rollback / DDL emission. The full AWS provider (``fluid_build.providers.aws.AwsProvider``) handles Redshift planning. Redshift's rollback semantics are SQL-DDL-based (``DROP`` + ``CREATE TABLE AS SELECT`` inside a transaction) while the rest of AWS (S3 / Glue / Athena) uses prefix-copy semantics. -Splitting Redshift out as its own provider class keeps the rollback +Splitting Redshift out as its own class keeps the rollback surface modular: ``cli/_rollback_writer.py::_delegate_to_provider`` asks each provider for its restore DDL by name, and Redshift owns its SQL while ``AwsProvider`` returns ``[]`` (S3 prefix-copy is non-DDL). @@ -32,9 +32,10 @@ two unrelated rollback semantics. One class per restore-semantics is cleaner than one class with two switches. -Registered as ``"redshift"`` in the provider registry so -``_delegate_to_provider("redshift", "restore_ddl", snapshot)`` succeeds -without falling through to the legacy lookup. +Not in the provider registry (``redshift`` is no longer a ``--provider``; +Redshift is part of ``aws``). +``_delegate_to_provider("redshift", "restore_ddl", snapshot)`` imports this +class directly, keyed by the snapshot's provider name. """ from __future__ import annotations @@ -46,7 +47,7 @@ class RedshiftProvider(BaseProvider): - """Thin Redshift provider — owns rollback semantics only. + """Thin Redshift helper — owns rollback semantics only. ``plan`` delegates to the full ``AwsProvider``; this class exists primarily so ``restore_ddl`` and ``cleanup_backups`` participate in diff --git a/fluid_build/providers/datamesh_manager/__init__.py b/fluid_build/providers/datamesh_manager/__init__.py index a420a93f3..5ead50cf8 100644 --- a/fluid_build/providers/datamesh_manager/__init__.py +++ b/fluid_build/providers/datamesh_manager/__init__.py @@ -20,17 +20,11 @@ from fluid_build.providers.datamesh_manager.datamesh_manager import DataMeshManagerProvider -# Self-register with the provider registry so ``fluid providers`` lists us -# and ``discover_providers()`` picks us up automatically. -try: - from fluid_build.providers import register_provider +# Data Mesh Manager is a catalog, not a deployment provider: it is reached by +# ``fluid publish --target datamesh-manager`` and ``fluid datamesh-manager``, +# which import the class directly. Opt out of every passive registration +# (including the single-BaseProvider-subclass fallback) so it stays out of +# ``fluid providers``. +__fluid_no_autoregister__ = True - register_provider("datamesh-manager", DataMeshManagerProvider) -except Exception: - pass # Registry not yet available (e.g. during isolated import) - -# Expose NAME + Provider for auto-discovery strategy 2 as a fallback -NAME = "datamesh-manager" -Provider = DataMeshManagerProvider - -__all__ = ["DataMeshManagerProvider", "NAME", "Provider"] +__all__ = ["DataMeshManagerProvider"] diff --git a/tests/cli/test_apply_simple_provider.py b/tests/cli/test_apply_simple_provider.py index e0bbbf5fb..df678d7c8 100644 --- a/tests/cli/test_apply_simple_provider.py +++ b/tests/cli/test_apply_simple_provider.py @@ -125,6 +125,29 @@ def _refusal(contract: Dict[str, Any]) -> CLIError: return exc.value +class TestRetiredRedshiftPlatform: + """``redshift`` stopped being a provider but is still a valid runtime platform; + it resolves to ``aws``, which the retired registry entry forwarded to.""" + + def test_a_redshift_runtime_build_beside_a_local_product_selects_aws(self) -> None: + contract = _contract( + [_expose("orders", "local")], [_sql("make_orders", ["orders"], platform="redshift")] + ) + assert _simple_apply_provider(contract, LOG) == "aws" + + def test_a_redshift_expose_selects_aws(self) -> None: + assert _simple_apply_provider(_contract([_expose("orders", "redshift")]), LOG) == "aws" + + def test_plan_resolves_the_same_way(self) -> None: + from fluid_build.cli._common import resolve_provider_from_contract + + contract = _contract( + [_expose("orders", "local")], [_sql("make_orders", ["orders"], platform="redshift")] + ) + contract["exposes"][0]["binding"].pop("platform") + assert resolve_provider_from_contract(contract)[0] == "aws" + + class TestOutsideForgeCli: """A job that runs outside forge-cli, and what it writes, choose no provider.""" diff --git a/tests/providers/test_retired_providers.py b/tests/providers/test_retired_providers.py new file mode 100644 index 000000000..1805407b9 --- /dev/null +++ b/tests/providers/test_retired_providers.py @@ -0,0 +1,269 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``redshift`` and ``datamesh-manager`` are no longer providers. + +``redshift`` only forwarded to ``aws`` (its rollback DDL is reached through the +snapshot's provider name) and Data Mesh Manager is a catalog. A retired name is +not "unknown": it exits 2 with the replacement, from ``--provider`` and +``FLUID_PROVIDER`` alike. What they did keeps working: the Redshift rollback +dispatch and ``fluid publish --target datamesh-manager``. +""" + +from __future__ import annotations + +import logging + +import pytest + +from fluid_build import providers as P +from fluid_build._contract_loader import CLIError +from fluid_build.cli import main +from fluid_build.cli._common import build_provider +from fluid_build.cli.plan import _gate_provider_flag + +_BUILTINS = {"aws", "databricks", "gcp", "local", "snowflake"} + +_REDSHIFT_HINT = ( + "Provider 'redshift' was retired: Redshift is part of the aws provider; " "use --provider aws" +) +_DMM_HINT = ( + "Provider '{name}' was retired: Data Mesh Manager is a catalog, not a provider; " + "use `fluid publish --target datamesh-manager` (or `fluid datamesh-manager`)" +) + + +def test_built_in_provider_set_is_exactly_five(): + P.discover_providers(force=True) + assert set(P.list_providers()) == _BUILTINS + + +def test_retired_packages_stay_importable_but_do_not_autoregister(): + import fluid_build.providers.datamesh_manager as dmm + from fluid_build.providers.aws import RedshiftProvider + + assert RedshiftProvider and dmm.DataMeshManagerProvider + assert getattr(dmm, "__fluid_no_autoregister__", False) is True + assert not hasattr(dmm, "NAME") and not hasattr(dmm, "Provider") + + +_RETIRED = [ + ("redshift", _REDSHIFT_HINT), + ("datamesh-manager", _DMM_HINT.format(name="datamesh-manager")), + ("datamesh_manager", _DMM_HINT.format(name="datamesh_manager")), +] + + +def _main_output(argv, caplog, capsys): + with caplog.at_level(logging.ERROR): + rc = main(argv) + out = capsys.readouterr() + return rc, caplog.text + out.err + out.out + + +@pytest.mark.parametrize("name, expected", _RETIRED) +def test_flag_exits_2_with_the_hint(name, expected, caplog, capsys, monkeypatch): + monkeypatch.delenv("FLUID_PROVIDER", raising=False) + rc, text = _main_output(["--provider", name, "providers"], caplog, capsys) + assert rc == 2 + assert expected in text + + +@pytest.mark.parametrize("name, expected", _RETIRED) +def test_env_var_behaves_like_the_flag(name, expected, caplog, capsys, monkeypatch): + monkeypatch.setenv("FLUID_PROVIDER", name) + rc, text = _main_output(["providers"], caplog, capsys) + assert rc == 2 + assert expected in text + + +@pytest.mark.parametrize("name, expected", _RETIRED) +def test_build_provider_and_plan_gate_raise_provider_retired(name, expected): + for call in ( + lambda: build_provider(name, None, None, logging.getLogger("t")), + lambda: _gate_provider_flag(name, logging.getLogger("t")), + ): + with pytest.raises(CLIError) as exc: + call() + assert exc.value.event == "provider_retired" + assert exc.value.exit_code == 2 + # Both normalise "-" to "_" before the lookup, so they echo that spelling. + assert exc.value.context["error"].split(": ", 1)[1] == expected.split(": ", 1)[1] + + +def test_an_unknown_name_is_still_just_unknown(capsys, monkeypatch, caplog): + monkeypatch.delenv("FLUID_PROVIDER", raising=False) + with caplog.at_level(logging.ERROR): + assert main(["--provider", "nosuchcloud", "providers"]) == 2 + out = capsys.readouterr() + assert "Unknown provider 'nosuchcloud'" in caplog.text + out.err + out.out + assert "was removed" not in caplog.text + out.err + out.out + + +def test_retired_hint_is_none_for_a_live_provider(): + assert P.retired_provider_hint("aws") is None + assert P.retired_provider_hint(None) is None + + +def test_redshift_rollback_dispatch_still_emits_ddl(): + from fluid_build.cli._rollback_writer import _restore_ddl_for_snapshot + + ddl = _restore_ddl_for_snapshot( + { + "location": { + "database": "db", + "schema": "sch", + "table": "t", + "backup_table": "t_bak", + } + }, + "redshift", + ) + assert ddl == [ + "BEGIN", + "DROP TABLE IF EXISTS db.sch.t", + "CREATE TABLE db.sch.t AS SELECT * FROM db.sch.t_bak", + "COMMIT", + ] + + +@pytest.mark.parametrize("alias", ["datamesh_manager", "datamesh-manager", "entropy-data", "dmm"]) +def test_datamesh_manager_still_resolves_as_a_publish_target(alias): + import fluid_build.build_runners.catalog_registrars # noqa: F401 (registers backends) + from fluid_build.api.catalog_backend import get_catalog_backend + + spec = get_catalog_backend(alias) + assert spec is not None and spec.name == "datamesh_manager" + + +class _FakeEntryPoint: + value = "evil_pkg.provider:Evil" + + def __init__(self, name): + self.name = name + self.loaded = False + + def load(self): + self.loaded = True + return type("Evil", (), {}) + + +class _FakeEntryPoints: + def __init__(self, *eps): + self._eps = eps + + def select(self, group): + return list(self._eps) if group == "fluid_build.providers" else [] + + def get(self, group, default=()): + return self.select(group) or list(default) + + +@pytest.fixture +def _restore_registry(): + before = dict(P.PROVIDERS) + done = P._DISCOVERY_DONE + yield + P.PROVIDERS.clear() + P.PROVIDERS.update(before) + P._DISCOVERY_DONE = done + + +@pytest.mark.parametrize("name", ["redshift", "datamesh-manager"]) +def test_a_plugin_cannot_claim_a_retired_name(name, monkeypatch, caplog, _restore_registry): + ep = _FakeEntryPoint(name) + monkeypatch.setattr(P.importlib.metadata, "entry_points", lambda: _FakeEntryPoints(ep)) + with caplog.at_level(logging.WARNING): + P._discover_entrypoints(logging.getLogger("t")) + assert not ep.loaded + assert P._normalize_name(name) not in P.PROVIDERS + records = [ + r for r in caplog.records if getattr(r, "evt", "") == "provider_retired_name_reserved" + ] + assert records and records[0].plugin == _FakeEntryPoint.value + # The CLI prints the message only, so the plugin must be named in it. + assert repr(_FakeEntryPoint.value) in records[0].getMessage() + + +def test_a_plugin_claiming_redshift_does_not_replace_the_retirement_hint( + monkeypatch, caplog, capsys, _restore_registry +): + monkeypatch.delenv("FLUID_PROVIDER", raising=False) + monkeypatch.setattr( + P.importlib.metadata, "entry_points", lambda: _FakeEntryPoints(_FakeEntryPoint("redshift")) + ) + P.PROVIDERS.clear() + P.discover_providers(force=True) + rc, text = _main_output(["--provider", "redshift", "providers"], caplog, capsys) + assert rc == 2 + assert _REDSHIFT_HINT in text + + +def test_an_explicit_registration_of_a_retired_name_stays_allowed(_restore_registry): + P.register_provider("redshift", object) + assert P.PROVIDERS["redshift"] is object + + +def test_the_hint_does_not_depend_on_the_registry_loading(monkeypatch, caplog, capsys): + # A registry that failed to load (``_known_provider_names`` -> []) must not hide it. + from fluid_build import cli + + monkeypatch.delenv("FLUID_PROVIDER", raising=False) + monkeypatch.setattr(cli, "_known_provider_names", lambda: []) + rc, text = _main_output(["--provider", "redshift", "providers"], caplog, capsys) + assert rc == 2 + assert _REDSHIFT_HINT in text + + +def test_a_blocking_allowlist_does_not_hide_the_hint(monkeypatch, caplog, capsys): + monkeypatch.delenv("FLUID_PROVIDER", raising=False) + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", "zzz") + rc, text = _main_output(["--provider", "redshift", "providers"], caplog, capsys) + assert rc == 2 + assert _REDSHIFT_HINT in text + + +def test_a_contract_naming_redshift_gets_the_aws_native_plan(monkeypatch): + # generate iac's native planner resolved a contract platform through build_provider + # and swallowed provider_retired, so a redshift contract lost its native actions. + from fluid_build.cli import generate_iac + + seen = {} + + def _build(name, project, region, logger): + seen["name"] = name + return type("P", (), {"plan": lambda self, c: []})() + + monkeypatch.setattr("fluid_build.cli._common.build_provider", _build) + contract = {"exposes": [{"binding": {"platform": "redshift", "location": {}}}]} + generate_iac.native_actions(contract, logging.getLogger("t")) + assert seen["name"] == "aws" + + +def test_policy_apply_bindings_resolve_a_retired_platform(): + from fluid_build.cli.policy_apply import _resolve_from_bindings + + assert _resolve_from_bindings({"bindings": [{"provider": "redshift"}]})[0] == "aws" + + +def test_a_non_string_platform_passes_through_unchanged(): + # LLM-drafted contracts can carry a non-string platform; the successor + # lookup must hand it on as before, not raise AttributeError. + from fluid_build.cli._common import resolve_provider_from_contract + from fluid_build.providers import _claims_retired_name, resolve_retired_platform + + assert resolve_retired_platform(123) == 123 + assert resolve_retired_platform(None) is None + assert _claims_retired_name(123, "plugin", None) is False + assert resolve_provider_from_contract({"exposes": [{"binding": {"platform": 123}}]})[0] == 123 diff --git a/tests/test_global_provider_flag.py b/tests/test_global_provider_flag.py index b555534e5..f006f75fd 100644 --- a/tests/test_global_provider_flag.py +++ b/tests/test_global_provider_flag.py @@ -15,7 +15,8 @@ """The global ``--provider`` flag must agree with the provider registry. `fluid providers` (help text: "List all discoverable infrastructure providers") -returned aws / datamesh_manager / gcp / local / redshift / snowflake, while the +returned aws / databricks / datamesh_manager / gcp / local / redshift / snowflake (then; +``redshift`` and ``datamesh_manager`` have since been retired as providers), while the global flag carried a hardcoded ``choices=["local","gcp","snowflake","aws", "azure"]``. The two disagreed in BOTH directions: @@ -57,12 +58,9 @@ def test_every_registered_provider_is_accepted(self, parser): assert args.provider == name def test_a_registry_provider_the_old_enum_rejected_now_parses(self, parser): - # 'redshift' and 'datamesh_manager' ship in the registry and were - # rejected by the enum. - known = _known_provider_names() - for name in ("redshift", "datamesh_manager"): - if name in known: - assert parser.parse_args(["--provider", name, "providers"]).provider == name + # 'databricks' ships in the registry and was rejected by the enum. + assert "databricks" in _known_provider_names() + assert parser.parse_args(["--provider", "databricks", "providers"]).provider == "databricks" def test_an_unregistered_provider_is_reported_as_unknown(self): from fluid_build.cli import _validate_global_args diff --git a/tests/test_plugin_policy_governs_builtins.py b/tests/test_plugin_policy_governs_builtins.py index 9c979ed98..f1b628b6f 100644 --- a/tests/test_plugin_policy_governs_builtins.py +++ b/tests/test_plugin_policy_governs_builtins.py @@ -128,17 +128,17 @@ def select(self, group): assert not registry.POLICY_BLOCKED def test_a_provider_registered_at_import_stays_blocked_through_rediscovery(self, monkeypatch): - """redshift registers when the aws package is imported, and a cached - module is not imported again: forgetting every record at the start of a - rediscovery reported a blocked redshift as no provider at all.""" - monkeypatch.setenv("FLUID_PLUGINS_BLOCKLIST", "redshift") - registry.register_provider("redshift", object, logger=LOG) # as at import - assert "redshift" in registry.POLICY_BLOCKED + """aws registers when its package is imported, and a cached module is + not imported again: forgetting every record at the start of a + rediscovery reported a blocked aws as no provider at all.""" + monkeypatch.setenv("FLUID_PLUGINS_BLOCKLIST", "aws") + registry.register_provider("aws", object, logger=LOG) # as at import + assert "aws" in registry.POLICY_BLOCKED _rediscover() - assert "redshift" in registry.POLICY_BLOCKED + assert "aws" in registry.POLICY_BLOCKED monkeypatch.delenv("FLUID_PLUGINS_BLOCKLIST") _rediscover() - assert "redshift" not in registry.POLICY_BLOCKED # allowed again, forgotten + assert "aws" not in registry.POLICY_BLOCKED # allowed again, forgotten def test_allowlist_excludes_unlisted_builtins(self, monkeypatch): """An allowlist means "only these load" — the built-ins were silently From 079200f3e91be6b0311c3e5fcf31f381d4c0b5b2 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:35:58 +0100 Subject: [PATCH 13/26] fix(iac): lock OpenTofu state on S3, and release the lock on timeout (#787) * fix(iac): lock OpenTofu state on S3 with use_lockfile The S3 backend was emitted as bucket and key only, so a scheduled apply and a manual one (or an apply and a teardown) wrote the same state with no lock. tofu init now gets -backend-config=use_lockfile=true for an S3 backend when the installed OpenTofu is 1.10 or newer; the emitted module is unchanged, so golden files stay valid. An older or unreadable OpenTofu skips the flag and logs s3_state_unlocked with the fix, and FLUID_TOFU_S3_LOCKFILE=0 opts out (listed in fluid doctor). A workdir kept from before this change recorded the backend without the lockfile, and tofu refuses the changed config ("Backend configuration changed"). init adds -reconfigure only when the recorded backend is this same S3 bucket and key (and region, when the module names one) without use_lockfile; any other difference is left to tofu's own error rather than silently repointed. Pinned by unit tests and an old-then-new apply on one kept workdir against moto. Upgrade note: tofu plan takes the lock too, so every role that runs plan, apply --dry-run, fluid diff or verify --state-drift against S3 state needs s3:GetObject, s3:PutObject and s3:DeleteObject on .tflock, or sets FLUID_TOFU_S3_LOCKFILE=0. Found by an external hands-on assessment of 0.26.0; Trello #738 (the locking half). * fix(iac): the S3 lock opt-out works on a kept workdir; a timed-out tofu is interrupted first tofu reuses the partial backend config a workdir saved whenever init gets no -backend-config override, so once a workdir had initialised with use_lockfile=true, FLUID_TOFU_S3_LOCKFILE=0 (or a tofu too old or unreadable for the flag) left locking on: the operator who needed the escape hatch stayed locked out. init now adds -reconfigure in that case, only when the saved backend is this same bucket and key, so no state moves. A run that hit FLUID_TOFU_TIMEOUT_SECONDS was SIGKILLed, so tofu never released its lock and every later run failed until someone ran tofu force-unlock. _run now sends SIGINT, which makes tofu stop and release the lock, and kills only after a 30 s grace period; the timeout result is unchanged. The lockfile-driven -reconfigure is never added with -force-copy, where tofu would let it win and silently skip the state copy. Found by the security review of this branch. * test(iac): fake Popen, not subprocess.run, where tests stand in for tofu runner._run now starts tofu with subprocess.Popen so a timeout can interrupt it before killing it. Two tests in test_iac_state_drift.py patched subprocess.run, so their fake was bypassed and a real Popen of /bin/tofu failed on CI. They now patch Popen with a stand-in child. * fix(credentials): a project .env cannot turn S3 state locking off FLUID_TOFU_S3_LOCKFILE=0 disables a safety control, so it is classified as protected: the operator's environment may set it, a checked-in .env may not. test_every_fluid_variable_in_the_package_is_classified failed on CI because the new variable was unclassified. --- CHANGELOG.md | 13 + fluid_build/cli/doctor.py | 6 + fluid_build/credentials/protected_env.py | 2 + fluid_build/iac/runner.py | 145 +++++++++-- tests/iac/test_iac_runner.py | 230 +++++++++++++++++- tests/iac/test_iac_state_drift.py | 23 +- .../iac/test_iac_state_key_migration_moto.py | 34 +++ 7 files changed, 417 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 245f37248..8a708826c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -167,6 +167,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `pip install fluid-build[...]`; they now name `data-product-forge` with the right extra, and a ratchet test keeps the wrong name out. `data-product-forge[mcp]` resolves as an empty alias. `fluid-build` on PyPI is now only an alias that installs `data-product-forge`. +- **Two applies on the same S3 state could race.** The S3 backend was emitted without locking, so a + scheduled apply and a manual one (or an apply and a teardown) wrote the same state unguarded. + `tofu init` now gets `-backend-config=use_lockfile=true` for an S3 backend on OpenTofu 1.10 or + newer; the emitted module is unchanged. On an older OpenTofu a WARNING says the state is + unlocked. Upgrade note: `tofu plan` takes the lock too, so every role that runs plan, + `fluid apply --dry-run`, `fluid diff` or `fluid verify --state-drift` against S3 state needs + `s3:GetObject`, `s3:PutObject` and `s3:DeleteObject` on `.tflock`, or sets + `FLUID_TOFU_S3_LOCKFILE=0`. A workdir kept from before the upgrade is re-initialised with + `-reconfigure` once, only when it recorded this same bucket, key and region. The opt-out also takes effect on a + workdir that already had locking on. A run killed hard can leave `.tflock` behind; + `tofu force-unlock ` in the workdir (the ID is in tofu's error) clears it. A run that + hits `FLUID_TOFU_TIMEOUT_SECONDS` is now interrupted (SIGINT) so tofu releases the lock, and + killed only if it ignores that for 30 seconds. - **A contract's governance labels were dropped silently on Databricks.** The Databricks emitter never read `labels`, so `data-classification: confidential`, the `gdpr-article`, and the sovereignty jurisdiction, regulation and residency reached AWS (tags, Glue parameters) and GCP diff --git a/fluid_build/cli/doctor.py b/fluid_build/cli/doctor.py index 3417119ba..f7eb225bb 100644 --- a/fluid_build/cli/doctor.py +++ b/fluid_build/cli/doctor.py @@ -173,6 +173,12 @@ "1800s default", "Per-`tofu` invocation wall-clock cap", ), + ( + "FLUID_TOFU_S3_LOCKFILE", + "on (OpenTofu 1.10+)", + "Set =0 to skip S3-native state locking when the deploying role cannot " + "write the `.tflock` object", + ), ( "FLUID_FEDERATION_TIMEOUT_SECONDS", "30s default", diff --git a/fluid_build/credentials/protected_env.py b/fluid_build/credentials/protected_env.py index 15af94a06..9c425457b 100644 --- a/fluid_build/credentials/protected_env.py +++ b/fluid_build/credentials/protected_env.py @@ -197,6 +197,8 @@ "FLUID_DIAG_OUT_DIR", "FLUID_PRICES_JSON", "FLUID_STATE_BACKEND", + # =0 turns S3 state locking off: the operator's call, never a checkout's + "FLUID_TOFU_S3_LOCKFILE", "FLUID_STORE_BACKEND", "FLUID_STORE_DSN", "FLUID_STORE_PATH", diff --git a/fluid_build/iac/runner.py b/fluid_build/iac/runner.py index c59140aa8..879dee5da 100644 --- a/fluid_build/iac/runner.py +++ b/fluid_build/iac/runner.py @@ -26,11 +26,14 @@ from __future__ import annotations import json +import logging import os import re import shutil +import signal import subprocess from dataclasses import dataclass, field +from pathlib import Path from typing import Any, Dict, List, Mapping, Optional, Tuple # Per-command wall-clock cap. A hung ``tofu`` (e.g., an unauthenticated @@ -43,6 +46,13 @@ # can take many minutes for the eventual consistency to settle). _DEFAULT_TOFU_TIMEOUT_SECONDS = 1800 _MIN_REQUIRED_VERSION = (1, 6, 0) +# S3-native state locking (``use_lockfile``) arrived in OpenTofu 1.10. +_S3_LOCKFILE_MIN_VERSION = (1, 10, 0) +# After SIGINT on a timeout, how long tofu gets to release its lock and exit +# before it is killed. +_TIMEOUT_GRACE_SECONDS = 30 + +logger = logging.getLogger(__name__) def _resolve_timeout() -> int: @@ -162,33 +172,41 @@ def _run( ) -> TofuResult: tofu = _require_tofu() timeout = _resolve_timeout() + child = subprocess.Popen( + [tofu, *args], + cwd=workdir, + env=dict(env) if env is not None else None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) try: - proc = subprocess.run( - [tofu, *args], - cwd=workdir, - env=dict(env) if env is not None else None, - capture_output=True, - text=True, - timeout=timeout, - ) - except subprocess.TimeoutExpired as exc: + stdout, stderr = child.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + # SIGINT first: tofu's interrupt stops the operation and releases the + # state lock. A SIGKILL leaves ``.tflock`` behind and every later + # run fails until someone runs ``tofu force-unlock``. Only a tofu that + # ignores the interrupt for the grace period is killed. + try: + child.send_signal(signal.SIGINT) + stdout, _ = child.communicate(timeout=_TIMEOUT_GRACE_SECONDS) + except (subprocess.TimeoutExpired, OSError, ValueError): + child.kill() + stdout, _ = child.communicate() # Surface the partial output if any — the timeout itself is the # signal (apply engine treats a TofuResult with returncode != 0 # as a failure). Use returncode=124 to mirror coreutils ``timeout``. return TofuResult( command=command, returncode=124, - stdout=( - (exc.stdout or b"").decode("utf-8", errors="replace") - if isinstance(exc.stdout, (bytes, bytearray)) - else (exc.stdout or "") - ), + stdout=stdout or "", stderr=( f"`tofu {command}` exceeded the {timeout}s wall-clock " "limit (FLUID_TOFU_TIMEOUT_SECONDS overrides the default)." ), events=[], ) + proc = subprocess.CompletedProcess(child.args, child.returncode, stdout, stderr) events = _parse_json_events(proc.stdout) if "-json" in args else [] return TofuResult( command=command, @@ -199,6 +217,101 @@ def _run( ) +def _needs_reconfigure_for_lockfile(workdir: str, config: Mapping[str, Any]) -> bool: + """True when ``workdir`` was initialised on this same S3 state, without the lockfile. + + Adding ``use_lockfile`` to a kept workdir changes its backend config, which + ``tofu init`` refuses without ``-reconfigure``. That is safe only when the + recorded backend is this one: the same bucket, key and region (compared when + the module names one), ``use_lockfile`` not already true. A recorded backend + that differs in any of them is left to tofu's own error, never repointed. + """ + from .state_migration import recorded_backend + + recorded = recorded_backend(Path(workdir)) + if not recorded or set(recorded) != {"s3"}: + return False + have = recorded["s3"] or {} + if have.get("use_lockfile") is True: + return False + fields = ("bucket", "key") + (("region",) if config.get("region") else ()) + return all(have.get(k) == config.get(k) for k in fields) + + +def _drops_saved_lockfile(workdir: str, config: Mapping[str, Any]) -> bool: + """True when ``workdir`` saved ``use_lockfile=true`` for this same bucket and key. + + With no ``-backend-config`` override tofu reuses that saved partial config, so + an opt-out (or a tofu we will not pass the flag to) needs ``-reconfigure`` to + drop it. Same bucket and key, so no state moves. + """ + from .state_migration import recorded_backend + + recorded = recorded_backend(Path(workdir)) + if not recorded or set(recorded) != {"s3"}: + return False + have = recorded["s3"] or {} + return have.get("use_lockfile") is True and all( + have.get(k) == config.get(k) for k in ("bucket", "key") + ) + + +def _s3_lockfile_args(workdir: str, *, force_copy: bool = False) -> List[str]: + """``-backend-config=use_lockfile=true`` when the module's backend is S3. + + The flag is passed at init, not emitted into the module, so the module's + bytes (an adopter may keep them as golden files) do not change. Gated on + the installed tofu (>= 1.10) and switched off by ``FLUID_TOFU_S3_LOCKFILE=0`` + for a role that cannot write the ``.tflock`` object. ``-reconfigure`` + rides along when a kept workdir recorded this state without the lockfile, and + when it saved the lockfile but the flag is no longer passed (so the opt-out + takes effect). Never with ``force_copy``: tofu lets ``-reconfigure`` win over + ``-force-copy`` and would silently skip the state copy. + """ + try: + with open(os.path.join(workdir, "main.tf.json"), encoding="utf-8") as fh: + backend = json.load(fh).get("terraform", {}).get("backend", {}) + except (OSError, ValueError, AttributeError): + return [] + if not isinstance(backend, dict) or not isinstance(backend.get("s3"), dict): + return [] + if os.environ.get("FLUID_TOFU_S3_LOCKFILE", "").strip().lower() in ("0", "false", "no", "off"): + return _unlock_args(workdir, backend["s3"], force_copy) + version = tofu_version() + if version is None or version < _S3_LOCKFILE_MIN_VERSION: + if version is None: + reason = ( + "the OpenTofu version could not be read, so the lock flag was skipped " + "rather than risk an unrecognised build failing init" + ) + else: + reason = ( + f"OpenTofu {'.'.join(str(p) for p in version)} predates 1.10, so two applies " + "on the same product can race. Upgrade OpenTofu to 1.10 or newer, " + "or run with --ensure-opentofu (pins a version that locks)" + ) + logger.warning( + json.dumps( + { + "event": "s3_state_unlocked", + "tofu_version": ".".join(str(p) for p in version) if version else None, + "message": f"S3 state is not locked: {reason}.", + } + ) + ) + return _unlock_args(workdir, backend["s3"], force_copy) + args = ["-backend-config=use_lockfile=true"] + if not force_copy and _needs_reconfigure_for_lockfile(workdir, backend["s3"]): + args.append("-reconfigure") + return args + + +def _unlock_args(workdir: str, config: Mapping[str, Any], force_copy: bool) -> List[str]: + if not force_copy and _drops_saved_lockfile(workdir, config): + return ["-reconfigure"] + return [] + + def tofu_init( workdir: str, *, @@ -223,7 +336,9 @@ def tofu_init( args = ["init", "-input=false", "-no-color"] if not backend: args.append("-backend=false") - if reconfigure: + else: + args.extend(a for a in _s3_lockfile_args(workdir, force_copy=force_copy) if a not in args) + if reconfigure and "-reconfigure" not in args: args.append("-reconfigure") if force_copy: args.append("-force-copy") diff --git a/tests/iac/test_iac_runner.py b/tests/iac/test_iac_runner.py index 6d952853c..0e3b45ba0 100644 --- a/tests/iac/test_iac_runner.py +++ b/tests/iac/test_iac_runner.py @@ -16,6 +16,9 @@ from __future__ import annotations +import signal +import subprocess + import pytest from fluid_build.iac import build_module, get_iac_plugin, runner @@ -131,24 +134,74 @@ def test_resolve_timeout_ignores_non_positive(self, monkeypatch): monkeypatch.setenv("FLUID_TOFU_TIMEOUT_SECONDS", "-100") assert runner._resolve_timeout() == runner._DEFAULT_TOFU_TIMEOUT_SECONDS - def test_run_returns_124_on_timeout(self, monkeypatch, tmp_path): - """When ``subprocess.run`` raises ``TimeoutExpired``, ``_run`` - synthesises a ``TofuResult`` with returncode 124 (mirrors - coreutils ``timeout``) and a clear stderr — never re-raises.""" - import subprocess as _subprocess - - monkeypatch.setattr(runner, "_require_tofu", lambda: "/usr/local/bin/tofu") - - def _raise_timeout(*_args, **kwargs): - raise _subprocess.TimeoutExpired(cmd="tofu apply", timeout=kwargs["timeout"]) - - monkeypatch.setattr(runner.subprocess, "run", _raise_timeout) - result = runner._run(["apply"], workdir=str(tmp_path), env=None, command="apply") + def _fake_tofu(self, monkeypatch, tmp_path, body): + import os + import stat + import sys + + script = tmp_path / "fake-tofu" + script.write_text(f"#!{sys.executable}\n{body}") + script.chmod(script.stat().st_mode | stat.S_IXUSR) + monkeypatch.setattr(runner, "_require_tofu", lambda: str(script)) + monkeypatch.setenv("FLUID_TOFU_TIMEOUT_SECONDS", "5") + return os.fspath(tmp_path) + + class _Child: + """A ``Popen`` stand-in: times out once, then answers SIGINT (or not).""" + + def __init__(self, answers_interrupt): + self.args = ["tofu", "apply"] + self.returncode = None + self.signals = [] + self.killed = False + self._answers = answers_interrupt + self._calls = 0 + + def communicate(self, timeout=None): + self._calls += 1 + if self._calls == 1 or (self._calls == 2 and not self._answers): + raise subprocess.TimeoutExpired(self.args, timeout) + self.returncode = 130 if self._answers else -9 + return ("released lock\n" if self._answers else "", "") + + def send_signal(self, sig): + self.signals.append(sig) + + def kill(self): + self.killed = True + + def _run_with(self, monkeypatch, child): + monkeypatch.setattr(runner, "_require_tofu", lambda: "tofu") + monkeypatch.setattr(runner.subprocess, "Popen", lambda *a, **k: child) + return runner._run(["apply"], workdir=".", env=None, command="apply") + + def test_run_returns_124_on_timeout_after_a_graceful_interrupt(self, monkeypatch): + """A timeout sends SIGINT, so tofu releases its lock and exits; the result + keeps the 124 shape, keeps tofu's output and never re-raises.""" + child = self._Child(answers_interrupt=True) + result = self._run_with(monkeypatch, child) + assert child.signals == [signal.SIGINT] + assert child.killed is False assert result.ok is False assert result.returncode == 124 + assert "released lock" in result.stdout assert "exceeded" in result.stderr assert "FLUID_TOFU_TIMEOUT_SECONDS" in result.stderr + def test_run_kills_only_after_the_interrupt_is_ignored(self, monkeypatch): + child = self._Child(answers_interrupt=False) + result = self._run_with(monkeypatch, child) + assert child.signals == [signal.SIGINT] + assert child.killed is True + assert result.returncode == 124 + + def test_run_kills_a_tofu_that_ignores_the_interrupt(self, monkeypatch, tmp_path): + body = "import signal, time\nsignal.signal(signal.SIGINT, signal.SIG_IGN)\ntime.sleep(60)\n" + wd = self._fake_tofu(monkeypatch, tmp_path, body) + monkeypatch.setattr(runner, "_TIMEOUT_GRACE_SECONDS", 1) + result = runner._run(["apply"], workdir=wd, env=None, command="apply") + assert result.returncode == 124 + @pytest.mark.integration @pytest.mark.skipif(runner.tofu_path() is None, reason="tofu binary not installed") @@ -193,3 +246,154 @@ def test_state_list_empty_when_no_state(self, monkeypatch): runner, "_run", lambda *a, **k: runner.TofuResult("state-list", 1, "", "no state") ) assert runner.tofu_state_list("/wd") == [] + + +class TestS3Lockfile: + """S3 state locks via ``-backend-config=use_lockfile=true`` at init (tofu >= 1.10).""" + + FLAG = "-backend-config=use_lockfile=true" + + def _init_args(self, monkeypatch, tmp_path, backend, version): + import json as _json + + doc = {"terraform": {"backend": backend} if backend else {}} + (tmp_path / "main.tf.json").write_text(_json.dumps(doc)) + seen = {} + monkeypatch.setattr(runner, "tofu_version", lambda: version) + monkeypatch.setattr(runner, "_run", lambda args, **kw: seen.setdefault("args", args)) + runner.tofu_init(str(tmp_path)) + return seen["args"] + + S3 = {"s3": {"bucket": "b", "key": "k"}} + + def test_added_on_new_tofu(self, monkeypatch, tmp_path): + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE", raising=False) + assert self.FLAG in self._init_args(monkeypatch, tmp_path, self.S3, (1, 10, 0)) + + def test_absent_and_warns_on_old_tofu(self, monkeypatch, tmp_path, caplog): + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE", raising=False) + with caplog.at_level("WARNING"): + args = self._init_args(monkeypatch, tmp_path, self.S3, (1, 9, 9)) + assert self.FLAG not in args + assert "s3_state_unlocked" in caplog.text and "1.10" in caplog.text + + def test_opt_out(self, monkeypatch, tmp_path): + monkeypatch.setenv("FLUID_TOFU_S3_LOCKFILE", "0") + assert self.FLAG not in self._init_args(monkeypatch, tmp_path, self.S3, (1, 12, 1)) + + def test_gcs_and_local_untouched(self, monkeypatch, tmp_path): + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE", raising=False) + assert self.FLAG not in self._init_args( + monkeypatch, tmp_path, {"gcs": {"bucket": "b"}}, (1, 12, 1) + ) + assert self.FLAG not in self._init_args(monkeypatch, tmp_path, None, (1, 12, 1)) + + def test_emitted_module_has_no_lockfile(self): + from fluid_build.iac.backend import parse_backend + + assert parse_backend("s3://b/k") == {"s3": {"bucket": "b", "key": "k"}} + + def test_unreadable_version_skips_flag_and_warns(self, monkeypatch, tmp_path, caplog): + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE", raising=False) + with caplog.at_level("WARNING"): + args = self._init_args(monkeypatch, tmp_path, self.S3, None) + assert self.FLAG not in args + assert "s3_state_unlocked" in caplog.text and "could not be read" in caplog.text + + +class TestLockfileReconfigure: + """``-reconfigure`` joins the lockfile flag only for a kept workdir on this same state.""" + + def _args(self, monkeypatch, tmp_path, recorded, module=None): + import json as _json + + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE", raising=False) + s3 = module or {"bucket": "b", "key": "k", "region": "eu-west-1"} + (tmp_path / "main.tf.json").write_text(_json.dumps({"terraform": {"backend": {"s3": s3}}})) + if recorded is not None: + (tmp_path / ".terraform").mkdir() + (tmp_path / ".terraform" / "terraform.tfstate").write_text( + _json.dumps({"backend": {"type": "s3", "config": recorded}}) + ) + seen = {} + monkeypatch.setattr(runner, "tofu_version", lambda: (1, 12, 1)) + monkeypatch.setattr(runner, "_run", lambda args, **kw: seen.setdefault("args", args)) + runner.tofu_init(str(tmp_path)) + return seen["args"] + + def test_same_backend_without_lockfile_reconfigures(self, monkeypatch, tmp_path): + rec = {"bucket": "b", "key": "k", "region": "eu-west-1", "use_lockfile": None} + args = self._args(monkeypatch, tmp_path, rec) + assert "-reconfigure" in args and "-backend-config=use_lockfile=true" in args + + def test_different_key_does_not(self, monkeypatch, tmp_path): + rec = {"bucket": "b", "key": "other", "region": "eu-west-1", "use_lockfile": None} + assert "-reconfigure" not in self._args(monkeypatch, tmp_path, rec) + + def test_different_region_does_not(self, monkeypatch, tmp_path): + rec = {"bucket": "b", "key": "k", "region": "us-east-1", "use_lockfile": None} + assert "-reconfigure" not in self._args(monkeypatch, tmp_path, rec) + + def test_already_locking_does_not(self, monkeypatch, tmp_path): + rec = {"bucket": "b", "key": "k", "region": "eu-west-1", "use_lockfile": True} + assert "-reconfigure" not in self._args(monkeypatch, tmp_path, rec) + + def test_nothing_recorded_does_not(self, monkeypatch, tmp_path): + assert "-reconfigure" not in self._args(monkeypatch, tmp_path, None) + + +class TestLockfileOptOutOnKeptWorkdir: + """A workdir that saved use_lockfile=true needs -reconfigure for the flag to go away.""" + + SAVED = {"bucket": "b", "key": "k", "region": "eu-west-1", "use_lockfile": True} + + def _args( + self, + monkeypatch, + tmp_path, + *, + version=(1, 12, 1), + env=None, + force_copy=False, + recorded=None, + ): + import json as _json + + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE", raising=False) + if env is not None: + monkeypatch.setenv("FLUID_TOFU_S3_LOCKFILE", env) + s3 = {"bucket": "b", "key": "k"} + (tmp_path / "main.tf.json").write_text(_json.dumps({"terraform": {"backend": {"s3": s3}}})) + (tmp_path / ".terraform").mkdir(exist_ok=True) + (tmp_path / ".terraform" / "terraform.tfstate").write_text( + _json.dumps({"backend": {"type": "s3", "config": recorded or self.SAVED}}) + ) + seen = {} + monkeypatch.setattr(runner, "tofu_version", lambda: version) + monkeypatch.setattr(runner, "_run", lambda args, **kw: seen.setdefault("args", args)) + runner.tofu_init(str(tmp_path), force_copy=force_copy) + return seen["args"] + + def test_opt_out_reconfigures(self, monkeypatch, tmp_path): + args = self._args(monkeypatch, tmp_path, env="0") + assert "-reconfigure" in args and "-backend-config=use_lockfile=true" not in args + + def test_old_tofu_reconfigures(self, monkeypatch, tmp_path): + assert "-reconfigure" in self._args(monkeypatch, tmp_path, version=(1, 9, 0)) + + def test_unreadable_version_reconfigures(self, monkeypatch, tmp_path): + assert "-reconfigure" in self._args(monkeypatch, tmp_path, version=None) + + def test_different_key_is_left_to_tofu(self, monkeypatch, tmp_path): + rec = dict(self.SAVED, key="other") + assert "-reconfigure" not in self._args(monkeypatch, tmp_path, env="0", recorded=rec) + + def test_lockfile_not_saved_adds_nothing(self, monkeypatch, tmp_path): + rec = dict(self.SAVED, use_lockfile=None) + assert "-reconfigure" not in self._args(monkeypatch, tmp_path, env="0", recorded=rec) + + def test_force_copy_never_gets_the_lockfile_reconfigure(self, monkeypatch, tmp_path): + assert "-reconfigure" not in self._args(monkeypatch, tmp_path, env="0", force_copy=True) + rec = dict(self.SAVED, use_lockfile=None) + args = self._args(monkeypatch, tmp_path, force_copy=True, recorded=rec) + assert "-reconfigure" not in args and "-force-copy" in args diff --git a/tests/iac/test_iac_state_drift.py b/tests/iac/test_iac_state_drift.py index e8c1f686f..04203c46e 100644 --- a/tests/iac/test_iac_state_drift.py +++ b/tests/iac/test_iac_state_drift.py @@ -26,7 +26,6 @@ from __future__ import annotations import json -import subprocess from typing import Any, Dict, List, Optional import pytest @@ -340,18 +339,26 @@ def test_format_path(): # --------------------------------------------------------------------------- -def _fake_run(calls: List[List[str]], *, returncode: int = 0, stdout: str = ""): - def _run(argv, **kwargs): - calls.append(list(argv)) - return subprocess.CompletedProcess(argv, returncode, stdout=stdout, stderr="") +def _fake_popen(calls: List[List[str]], *, returncode: int = 0, stdout: str = ""): + """A ``subprocess.Popen`` stand-in: ``runner._run`` starts tofu with Popen so a + timeout can interrupt it before killing it.""" - return _run + class _Child: + def __init__(self, argv, **kwargs): + calls.append(list(argv)) + self.args = argv + self.returncode = returncode + + def communicate(self, timeout=None): + return stdout, "" + + return _Child def test_tofu_plan_detailed_asks_for_the_detailed_exit_code(monkeypatch, tmp_path): calls: List[List[str]] = [] monkeypatch.setattr(runner, "tofu_path", lambda: "/bin/tofu") - monkeypatch.setattr(runner.subprocess, "run", _fake_run(calls, returncode=2)) + monkeypatch.setattr(runner.subprocess, "Popen", _fake_popen(calls, returncode=2)) result = runner.tofu_plan_detailed(str(tmp_path), out_file="fluid-drift.tfplan") assert calls[0][1:] == [ "plan", @@ -377,7 +384,7 @@ def test_tofu_show_plan(monkeypatch, tmp_path, returncode, stdout, expected): calls: List[List[str]] = [] monkeypatch.setattr(runner, "tofu_path", lambda: "/bin/tofu") monkeypatch.setattr( - runner.subprocess, "run", _fake_run(calls, returncode=returncode, stdout=stdout) + runner.subprocess, "Popen", _fake_popen(calls, returncode=returncode, stdout=stdout) ) assert runner.tofu_show_plan(str(tmp_path), plan_file="p") == expected assert calls[0][1:] == ["show", "-json", "p"] diff --git a/tests/iac/test_iac_state_key_migration_moto.py b/tests/iac/test_iac_state_key_migration_moto.py index f342b8b4e..e0d14f6c1 100644 --- a/tests/iac/test_iac_state_key_migration_moto.py +++ b/tests/iac/test_iac_state_key_migration_moto.py @@ -418,3 +418,37 @@ def test_a_read_only_caller_reads_the_old_key_until_the_apply_moves_it(moto, tmp assert outcome.outcome == mig.PENDING assert outcome.read_from == _block(_LEGACY_KEY) assert _AWS_KEY not in _keys(moto) + + +def test_a_workdir_kept_from_before_the_lockfile_upgrades_in_place(moto, tmp_path, monkeypatch): + """Old release's init (no use_lockfile) then the new one on the same kept workdir.""" + contract = _write(tmp_path, _contract()) + monkeypatch.setenv("FLUID_TOFU_S3_LOCKFILE", "0") + _apply_as_the_old_release(contract, tmp_path) + before = _object(moto, _LEGACY_KEY) + assert before["resources"] + + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE") + _apply_as_the_old_release(contract, tmp_path) # must not raise opentofu_init_failed + + after = _object(moto, _LEGACY_KEY) + assert _addresses(after) == _addresses(before) + (workdir,) = [p.parent.parent for p in tmp_path.rglob(".terraform/terraform.tfstate")] + assert mig.recorded_backend(workdir)["s3"]["use_lockfile"] is True + + +def test_the_opt_out_takes_effect_on_a_workdir_that_had_locking_on(moto, tmp_path, monkeypatch): + """Apply with locking, then with FLUID_TOFU_S3_LOCKFILE=0 on the same kept workdir.""" + contract = _write(tmp_path, _contract()) + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE", raising=False) + _apply_as_the_old_release(contract, tmp_path) + before = _object(moto, _LEGACY_KEY) + assert before["resources"] + (workdir,) = [p.parent.parent for p in tmp_path.rglob(".terraform/terraform.tfstate")] + assert mig.recorded_backend(workdir)["s3"]["use_lockfile"] is True + + monkeypatch.setenv("FLUID_TOFU_S3_LOCKFILE", "0") + _apply_as_the_old_release(contract, tmp_path) + + assert mig.recorded_backend(workdir)["s3"]["use_lockfile"] in (None, False) + assert _addresses(_object(moto, _LEGACY_KEY)) == _addresses(before) From 9260686d02becc94bd8749919273ffa282ba5596 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:44:14 +0100 Subject: [PATCH 14/26] security: refuse a trailing newline in whole-string validators (#797) * security: refuse a trailing newline in whole-string validators Python's `$` also matches just before a final newline, so `re.match(r"^[a-z]+$", "orders\n")` succeeds and the caller goes on to use "orders\n". #788 closed this in providers/_sql_safety.py. The same `^...$` plus `.match` shape guarded 36 more validator patterns across fluid_build/, fed by contract YAML (a `|` block scalar ends in a newline), .env values, CLI flags, LLM tool calls and percent-encoded URL queries. 31 of those patterns now end in `\Z`. Two call sites use `fullmatch` instead: the catalog quoting chokepoint, which also covers the BigQuery patterns callers pass in, and the federation workspace id, whose error message prints the pattern. Seen through the CLI before the fix, `fluid generate iac` put `region = "eu-west-1\n"` on the AWS provider, and `branch = "main\n"` on a Databricks job from `?branch=main%0A`. 81 other `$`-ending patterns stay as they are. Their input is stripped first, or they are deny lists and detectors where `\Z` would loosen the check, or they are parsers whose captured groups are what is used. tests/security/test_trailing_newline_anchors.py pins each fixed site with a value and the same value plus "\n". All 39 cases fail on the parent commit. The fix follows Django's username validators (ticket #30257, `$` to `\Z`) and the MCP Python SDK's tool-name check (#3076, `fullmatch`). * docs(changelog): 36 more validators refuse a value with a trailing newline * fix(iac): refuse an AWS binding region with whitespace around it With `\Z`, provider_block_for dropped a binding region of "eu-west-1\n" as no region code. That left the provider on the environment's region and skipped _guard_region_move. Pinning the stripped value is no better. Both sovereignty checkers (policy/sovereignty.py, providers/aws/util/sovereignty.py) compare the region as written, so "us-east-1\n" passes `deniedRegions: [us-east-1]` while the provider would be pinned to us-east-1. provider_block_for now raises UnsupportedBindingError (aws-region-whitespace) for a region code with whitespace around it. generate iac and apply render it as a typed CLI error before anything is applied, and fluid diff reports the inspector error. On main the value went onto the provider with its newline. * security: match the BigQuery catalog and imported-schema patterns whole copilot/catalog/bigquery.py::_validate_ident checked the same `$` patterns with `.match` that the quoting chokepoint now checks with `fullmatch`. So `scope.tables` values and FQN parts such as "orders\n" reached `table_name IN ('...')`. The ODCS and ODPS importers compile the bundled schema's identifier and semver patterns, which end in `$`, and also used `.match`: an imported id "orders\n" was kept verbatim, and "1.2.3\n" counted as semver. All three now call `fullmatch`. The two patterns that keep `$` because their call sites use `fullmatch` (`_DEFAULT_IDENTIFIER`, `_WORKSPACE_ID_RE`) now say so in a comment beside them. * fix: show the refused newline in AWS name and mission path messages These messages quoted the value as '{value}', so a name refused for a trailing newline printed as a valid-looking name split over two lines. They now use repr, which prints any other value exactly as before. * chore: keep the domain slug patterns in step with the profile-name guard The comment on forge_domain_enrichment._SAFE_DOMAIN_RE says it matches forge_copilot_prompts._PROFILE_NAME_RE, and the comment on _ai_setup_storage._DOMAIN_SLUG_RE says it mirrors _SAFE_DOMAIN_RE. The profile-name guard now ends in `\Z`, so both do too. Neither is reachable with a trailing newline: one runs after a disk lookup, the other on stripped input. * docs: drop the validator count and reword validate_id's summary The count in the changelog entry depended on how patterns were grouped. The entry now lists what is covered instead, including the AWS region refusal and the imports. validate_id's summary said it refused a trailing newline "as well", which the shared grammar now does itself. * test: skip the BigQuery validation pin without the GCP SDK bigquery_validation imports google.cloud.bigquery at module load. The iac Stage 1 unit lane installs no GCP extra, so the new pin failed collection there with ModuleNotFoundError. It now skips via pytest.importorskip, the guard tests/providers/test_validation_sql_injection.py already applies to the same module. --- CHANGELOG.md | 14 + fluid_build/build_runners/_ids.py | 2 +- fluid_build/build_runners/meltano/runner.py | 4 +- fluid_build/cli/_ai_setup_storage.py | 2 +- fluid_build/cli/_forge_data_model_jdbc.py | 2 +- fluid_build/cli/discover/_jdbc_introspect.py | 2 +- fluid_build/cli/forge_copilot_prompts.py | 2 +- fluid_build/cli/forge_domain_enrichment.py | 2 +- fluid_build/cli/forge_prompt_overlays.py | 2 +- fluid_build/cli/forge_validation.py | 4 +- fluid_build/cli/import_workflow/dbt.py | 2 +- fluid_build/cli/security.py | 2 +- fluid_build/copilot/catalog/_patterns.py | 5 +- fluid_build/copilot/catalog/bigquery.py | 4 +- fluid_build/copilot/missions/spec.py | 4 +- fluid_build/forge/core/bundle.py | 2 +- fluid_build/forge/federation.py | 3 +- fluid_build/iac/providers/aws.py | 25 +- fluid_build/iac/providers/databricks_jobs.py | 6 +- fluid_build/iac/row_access.py | 2 +- fluid_build/providers/aws/util/validation.py | 16 +- fluid_build/providers/bigquery_validation.py | 2 +- fluid_build/providers/gcp/plan/planner.py | 2 +- fluid_build/providers/gcp/util/names.py | 8 +- .../providers/odcs/mappers/normalize.py | 2 +- .../providers/odps_standard/provider.py | 2 +- fluid_build/providers/quality_engine.py | 2 +- .../providers/snowflake/util/config.py | 4 +- fluid_build/schedulers/airflow/fluid_apply.py | 8 +- .../security/test_trailing_newline_anchors.py | 429 ++++++++++++++++++ 30 files changed, 518 insertions(+), 48 deletions(-) create mode 100644 tests/security/test_trailing_newline_anchors.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a708826c..c522dfc67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -252,6 +252,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `Invalid SQL identifier: 'spectrum_events\n'`. That includes a name that reaches it with a trailing newline from an environment variable or a file; strip the value where it is set. +- **Other validators let a value with a trailing newline through, too.** The same `$` ended other + whole-string patterns, so a value written as a YAML `|` block, read from a `.env` file or decoded + from `%0A` in a URL kept its newline: `fluid generate iac` put `region = "eu-west-1\n"` on the + AWS provider, and a Databricks job got `branch = "main\n"` from `?branch=main%0A`. They now end + in `\Z` or are matched with `fullmatch`, so such a value is refused with the validator's usual + message, or takes the path the code already has for a value that is not a plain name: a product + id forged from a JDBC source or imported from dbt or ODCS is slugified. An AWS binding region + with whitespace around it is refused (`aws-region-whitespace`), because the sovereignty policy + checks the region as written. Covered: quality-check selectors, catalog and JDBC identifiers, + BigQuery and GCP project ids, contract, build, federation-workspace and overlay-env ids, + row-filter names, Singer tap and target names, prompt profile and overlay names, mission + predicate paths, AWS and GCP resource names, Snowflake account segments, Databricks git refs, + and the identifier and version checks of ODCS and ODPS imports. + ## [0.26.0] — 2026-11-01 ### Added diff --git a/fluid_build/build_runners/_ids.py b/fluid_build/build_runners/_ids.py index b2d3abde5..831263a63 100644 --- a/fluid_build/build_runners/_ids.py +++ b/fluid_build/build_runners/_ids.py @@ -44,7 +44,7 @@ # DOS-via-filename attacks. Crucially this excludes ``/``, ``\``, ``..`` # (a leading dot is rejected) and any path separator, so a validated id # can never traverse out of the directory it is joined into. -_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.\-]{0,127}$") +_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.\-]{0,127}\Z") class IdentifierViolation(ValueError): diff --git a/fluid_build/build_runners/meltano/runner.py b/fluid_build/build_runners/meltano/runner.py index 25dc10565..25b2769f6 100644 --- a/fluid_build/build_runners/meltano/runner.py +++ b/fluid_build/build_runners/meltano/runner.py @@ -112,8 +112,8 @@ def collect_singer_output( # ── Tap invocation ────────────────────────────────────────────────────── -_TAP_NAME_RE = re.compile(r"^tap-[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?$") -_TARGET_NAME_RE = re.compile(r"^target-[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?$") +_TAP_NAME_RE = re.compile(r"^tap-[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?\Z") +_TARGET_NAME_RE = re.compile(r"^target-[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?\Z") def _resolve_tap_binary(tap_name: str, *, project_dir: Optional[Path] = None) -> Optional[str]: diff --git a/fluid_build/cli/_ai_setup_storage.py b/fluid_build/cli/_ai_setup_storage.py index ecef764be..7d74ff9dd 100644 --- a/fluid_build/cli/_ai_setup_storage.py +++ b/fluid_build/cli/_ai_setup_storage.py @@ -115,7 +115,7 @@ def _plaintext_env_var() -> str: # Privacy guard: only a bare domain slug is ever persisted. Mirrors # ``forge_domain_enrichment._SAFE_DOMAIN_RE``, lower-cased and length-capped — # the regex is what keeps prompt text / paths / PII out of the history file. -_DOMAIN_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$") +_DOMAIN_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}\Z") def _allow_plaintext_ai_secrets() -> bool: diff --git a/fluid_build/cli/_forge_data_model_jdbc.py b/fluid_build/cli/_forge_data_model_jdbc.py index 6daddcfac..0be635a48 100644 --- a/fluid_build/cli/_forge_data_model_jdbc.py +++ b/fluid_build/cli/_forge_data_model_jdbc.py @@ -36,7 +36,7 @@ # Canonical FLUID contract-id pattern (lowercased form — ``product_id`` is # lowercased before the check). Mirrors the JSON-schema id constraint. -_FLUID_ID_RE = re.compile(r"^[a-z0-9_][a-z0-9_.-]*[a-z0-9_]$|^[a-z0-9_]$") +_FLUID_ID_RE = re.compile(r"^[a-z0-9_][a-z0-9_.-]*[a-z0-9_]\Z|^[a-z0-9_]\Z") def _run_from_jdbc_source( diff --git a/fluid_build/cli/discover/_jdbc_introspect.py b/fluid_build/cli/discover/_jdbc_introspect.py index 39d04be79..146739408 100644 --- a/fluid_build/cli/discover/_jdbc_introspect.py +++ b/fluid_build/cli/discover/_jdbc_introspect.py @@ -541,7 +541,7 @@ def _attach_string_sqlite(args: Dict[str, str], alias: str) -> str: return f"ATTACH {quote_ansi_string_literal(path)} AS {alias} (TYPE sqlite)" -_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*\Z") def _validate_alias(alias: str) -> str: diff --git a/fluid_build/cli/forge_copilot_prompts.py b/fluid_build/cli/forge_copilot_prompts.py index cb5dade65..319c3acd2 100644 --- a/fluid_build/cli/forge_copilot_prompts.py +++ b/fluid_build/cli/forge_copilot_prompts.py @@ -253,7 +253,7 @@ def get_active_domain() -> Optional[str]: # leading dot/underscore. This is the first line of defence against # ``--prompt-profile ../../etc``; ``_resolve_profile_dir`` adds a resolved-path # containment check as the second. -_PROFILE_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_PROFILE_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*\Z") class PromptProfileError(ValueError): diff --git a/fluid_build/cli/forge_domain_enrichment.py b/fluid_build/cli/forge_domain_enrichment.py index ba581b4cc..04d9620e3 100644 --- a/fluid_build/cli/forge_domain_enrichment.py +++ b/fluid_build/cli/forge_domain_enrichment.py @@ -47,7 +47,7 @@ # override — defence in depth against a ``../``-style traversal reaching a # fragment block in an out-of-tree YAML. Matches the profile-name guard in # ``forge_copilot_prompts._PROFILE_NAME_RE``. -_SAFE_DOMAIN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_SAFE_DOMAIN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*\Z") @lru_cache(maxsize=1) diff --git a/fluid_build/cli/forge_prompt_overlays.py b/fluid_build/cli/forge_prompt_overlays.py index 5842f2ac7..6accf0781 100644 --- a/fluid_build/cli/forge_prompt_overlays.py +++ b/fluid_build/cli/forge_prompt_overlays.py @@ -96,7 +96,7 @@ # leading dot/underscore. First line of defence against # ``--prompt-overlay ../../etc``; :func:`_resolve_overlay_path` adds a # resolved-realpath containment check as the second. -_OVERLAY_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_OVERLAY_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*\Z") _ALLOWED_MODES = ("replace", "append", "prepend") diff --git a/fluid_build/cli/forge_validation.py b/fluid_build/cli/forge_validation.py index c6cb37867..966ee0058 100644 --- a/fluid_build/cli/forge_validation.py +++ b/fluid_build/cli/forge_validation.py @@ -46,7 +46,7 @@ def validate_project_name(name: str) -> Tuple[bool, Optional[str]]: if len(name) > 63: return False, "Project name must be 63 characters or less" - if not re.match(r"^[a-z][a-z0-9-]*[a-z0-9]$", name): + if not re.match(r"^[a-z][a-z0-9-]*[a-z0-9]\Z", name): return ( False, "Project name must start with a letter, end with letter/number, and contain only lowercase letters, numbers, and hyphens", @@ -147,7 +147,7 @@ def validate_template_name(template: str) -> Tuple[bool, Optional[str]]: if len(template) < 2: return False, "Template name must be at least 2 characters" - if not re.match(r"^[a-z][a-z0-9-]*$", template): + if not re.match(r"^[a-z][a-z0-9-]*\Z", template): return ( False, "Template name must start with a letter and contain only lowercase letters, numbers, and hyphens", diff --git a/fluid_build/cli/import_workflow/dbt.py b/fluid_build/cli/import_workflow/dbt.py index 34edaad3e..422d5f7a2 100644 --- a/fluid_build/cli/import_workflow/dbt.py +++ b/fluid_build/cli/import_workflow/dbt.py @@ -64,7 +64,7 @@ _REF_RE = re.compile(r"ref\(\s*(?:['\"][^'\"]+['\"]\s*,\s*)?['\"]([^'\"]+)['\"]\s*\)") # FLUID identifier pattern (schema `$defs.identifier`). -_IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]*[A-Za-z0-9_]$|^[A-Za-z0-9_]$") +_IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]*[A-Za-z0-9_]\Z|^[A-Za-z0-9_]\Z") # dbt folder layer → medallion layer (mirrors engines/dbt/models.py). _LAYER_TO_MEDALLION = {"staging": "Bronze", "intermediate": "Silver", "marts": "Gold"} diff --git a/fluid_build/cli/security.py b/fluid_build/cli/security.py index 3e2f82f9d..d2add1e77 100644 --- a/fluid_build/cli/security.py +++ b/fluid_build/cli/security.py @@ -662,7 +662,7 @@ def validate_project_name(name: str) -> bool: return False # Check format: alphanumeric, hyphens, underscores only - if not re.match(r"^[a-zA-Z0-9_-]+$", name): + if not re.match(r"^[a-zA-Z0-9_-]+\Z", name): return False # Check length diff --git a/fluid_build/copilot/catalog/_patterns.py b/fluid_build/copilot/catalog/_patterns.py index 19edce90e..7c443ce30 100644 --- a/fluid_build/copilot/catalog/_patterns.py +++ b/fluid_build/copilot/catalog/_patterns.py @@ -99,7 +99,8 @@ # Conservative default identifier shape — covers Snowflake / BigQuery / # Unity / Glue without quoting weirdness. Catalogs with looser rules -# (e.g., DataHub URN strings) override the regex per-adapter. +# (e.g., DataHub URN strings) override the regex per-adapter. Matched +# with ``fullmatch``: ``$`` also matches before a trailing newline. _DEFAULT_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") @@ -143,7 +144,7 @@ def validate_and_quote_identifier( non-SQL-shaped identifiers. """ rule = pattern or _DEFAULT_IDENTIFIER - if not rule.match(value or ""): + if not rule.fullmatch(value or ""): raise CatalogConfigError( message=f"Invalid {kind} identifier: {value!r}", suggestions=[ diff --git a/fluid_build/copilot/catalog/bigquery.py b/fluid_build/copilot/catalog/bigquery.py index bfaba7e61..aa9de5af3 100644 --- a/fluid_build/copilot/catalog/bigquery.py +++ b/fluid_build/copilot/catalog/bigquery.py @@ -86,6 +86,8 @@ # # The pattern table here keeps both rules visible — adding a new # identifier kind (e.g., a connection name) just adds one entry. +# Match these with ``fullmatch``: their ``$`` also matches before a +# trailing newline, so ``.match`` lets ``"orders\n"`` through. _BQ_IDENT_PATTERNS = { # Allow hyphens for project IDs. "project": re.compile(r"^[A-Za-z][A-Za-z0-9-]*$"), @@ -113,7 +115,7 @@ def _quote_ident(value: str, *, kind: str) -> str: def _validate_ident(value: str, *, kind: str) -> str: """Return a BigQuery identifier after strict kind-specific validation.""" pattern = _BQ_IDENT_PATTERNS.get(kind) - if pattern is None or not pattern.match(value or ""): + if pattern is None or not pattern.fullmatch(value or ""): raise CatalogConfigError( message=f"Invalid BigQuery {kind} identifier: {value!r}", suggestions=[ diff --git a/fluid_build/copilot/missions/spec.py b/fluid_build/copilot/missions/spec.py index 879c019fc..64c435977 100644 --- a/fluid_build/copilot/missions/spec.py +++ b/fluid_build/copilot/missions/spec.py @@ -70,7 +70,7 @@ _BUDGET_KEYS = frozenset({"max_usd", "max_iterations", "max_wall_seconds"}) #: One dotted-path segment: a key, optionally fanned out over an array. -_SEGMENT_RE = re.compile(r"^([A-Za-z_][A-Za-z0-9_-]*)(\[\*\])?$") +_SEGMENT_RE = re.compile(r"^([A-Za-z_][A-Za-z0-9_-]*)(\[\*\])?\Z") _MISSION_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$") @@ -172,7 +172,7 @@ def validate_predicate_path(path: str, *, label: str) -> None: for segment in path.split("."): if not _SEGMENT_RE.match(segment): raise MissionSpecError( - f"{label}: invalid path segment '{segment}' in '{path}'. The predicate " + f"{label}: invalid path segment {segment!r} in {path!r}. The predicate " "grammar is frozen: dotted keys with optional [*] array fan-out " "(e.g. exposes[*].contract.dq.rules)." ) diff --git a/fluid_build/forge/core/bundle.py b/fluid_build/forge/core/bundle.py index 3758ed067..ca4e757af 100644 --- a/fluid_build/forge/core/bundle.py +++ b/fluid_build/forge/core/bundle.py @@ -492,7 +492,7 @@ def bundle_overlay_warnings(source: Optional[Mapping[str, Any]]) -> List[str]: #: path. It selects the overlay file whose ``id`` the recorded source #: contract is checked with, so a value like ``../../x`` must not be able to #: pick an arbitrary file for that check. -_ENV_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$") +_ENV_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*\Z") def bundle_source_contract( diff --git a/fluid_build/forge/federation.py b/fluid_build/forge/federation.py index 76df5e109..4a1c7086f 100644 --- a/fluid_build/forge/federation.py +++ b/fluid_build/forge/federation.py @@ -236,6 +236,7 @@ def _federation_git_timeout() -> float: # (``~/.cache/fluid/federation-git/`` and the digest-cache file). # Constrain it to a conservative slug so a tampered manifest row can't # smuggle ``../`` traversal or absolute-path segments into those sinks. +# Matched with ``fullmatch``: ``$`` also matches before a trailing newline. _WORKSPACE_ID_RE = re.compile(r"^[A-Za-z0-9_.-]{1,64}$") @@ -589,7 +590,7 @@ def from_dict(cls, d: Mapping[str, Any]) -> "FederatedWorkspace": # anything outside a conservative slug so a tampered manifest # row cannot smuggle ``../`` traversal or absolute segments into # those path sinks. - if not _WORKSPACE_ID_RE.match(ws_id): + if not _WORKSPACE_ID_RE.fullmatch(ws_id): raise ValueError( f"federation workspace id {ws_id!r} is invalid: must match " f"{_WORKSPACE_ID_RE.pattern} (alphanumerics, dot, dash, " diff --git a/fluid_build/iac/providers/aws.py b/fluid_build/iac/providers/aws.py index da0f890cc..d9c04dfc1 100644 --- a/fluid_build/iac/providers/aws.py +++ b/fluid_build/iac/providers/aws.py @@ -103,7 +103,7 @@ # binding carries (a jurisdiction such as "EU", a Google region, an unresolved # "{{ env.AWS_REGION }}") must not become the provider's region; it used to be # ignored, and still is. -_AWS_REGION_RE = re.compile(r"^[a-z]{2}(-[a-z]+)+-\d+$") +_AWS_REGION_RE = re.compile(r"^[a-z]{2}(-[a-z]+)+-\d+\Z") def recorded_regions(resources: Iterable[Mapping[str, Any]]) -> set: @@ -735,6 +735,29 @@ def provider_block_for(self, contract: Mapping[str, Any]) -> Dict[str, Any]: provider v6). """ cfg = dict(self.provider_block()) + for index, expose in enumerate(contract.get("exposes") or []): + binding = expose.get("binding") or {} + raw = (binding.get("location") or {}).get("region") + # A region code with whitespace around it (a YAML ``|`` block ends in a + # newline) is refused. Dropping it leaves the provider on the + # environment's region; pinning it stripped applies a region the + # sovereignty policy, which reads the value as written, never checked. + if ( + is_cloud(binding, "aws") + and isinstance(raw, str) + and raw != raw.strip() + and _AWS_REGION_RE.match(raw.strip()) + ): + raise UnsupportedBindingError( + "aws-region-whitespace", + f"exposes[{expose.get('exposeId') or index}] binding.location.region is " + f"{raw!r}: a region code with whitespace around it. The sovereignty policy " + "checks the region as written, so it is refused rather than pinned stripped.", + ( + f"Write the region alone: region: {raw.strip()} (a YAML '|' block ends " + "the value in a newline).", + ), + ) named = { str(((e.get("binding") or {}).get("location") or {}).get("region")) for e in contract.get("exposes") or [] diff --git a/fluid_build/iac/providers/databricks_jobs.py b/fluid_build/iac/providers/databricks_jobs.py index d78abac85..3efa289a0 100644 --- a/fluid_build/iac/providers/databricks_jobs.py +++ b/fluid_build/iac/providers/databricks_jobs.py @@ -123,9 +123,9 @@ def _under_home(prefix: str, suffix: str) -> TofuExpr: "dev.azure.com": "azureDevOpsServices", } _GIT_REFS = ("branch", "tag", "commit") -_GIT_REF_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]*$") -_COMMIT_RE = re.compile(r"^[0-9a-fA-F]{7,40}$") -_REPO_PATH_RE = re.compile(r"^[A-Za-z0-9._/-]+$") +_GIT_REF_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]*\Z") +_COMMIT_RE = re.compile(r"^[0-9a-fA-F]{7,40}\Z") +_REPO_PATH_RE = re.compile(r"^[A-Za-z0-9._/-]+\Z") #: ``1h``, ``30m``, ``45s``, ``2d``, a bare number of seconds, or ISO 8601 ``PT1H30M``. _DURATION_RE = re.compile(r"^(\d+)\s*([smhd]?)$") diff --git a/fluid_build/iac/row_access.py b/fluid_build/iac/row_access.py index 9d758b24f..e234ac5b0 100644 --- a/fluid_build/iac/row_access.py +++ b/fluid_build/iac/row_access.py @@ -43,7 +43,7 @@ from .base import UnsupportedBindingError from .column_access import policy_where -_NAME = re.compile(r"^[a-z][a-z0-9_]{0,62}$") +_NAME = re.compile(r"^[a-z][a-z0-9_]{0,62}\Z") @dataclass(frozen=True) diff --git a/fluid_build/providers/aws/util/validation.py b/fluid_build/providers/aws/util/validation.py index 2eb820e40..38a7bb770 100644 --- a/fluid_build/providers/aws/util/validation.py +++ b/fluid_build/providers/aws/util/validation.py @@ -37,10 +37,10 @@ class ResourceValidator: """Validates AWS resource configurations before execution.""" # AWS naming rules - S3_BUCKET_PATTERN = re.compile(r"^[a-z0-9][a-z0-9\-]{1,61}[a-z0-9]$") - GLUE_DATABASE_PATTERN = re.compile(r"^[a-z0-9_]+$") - GLUE_TABLE_PATTERN = re.compile(r"^[a-z0-9_]+$") - LAMBDA_FUNCTION_PATTERN = re.compile(r"^[a-zA-Z0-9\-_]+$") + S3_BUCKET_PATTERN = re.compile(r"^[a-z0-9][a-z0-9\-]{1,61}[a-z0-9]\Z") + GLUE_DATABASE_PATTERN = re.compile(r"^[a-z0-9_]+\Z") + GLUE_TABLE_PATTERN = re.compile(r"^[a-z0-9_]+\Z") + LAMBDA_FUNCTION_PATTERN = re.compile(r"^[a-zA-Z0-9\-_]+\Z") # AWS service quotas (defaults - can be increased) DEFAULT_QUOTAS = { @@ -91,7 +91,7 @@ def validate_s3_bucket_name(self, bucket: str) -> Tuple[bool, Optional[str]]: if not self.S3_BUCKET_PATTERN.match(bucket): return False, ( - f"Bucket name '{bucket}' is invalid. " + f"Bucket name {bucket!r} is invalid. " "Must contain only lowercase letters, numbers, and hyphens. " "Must start and end with letter or number." ) @@ -135,7 +135,7 @@ def validate_glue_database_name(self, database: str) -> Tuple[bool, Optional[str if not self.GLUE_DATABASE_PATTERN.match(database): return False, ( - f"Database name '{database}' is invalid. " + f"Database name {database!r} is invalid. " "Must contain only lowercase letters, numbers, and underscores." ) @@ -164,7 +164,7 @@ def validate_glue_table_name(self, table: str) -> Tuple[bool, Optional[str]]: if not self.GLUE_TABLE_PATTERN.match(table): return False, ( - f"Table name '{table}' is invalid. " + f"Table name {table!r} is invalid. " "Must contain only lowercase letters, numbers, and underscores." ) @@ -193,7 +193,7 @@ def validate_lambda_function_name(self, function: str) -> Tuple[bool, Optional[s if not self.LAMBDA_FUNCTION_PATTERN.match(function): return False, ( - f"Function name '{function}' is invalid. " + f"Function name {function!r} is invalid. " "Must contain only letters, numbers, hyphens, and underscores." ) diff --git a/fluid_build/providers/bigquery_validation.py b/fluid_build/providers/bigquery_validation.py index 4d6864e90..6482cb21c 100644 --- a/fluid_build/providers/bigquery_validation.py +++ b/fluid_build/providers/bigquery_validation.py @@ -43,7 +43,7 @@ # need a dedicated allowlist: 6–30 chars, lowercase-or-mixed letter start, # letters/digits/hyphens in the middle, and a letter/digit terminator. # See https://cloud.google.com/resource-manager/docs/creating-managing-projects. -_BQ_PROJECT_ID = re.compile(r"^[A-Za-z][A-Za-z0-9-]{4,28}[A-Za-z0-9]$") +_BQ_PROJECT_ID = re.compile(r"^[A-Za-z][A-Za-z0-9-]{4,28}[A-Za-z0-9]\Z") def _quote_bq_project(project: str) -> str: diff --git a/fluid_build/providers/gcp/plan/planner.py b/fluid_build/providers/gcp/plan/planner.py index a4d4ce62b..91883c8aa 100644 --- a/fluid_build/providers/gcp/plan/planner.py +++ b/fluid_build/providers/gcp/plan/planner.py @@ -37,7 +37,7 @@ # contain hyphens, so ``_sql_safety.validate_ident`` (which rejects ``-``) # cannot vet them — this dedicated check guards the project component of a # BigQuery fully-qualified name before it is interpolated into DDL/DML. -_GCP_PROJECT_RE = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") +_GCP_PROJECT_RE = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]\Z") def _validate_bq_project(project: str) -> str: diff --git a/fluid_build/providers/gcp/util/names.py b/fluid_build/providers/gcp/util/names.py index 6a3f80fe5..c25db891b 100644 --- a/fluid_build/providers/gcp/util/names.py +++ b/fluid_build/providers/gcp/util/names.py @@ -324,7 +324,7 @@ def validate_name(name: str, resource_type: str) -> bool: try: if resource_type in ["dataset", "table"]: # BigQuery naming rules - if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", name): + if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*\Z", name): return False return len(name) <= 1024 @@ -332,7 +332,7 @@ def validate_name(name: str, resource_type: str) -> bool: # Cloud Storage naming rules if len(name) < 3 or len(name) > 63: return False - if not re.match(r"^[a-z0-9][a-z0-9\-]*[a-z0-9]$", name): + if not re.match(r"^[a-z0-9][a-z0-9\-]*[a-z0-9]\Z", name): return False if re.match(r"^\d+\.\d+\.\d+\.\d+$", name): return False @@ -344,7 +344,7 @@ def validate_name(name: str, resource_type: str) -> bool: return False if not name[0].isalpha(): return False - return re.match(r"^[a-zA-Z][a-zA-Z0-9\-\._~%+]*$", name) is not None + return re.match(r"^[a-zA-Z][a-zA-Z0-9\-\._~%+]*\Z", name) is not None elif resource_type == "job": # Job naming rules @@ -354,7 +354,7 @@ def validate_name(name: str, resource_type: str) -> bool: return False if name[-1] == "-": return False - return re.match(r"^[a-z][a-z0-9\-]*[a-z0-9]$", name) is not None + return re.match(r"^[a-z][a-z0-9\-]*[a-z0-9]\Z", name) is not None else: # Unknown resource type diff --git a/fluid_build/providers/odcs/mappers/normalize.py b/fluid_build/providers/odcs/mappers/normalize.py index 94ff451dc..b7e028deb 100644 --- a/fluid_build/providers/odcs/mappers/normalize.py +++ b/fluid_build/providers/odcs/mappers/normalize.py @@ -462,7 +462,7 @@ def _root_identifier(raw: Any, version: str) -> tuple[str, Optional[str]]: return _IMPORTED_ID_FALLBACK, None text = str(raw) pattern = _identifier_pattern(version) - if pattern is None or pattern.match(text): + if pattern is None or pattern.fullmatch(text): return text, None slug = _slugged_root_id(text) # %r: the id is document text and may carry control characters. diff --git a/fluid_build/providers/odps_standard/provider.py b/fluid_build/providers/odps_standard/provider.py index e03e17edd..c03a07c50 100644 --- a/fluid_build/providers/odps_standard/provider.py +++ b/fluid_build/providers/odps_standard/provider.py @@ -1141,7 +1141,7 @@ def _schema_defs(version: str) -> Mapping[str, Any]: def _is_semver(value: Any, version: str) -> bool: pattern = _semver_pattern(version) - return pattern is None or bool(pattern.match(str(value))) + return pattern is None or bool(pattern.fullmatch(str(value))) def _rehydrate_document(fluid: Mapping[str, Any]) -> Mapping[str, Any]: diff --git a/fluid_build/providers/quality_engine.py b/fluid_build/providers/quality_engine.py index d4ba6191c..a3f8741d7 100644 --- a/fluid_build/providers/quality_engine.py +++ b/fluid_build/providers/quality_engine.py @@ -41,7 +41,7 @@ LOG = logging.getLogger("fluid.providers.quality_engine") # Safe SQL identifier regex (letters, digits, underscores only) -_SAFE_IDENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_SAFE_IDENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*\Z") def _validate_ident(name: str) -> str: diff --git a/fluid_build/providers/snowflake/util/config.py b/fluid_build/providers/snowflake/util/config.py index 6a44ade67..e2fc67f9f 100644 --- a/fluid_build/providers/snowflake/util/config.py +++ b/fluid_build/providers/snowflake/util/config.py @@ -36,8 +36,8 @@ # share the canonical pattern without redefining or importing a private name. ENV_TEMPLATE_RE = _ENV_TEMPLATE_RE _ACCOUNT_HOST_RE = re.compile(r"\.snowflakecomputing\.com$", re.IGNORECASE) -_ACCOUNT_SEGMENT_RE = re.compile(r"^[A-Za-z0-9_-]+$") -_ACCOUNT_REGION_SEGMENT_RE = re.compile(r"^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)+$") +_ACCOUNT_SEGMENT_RE = re.compile(r"^[A-Za-z0-9_-]+\Z") +_ACCOUNT_REGION_SEGMENT_RE = re.compile(r"^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)+\Z") _ACCOUNT_CLOUD_SUFFIXES = frozenset({"aws", "azure", "gcp"}) # Keys that carry secret material. These MUST never appear in the `_sources` diff --git a/fluid_build/schedulers/airflow/fluid_apply.py b/fluid_build/schedulers/airflow/fluid_apply.py index 7a38496fe..5c38c18bd 100644 --- a/fluid_build/schedulers/airflow/fluid_apply.py +++ b/fluid_build/schedulers/airflow/fluid_apply.py @@ -487,11 +487,11 @@ def validate_contract_path(raw: Any) -> str: def validate_id(raw: Any, *, kind: str) -> str: - """:func:`validate_identifier`, refusing a trailing newline as well. + """:func:`validate_identifier` as a :class:`ScheduleRenderError`, with its own newline check. - The shared grammar ends in ``$``, which also matches before a final - ``\\n``; an id or env that carries one names nothing fluid can find - (``--env 'aws\\n'`` silently applies no overlay). + An id or env that ends in a newline names nothing fluid can find (``--env + 'aws\\n'`` silently applies no overlay). The shared grammar ends in + ``\\Z`` and refuses one already; the check below still holds if it changes. """ try: value = validate_identifier(raw, kind=kind) diff --git a/tests/security/test_trailing_newline_anchors.py b/tests/security/test_trailing_newline_anchors.py new file mode 100644 index 000000000..445209b87 --- /dev/null +++ b/tests/security/test_trailing_newline_anchors.py @@ -0,0 +1,429 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Whole-string validators refuse a value that ends in a newline. + +Python's ``$`` matches at the end of the string and also just before a final +``\\n``, so ``re.match(r"^[a-z]+$", "orders\\n")`` succeeds and the caller goes on +to use ``"orders\\n"``: as a path segment, a SQL identifier or literal, a +Terraform attribute, a cache directory. Contract YAML yields such values easily +(a ``|`` block scalar ends in one), and so do ``.env`` files and percent-encoded +URL queries. The validators below end in ``\\Z`` (or call ``fullmatch``), which +matches only at the end of the string. + +Each test hands one validator a value with a trailing newline, next to the same +value without it. It fails while that validator's pattern still ends in ``$``. +""" + +from __future__ import annotations + +import logging +import os +import sqlite3 +from types import SimpleNamespace + +import pytest + +pytestmark = [pytest.mark.unit] + + +# --- SQL identifiers and literals -------------------------------------------- + + +def test_quality_engine_refuses_a_selector_ending_in_a_newline(): + from fluid_build.providers import quality_engine + + assert '"amount"' in quality_engine._completeness_sql("t", "amount") + with pytest.raises(ValueError, match="Invalid SQL identifier"): + quality_engine._completeness_sql("t", "amount\n") + + +def test_catalog_identifier_quoting_refuses_a_trailing_newline(): + from fluid_build.copilot.catalog._patterns import validate_and_quote_identifier + from fluid_build.copilot.catalog.base import CatalogConfigError + + assert validate_and_quote_identifier("SALES", kind="database") == '"SALES"' + with pytest.raises(CatalogConfigError): + validate_and_quote_identifier("SALES\n", kind="database") + + +@pytest.mark.parametrize( + ("kind", "value"), [("project", "acme-proj"), ("dataset", "sales"), ("table", "orders")] +) +def test_bigquery_catalog_identifiers_refuse_a_trailing_newline(kind, value): + from fluid_build.copilot.catalog.base import CatalogConfigError + from fluid_build.copilot.catalog.bigquery import _quote_ident + + assert _quote_ident(value, kind=kind) == f"`{value}`" + with pytest.raises(CatalogConfigError): + _quote_ident(f"{value}\n", kind=kind) + + +@pytest.mark.parametrize( + ("kind", "value"), [("project", "acme-proj"), ("dataset", "sales"), ("table", "orders")] +) +def test_bigquery_catalog_name_check_refuses_a_trailing_newline(kind, value): + # The unquoted check behind `table_name IN ('...')` and FQN parsing. + from fluid_build.copilot.catalog.base import CatalogConfigError + from fluid_build.copilot.catalog.bigquery import _validate_ident + + assert _validate_ident(value, kind=kind) == value + with pytest.raises(CatalogConfigError): + _validate_ident(f"{value}\n", kind=kind) + + +def test_jdbc_schema_filter_refuses_a_trailing_newline(monkeypatch, tmp_path): + pytest.importorskip("duckdb") + from fluid_build.cli.discover import _jdbc_introspect + from fluid_build.providers import _duckdb_sandbox + + def _no_connect(*_args, **_kwargs): + raise AssertionError("introspection connected with an invalid schema filter") + + monkeypatch.setattr(_duckdb_sandbox, "secure_duckdb_connect", _no_connect) + db_path = tmp_path / "source.sqlite" + sqlite3.connect(str(db_path)).close() + with pytest.raises(ValueError, match="Invalid schema filter"): + _jdbc_introspect.introspect_jdbc( + source="sqlite", uri=f"sqlite:///{db_path}", schema_filter="main\n" + ) + + +def test_bigquery_project_quoting_refuses_a_trailing_newline(): + # bigquery_validation imports google.cloud.bigquery at module load, and the + # GCP SDK is a provider extra the minimal unit lanes do not install. + pytest.importorskip("google.cloud.bigquery") + from fluid_build.providers.bigquery_validation import _quote_bq_project + + assert _quote_bq_project("acme-analytics") == "`acme-analytics`" + with pytest.raises(ValueError, match="Invalid BigQuery project id"): + _quote_bq_project("acme-analytics\n") + + +def test_gcp_planner_project_refuses_a_trailing_newline(): + from fluid_build.providers.gcp.plan.planner import _validate_bq_project + + assert _validate_bq_project("acme-analytics") == "acme-analytics" + with pytest.raises(ValueError, match="Invalid GCP project ID"): + _validate_bq_project("acme-analytics\n") + + +def test_snowflake_account_segment_refuses_a_trailing_newline(): + from fluid_build.providers.snowflake.util import config + + assert config._normalize_account_identifier("xy12345.eu-west-1") == "xy12345.eu-west-1" + with pytest.raises(ValueError, match="Invalid Snowflake account identifier"): + config._normalize_account_identifier("xy12345\n.eu-west-1") + + +def test_snowflake_account_region_pattern_refuses_a_trailing_newline(): + # The segment rule runs first and now refuses this input on its own; the + # region rule must hold without it. + from fluid_build.providers.snowflake.util import config + + assert config._ACCOUNT_REGION_SEGMENT_RE.match("eu-west-1") + assert config._ACCOUNT_REGION_SEGMENT_RE.match("eu-west-1\n") is None + + +# --- Names that become paths, files and cache directories --------------------- + + +def test_build_identifier_refuses_a_trailing_newline(): + from fluid_build.build_runners._ids import IdentifierViolation, validate_identifier + + assert validate_identifier("orders", kind="contract.id") == "orders" + with pytest.raises(IdentifierViolation): + validate_identifier("orders\n", kind="contract.id") + + +def test_federation_workspace_id_refuses_a_trailing_newline(): + from fluid_build.forge.federation import FederatedWorkspace + + row = {"id": "telco", "kind": "git_registry", "endpoint": "https://github.com/acme/telco"} + assert FederatedWorkspace.from_dict(row).id == "telco" + with pytest.raises(ValueError, match="federation workspace id"): + FederatedWorkspace.from_dict({**row, "id": "telco\n"}) + + +def test_bundle_recorded_env_refuses_a_trailing_newline(tmp_path): + from fluid_build.forge.core.bundle import resolve_bundle_source_contract + + tgz = tmp_path / "bundle.tgz" + _, why = resolve_bundle_source_contract(tgz, {"contract": "contract.fluid.yaml", "env": "aws"}) + assert "environment name" not in why + path, why = resolve_bundle_source_contract( + tgz, {"contract": "contract.fluid.yaml", "env": "aws\n"} + ) + assert path is None + assert "is not an environment name" in why + + +def test_consumes_overlay_env_refuses_a_trailing_newline(): + from fluid_build.build_runners._embedded_sql_io import ConsumesResolutionError, _validated_env + + assert _validated_env("aws") == "aws" + with pytest.raises(ConsumesResolutionError): + _validated_env("aws\n") + + +@pytest.mark.skipif(os.name == "nt", reason="a Windows file name cannot hold a newline") +@pytest.mark.parametrize( + ("resolver", "name"), + [("_resolve_tap_binary", "tap-postgres"), ("_resolve_target_binary", "target-jsonl")], +) +def test_singer_binary_lookup_refuses_a_name_ending_in_a_newline( + monkeypatch, tmp_path, resolver, name +): + from fluid_build.build_runners.meltano import runner + + for filename in (name, f"{name}\n"): + binary = tmp_path / filename + binary.write_text("#!/bin/sh\n", encoding="utf-8") + binary.chmod(0o755) + monkeypatch.setenv("PATH", str(tmp_path)) + resolve = getattr(runner, resolver) + + assert resolve(name) == str(tmp_path / name) + assert resolve(f"{name}\n") is None + + +def test_prompt_profile_name_refuses_a_trailing_newline(): + from fluid_build.cli.forge_copilot_prompts import PromptProfileError, _resolve_profile_dir + + assert _resolve_profile_dir("eu-gdpr-strict").name == "eu-gdpr-strict" + with pytest.raises(PromptProfileError, match="invalid prompt profile name"): + _resolve_profile_dir("eu-gdpr-strict\n") + + +@pytest.mark.parametrize( + ("module", "name"), + [ + ("fluid_build.cli.forge_domain_enrichment", "_SAFE_DOMAIN_RE"), + ("fluid_build.cli._ai_setup_storage", "_DOMAIN_SLUG_RE"), + ], +) +def test_domain_slug_patterns_refuse_a_trailing_newline(module, name): + # Fed a stripped or on-disk value, so pinned on the pattern: their comments + # say they mirror the profile-name guard, and they now do. + import importlib + + pattern = getattr(importlib.import_module(module), name) + assert pattern.match("finance") + assert pattern.match("finance\n") is None + + +def test_prompt_overlay_name_refuses_a_trailing_newline(): + from fluid_build.cli.forge_prompt_overlays import PromptOverlayError, _resolve_overlay_path + + assert _resolve_overlay_path("pii-lockdown").name == "pii-lockdown.yaml" + with pytest.raises(PromptOverlayError, match="invalid prompt overlay name"): + _resolve_overlay_path("pii-lockdown\n") + + +# --- Identifiers written into a contract ------------------------------------- + + +def test_jdbc_forged_product_id_drops_a_trailing_newline(monkeypatch, tmp_path): + import yaml + + from fluid_build.cli import _forge_data_model_jdbc + from fluid_build.cli.discover import _jdbc_introspect + + table = _jdbc_introspect.IntrospectedTable( + schema="public", + name="orders", + columns=[_jdbc_introspect.IntrospectedColumn(name="id", type_name="integer")], + ) + db = _jdbc_introspect.IntrospectedDatabase( + source_kind="postgres", database="orders", tables=[table] + ) + monkeypatch.setattr(_jdbc_introspect, "introspect_jdbc", lambda **_kwargs: db) + + def _forged_id(name): + out = tmp_path / "out.fluid.yaml" + args = SimpleNamespace( + source="postgres", + uri="postgresql://forge@db.example/orders", + schema_name=None, + tables=None, + name=name, + output=str(out), + ) + assert _forge_data_model_jdbc._run_from_jdbc_source(args, logging.getLogger("test")) == 0 + return yaml.safe_load(out.read_text(encoding="utf-8"))["id"] + + assert _forged_id("orders") == "orders" + assert _forged_id("orders\n") == "orders" + + +def test_dbt_importer_slugifies_an_identifier_ending_in_a_newline(): + from fluid_build.cli.import_workflow.dbt import _safe_identifier + + assert _safe_identifier("orders") == "orders" + assert _safe_identifier("orders\n") == "orders" + + +def test_odcs_import_slugifies_an_id_ending_in_a_newline(): + # The bundled schema's identifier pattern ends in `$`; it is matched whole. + from fluid_build.providers.odcs.mappers.normalize import _root_identifier + + assert _root_identifier("orders", "0.7.5") == ("orders", None) + assert _root_identifier("orders\n", "0.7.5")[0] == "orders" + + +def test_odps_import_semver_check_refuses_a_trailing_newline(): + from fluid_build.providers.odps_standard.provider import _is_semver + + assert _is_semver("1.2.3", "0.7.5") is True + assert _is_semver("1.2.3\n", "0.7.5") is False + + +def test_mission_predicate_path_refuses_a_trailing_newline(): + from fluid_build.copilot.missions.spec import MissionSpecError, validate_predicate_path + + # A stripped path can still carry one at the end of an inner segment. + validate_predicate_path("exposes[*].contract.dq", label="checks[0]") + with pytest.raises(MissionSpecError, match=r"invalid path segment 'contract\\n'"): + validate_predicate_path("exposes[*].contract\n.dq", label="checks[0]") + + +# --- Values emitted into OpenTofu --------------------------------------------- + + +def test_row_filter_name_refuses_a_trailing_newline(): + from fluid_build.iac.base import UnsupportedBindingError + from fluid_build.iac.row_access import row_filters_for + + def _expose(name): + row_filter = { + "principal": "group:analysts@northwind.example", + "name": name, + "where": "consent = true", + } + return { + "exposeId": "profile", + "contract": {"schema": [{"name": "consent", "type": "boolean"}]}, + "policy": {"authz": {"rowFilters": [row_filter]}}, + } + + assert row_filters_for(_expose("analysts_consented"))[0].name == "analysts_consented" + with pytest.raises(UnsupportedBindingError, match="must be lowercase letters"): + row_filters_for(_expose("analysts_consented\n")) + + +def _aws_contract(region): + location = { + "bucket": "acme-lake", + "path": "bronze/orders/", + "database": "bronze", + "table": "orders", + "region": region, + } + expose = { + "exposeId": "orders", + "binding": {"platform": "aws", "format": "parquet", "location": location}, + "contract": {"schema": [{"name": "id", "type": "string"}]}, + } + return {"id": "bronze.orders", "exposes": [expose]} + + +def test_aws_binding_region_ending_in_a_newline_is_refused(monkeypatch): + # Neither dropped (the provider would take the environment's region) nor + # stripped (sovereignty checks the value as written, so a stripped pin + # could apply a region its deniedRegions names). + from fluid_build.iac import get_iac_plugin, provider_config + from fluid_build.iac.base import UnsupportedBindingError + + for name in [k for k in os.environ if k.startswith("AWS_ENDPOINT_URL")]: + monkeypatch.delenv(name) + plugin = get_iac_plugin("aws") + + assert provider_config(plugin, _aws_contract("eu-west-1"))["region"] == "eu-west-1" + with pytest.raises(UnsupportedBindingError, match="whitespace around it"): + provider_config(plugin, _aws_contract("eu-west-1\n")) + + +def test_aws_recorded_region_ending_in_a_newline_is_not_a_region(): + from fluid_build.iac.providers.aws import recorded_regions + + assert recorded_regions([{"values": {"region": "eu-west-1"}}]) == {"eu-west-1"} + assert recorded_regions([{"values": {"region": "eu-west-1\n"}}]) == set() + + +@pytest.mark.parametrize( + "query", ["branch=main%0A", "tag=v1.2%0A", "commit=0a1b2c3%0A", "branch=main&path=dbt%0A"] +) +def test_databricks_git_source_refuses_a_ref_or_path_ending_in_a_newline(query): + from fluid_build.iac.providers.databricks_jobs import _git_source + + clean, refused = [], [] + repository = f"https://github.com/acme/jobs?{query}" + assert _git_source(repository.replace("%0A", ""), "builds[0]", clean) is not None + assert not clean + assert _git_source(repository, "builds[0]", refused) is None + assert refused + + +# --- Name validators other code calls --------------------------------------- + + +@pytest.mark.parametrize( + ("method", "value"), + [ + ("validate_s3_bucket_name", "acme-lake"), + ("validate_glue_database_name", "bronze"), + ("validate_glue_table_name", "orders"), + ("validate_lambda_function_name", "ingest-orders"), + ], +) +def test_aws_resource_names_refuse_a_trailing_newline(method, value): + from fluid_build.providers.aws.util.validation import ResourceValidator + + check = getattr(ResourceValidator("123456789012", "eu-west-1"), method) + assert check(value) == (True, None) + valid, message = check(f"{value}\n") + assert valid is False + assert f"{value}\\n" in message # the refused newline is shown, not printed + + +@pytest.mark.parametrize( + ("resource_type", "name"), + [("dataset", "sales"), ("bucket", "acme-lake"), ("topic", "orders-events"), ("job", "nightly")], +) +def test_gcp_resource_names_refuse_a_trailing_newline(resource_type, name): + from fluid_build.providers.gcp.util.names import validate_name + + assert validate_name(name, resource_type) is True + assert validate_name(f"{name}\n", resource_type) is False + + +def test_project_name_check_refuses_a_trailing_newline(): + from fluid_build.cli.security import InputSanitizer + + assert InputSanitizer.validate_project_name("my-project") is True + assert InputSanitizer.validate_project_name("my-project\n") is False + + +def test_forge_project_name_refuses_a_trailing_newline(): + from fluid_build.cli.forge_validation import validate_project_name + + assert validate_project_name("orders-product") == (True, None) + assert validate_project_name("orders-product\n")[0] is False + + +def test_forge_template_name_refuses_a_trailing_newline(): + from fluid_build.cli.forge_validation import validate_template_name + + assert validate_template_name("starter") == (True, None) + assert validate_template_name("starter\n")[0] is False From 1a8bcdeeb5ff2c39f4ca6884fb0efb2f118f7bdc Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:09:46 +0100 Subject: [PATCH 15/26] fix(iac): column types reach Glue and BigQuery faithfully, or validate refuses them (#793) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(iac): column types reach Glue and BigQuery faithfully or are refused at validate fluid validate --strict exited 0 while the emitters wrote wrong or invalid column types. Glue wrote 47 of the 79 schema types as `string`, 29 of them scalars with a real Hive type (smallint, int4, timestamptz, real, varbinary, money ...). BigQuery got `ARRAY` / `STRUCT` with no element type or fields through the ODCS fallback, which no BigQuery table accepts, so the apply failed late. `array(string)` lost its element type on both clouds. fluid_build/iac/column_types.py classifies every type once per cloud, and emit and validate both call it, so they cannot disagree: - exact: the scalar maps to its faithful Glue / BigQuery type; - lossy: a fallback the target needs (json/jsonb/variant/super/time/ interval/bignumeric -> string on Glue; geometry -> GEOGRAPHY on BigQuery) is kept with a validate warning, so --strict fails; - refused (kind column-type): array, struct, map, object, record, row and any parameterised structured type on both clouds (the schema cannot declare element types or fields yet), bit(n) with n > 1, unmapped types, and on Glue geography/geometry/geom/point/hll and a decimal Hive rejects. The BigQuery fallback to the ODCS table is gone. _hive_type and _bq_type stay total for fluid verify, fluid diff and the Dataform engine. On a Glue table applied by an earlier version, the next apply changes the column types in place (string -> smallint and so on). That update is not a delete, so --allow-data-loss never sees it. A new optional plugin hook, plan_update_notices(plan), lets the AWS plugin announce it on the dry run and before the apply: one note per table, `column: old -> new`. The CHANGELOG upgrade notes cover this, the verify/diff drift a user sees until that apply, and every new refusal and warning. Found by an external hands-on assessment of 0.26.0. Measured against moto with real tofu: base-applied table, dry-run notice, apply without --allow-data-loss, live types after. * fix(iac): an in-place Glue column-type change needs --allow-data-loss; print plan data literally A Glue column whose type changes in place (string -> smallint on a table an earlier version applied) can stop Athena reading files written under the old type until they are rewritten; reverting the type restores reads. The AWS plugin now implements plan_data_changes: each aws_glue_catalog_ table update whose column types differ is listed, `column: old -> new`, and the data-loss gate refuses the apply (and the dry run) until --allow-data-loss. A whitespace-only difference (decimal(10, 2) -> decimal(10,2)) never trips it. The advisory plan_update_notices hook from the previous commit is gone; the gate carries the list. Like Databricks, an AWS apply with in-place updates whose saved plan cannot be read back now needs the flag too (fail-closed). Plan data quotes live catalog values, so the gate list and plan notices print through cprint_literal: a live type "string[/]" no longer crashes every apply with a Rich MarkupError, and "[link=...]" no longer renders a hyperlink. Also from review: type parameters must be ASCII digits (decimal(١٠,٢) was classified exact and emitted as written); a BigQuery decimal beyond NUMERIC's range (scale > 9 or more than 29 integer digits) warns and suggests bignumeric; a malformed binding is skipped by the type check instead of raising and silently skipping the governance checks after it. Measured against moto with real tofu: refused on dry run and apply without the flag (live table untouched), applied with it; whitespace-only applied without it. --- CHANGELOG.md | 72 +++ fluid_build/cli/_apply_opentofu_engine.py | 15 +- fluid_build/iac/column_types.py | 153 ++++++ fluid_build/iac/governance_validation.py | 6 + fluid_build/iac/providers/aws.py | 261 +++++++++-- fluid_build/iac/providers/gcp.py | 173 +++++-- tests/iac/test_iac_column_type_gate.py | 540 ++++++++++++++++++++++ 7 files changed, 1147 insertions(+), 73 deletions(-) create mode 100644 fluid_build/iac/column_types.py create mode 100644 tests/iac/test_iac_column_type_gate.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c522dfc67..5bafd6495 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -180,6 +180,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `tofu force-unlock ` in the workdir (the ID is in tofu's error) clears it. A run that hits `FLUID_TOFU_TIMEOUT_SECONDS` is now interrupted (SIGINT) so tofu releases the lock, and killed only if it ignores that for 30 seconds. +- **A column type AWS Glue or BigQuery could not hold was written anyway, and `fluid validate + --strict` passed.** The Glue emitter mapped 22 type names and turned every other one into + `string`: 47 of the 79 types the schema admits, among them `smallint`, `tinyint`, `int2`, + `int4`, `int8`, `real`, `money`, `bit`, `year`, `datetime2`, `timestamptz` and the other + timestamp spellings, `varbinary` and `blob`, and every array, struct, map, object, record and + row. The BigQuery emitter wrote `ARRAY` for an array, which is not a BigQuery column type (an + array is a REPEATED column of an element type), and a `STRUCT` with no fields, which BigQuery + rejects at apply, after the dataset was created; the element type of `array(string)` was + dropped on both clouds. Every type is now classified for each cloud. Each scalar with a Hive or + Athena type gets it (`smallint`, `int`, `bigint`, `float`, `double`, `timestamp`, `binary`, + `money` as `decimal(19,4)`, `bit` as `boolean`, `year` as `smallint`; the zone-aware + timestamps are `timestamp`, which keeps the UTC instant and not the offset; a bare `number` is + `decimal(38,9)`, as a bare `decimal` always was). A type Glue has no form for but that keeps + its value as text (`json`, `jsonb`, `variant`, `super`, `time`, `interval`, `bignumeric`) is + still written as `string`, and `fluid validate` now warns for it, so `--strict` fails; BigQuery + warns for `geometry` and `geom`, which are planar and become the spherical `GEOGRAPHY`, and + for a `decimal(p,s)` wider than a BigQuery NUMERIC holds (29 integer and 9 fractional digits; + declare `bignumeric`). A + type with no faithful form at all is refused with kind `column-type`: an array, struct, map, + object, record or row (also `array(string)`), a `bit(n)` wider than one bit and a type no + table maps, on both clouds, and on Glue also `geography`, `geometry`, `geom`, `point` and + `hll`, and a decimal Hive would reject (precision over 38 or 0, a scale over the precision + or below 0). The + refusal names the expose, each column, its declared type and why, and the fix (declare `json` + or `string` for now; typed nested columns are planned). `fluid validate`, `fluid generate iac` + and `fluid apply` raise the same refusal from the same function, so a contract that passes + validate cannot fail generate on a type. A BigQuery view carries no schema and is not checked. + `fluid verify` and `fluid diff` compare the live table with the new mapping, so a Glue table + applied by an earlier version reports type mismatches (verify: CRITICAL; diff: drift) on those + columns until `fluid apply` runs; verify's "require table recreation" wording is wrong for + these columns, a re-apply updates them in place. Changing a Glue column's type in place is + held by the data-loss gate: `fluid apply` (and its dry run) refuses, naming each table and + `column: old -> new`, until `--allow-data-loss`, because Parquet written under the old type + (Athena INSERT or CTAS, dbt) may stop reading until it is rewritten, and reverting the type + restores reads; a difference in spacing alone (`decimal(10, 2)`) is no change. The 0.16.2 + note that only a bare `array` has no BigQuery form is superseded: an array of any element type and every + struct-like type are refused until the contract schema can declare element and field types. + BigQuery decimal precision and scale are still not emitted. + - **A contract's governance labels were dropped silently on Databricks.** The Databricks emitter never read `labels`, so `data-classification: confidential`, the `gdpr-article`, and the sovereignty jurisdiction, regulation and residency reached AWS (tags, Glue parameters) and GCP @@ -251,6 +290,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 into `main.tf.json`. The module's allowlists now end in `\Z`, so such a name is refused with `Invalid SQL identifier: 'spectrum_events\n'`. That includes a name that reaches it with a trailing newline from an environment variable or a file; strip the value where it is set. +### Upgrade notes + +- A contract with a column typed `array`, `struct`, `map`, `object`, `record` or `row` (or + `array(...)`) on an `aws` Glue table or a `gcp` BigQuery table now fails `fluid validate`, + `fluid generate iac` and `fluid apply` with `column-type`, where it validated and wrote a + `string` column (Glue) or an invalid one (BigQuery). Declare the column `json` (BigQuery's + JSON type; Glue stores it as `string` and validate warns) or `string` until typed nested + columns are supported. +- Other new refusals (kind `column-type`): on Glue, `geography`, `geometry`, `geom`, `point` and + `hll` (previously a silent `string`; declare `string` or `binary`) and a decimal Hive rejects + (precision over 38 or 0, scale over the precision or below 0); on both clouds, `bit(n)` with + n over 1 and any type no table maps. +- New validate warnings, so `--strict` fails: on Glue, `json`, `jsonb`, `variant`, `super`, + `time`, `interval` and `bignumeric` (written as `string`, as before; declare `string` to say + so); on BigQuery, `geometry` and `geom` (planar, written as the spherical `GEOGRAPHY`) and a + `decimal(p,s)` wider than a NUMERIC holds (29 integer or 9 fractional digits; declare + `bignumeric`). +- A Glue table's `smallint`, `tinyint`, `real`, `money`, `timestamptz` and the other types that + used to become `string` now get their Hive type, which is a column type change on a table an + earlier version applied. `fluid apply` refuses it, and so does its dry run, until + `--allow-data-loss`: the refusal names each table and `column: old -> new`. Parquet files + written under the old `string` schema (by Athena INSERT or CTAS, or dbt) hold strings, so + after `string` -> `smallint` Athena fails every read of them until they are rewritten (files + forge's own DuckDB build wrote are fixed by the change). Reverting the column type in the + contract restores reads, and `--allow-data-loss` proceeds (measured against moto with real + `tofu`: refused without the flag, `~1` applied with it). A difference in spacing alone, such as + `decimal(10, 2)` and `decimal(10,2)`, never trips the gate, and neither does an added or + removed column, which the gate did not cover before. Consumers that compare these columns as + strings (`small = '5'`) may need changes. Until the apply, `fluid verify` and `fluid diff` + report the mismatch. With the plan's change list unreadable, an AWS apply that updates + anything in place now also needs the flag, as a Databricks apply already did. +- `decimal(10, 2)` is now written `decimal(10,2)`. Against moto, that alone applies without the + flag; real Glue was not measured. - **Other validators let a value with a trailing newline through, too.** The same `$` ended other whole-string patterns, so a value written as a YAML `|` block, read from a `.env` file or decoded diff --git a/fluid_build/cli/_apply_opentofu_engine.py b/fluid_build/cli/_apply_opentofu_engine.py index d20b70a50..4ee13473a 100644 --- a/fluid_build/cli/_apply_opentofu_engine.py +++ b/fluid_build/cli/_apply_opentofu_engine.py @@ -34,7 +34,7 @@ from pathlib import Path from typing import Any, Dict, List, Mapping, Optional, Tuple -from fluid_build.cli.console import cprint +from fluid_build.cli.console import cprint, cprint_literal from fluid_build.iac import build_module, get_iac_plugin, runner from fluid_build.iac.backend import ( STATE_BACKEND_ENV, @@ -1074,9 +1074,11 @@ def _enforce_data_loss_gate( changes["remove"], ) if lost_in_place: - cprint( + # The lines quote live objects (a table's column types): data, not markup. + cprint_literal( "\n ⚠️ --allow-data-loss: existing data is changed in place with no snapshot: " - + "; ".join(lost_in_place) + + "; ".join(lost_in_place), + soft_wrap=True, ) logger.warning( "--allow-data-loss: bypassing the data-loss gate for in-place data changes " @@ -1470,12 +1472,11 @@ def _report_plan_notices(plugin: Any, contract: Mapping[str, Any], logger: loggi if not callable(notices): return try: - lines = list(notices(contract)) + for line in notices(contract): + # A notice can quote live objects, so it is data, not markup. + cprint_literal(f" note: {line}") except Exception as exc: # noqa: BLE001 — advisory only, never fails an apply logger.debug("plan notices unavailable: %s", type(exc).__name__) - return - for line in lines: - cprint(f" note: {line}") def _reconcile_with_state( diff --git a/fluid_build/iac/column_types.py b/fluid_build/iac/column_types.py new file mode 100644 index 000000000..d4ee94fce --- /dev/null +++ b/fluid_build/iac/column_types.py @@ -0,0 +1,153 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""What a column type becomes on a cloud table, and what that costs. + +The AWS (Glue) and GCP (BigQuery) emitters each translate a contract's column +types into their own. A type with no faithful target used to be written anyway: +Glue turned 47 of the 79 types the schema admits into ``string``, and BigQuery +emitted ``ARRAY`` (no such column type: an array is a REPEATED column of an +element type) or a ``STRUCT`` with no fields, which it rejects at apply. Neither +said so, and ``fluid validate --strict`` passed. + +Each emitter now classifies a type as one of: + +* ``exact``: the target type holds every value the declared type does. +* ``lossy``: the target has no such type, a fallback is written (``json`` becomes + ``string`` on Glue) and ``fluid validate`` warns, so ``--strict`` fails. +* ``refused``: there is no form worth writing; validate, ``generate iac`` and + ``apply`` all refuse with an :class:`UnsupportedBindingError` of kind + ``column-type``. + +The classification lives with each emitter's type table (``_hive_outcome`` in +``providers/aws.py``, ``_bq_outcome`` in ``providers/gcp.py``). This module holds +what both share, and :func:`column_type_findings` is the one function the emitter +and the validate-time gate both call, so they cannot disagree. +""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, Iterable, List, Mapping, NamedTuple, Optional, Tuple + +from .base import UnsupportedBindingError + +EXACT = "exact" +LOSSY = "lossy" +REFUSED = "refused" + + +class TypeOutcome(NamedTuple): + """One declared type's translation: ``target`` is ``None`` when refused.""" + + target: Optional[str] + status: str + why: str = "" + fix: str = "" + + +#: Types that carry element or field types, which the contract schema cannot +#: declare yet: ``$defs.column`` is closed, and ``array(string)`` is only a type +#: pattern. No cloud table takes them without that detail. +STRUCTURED = frozenset({"array", "struct", "map", "object", "record", "row"}) + +#: The fix for a refused structured column, whichever cloud refused it. +NESTED_FIX = "Declare json or string for now; typed nested columns are planned." + + +def expand(groups: Mapping[str, str]) -> Dict[str, str]: + """``{"INT64": "int, bigint"}`` as ``{"int": "INT64", "bigint": "INT64"}``: a type + table written one target at a time, read one declared type at a time.""" + return {name.strip(): target for target, names in groups.items() for name in names.split(",")} + + +def exact(target: str) -> TypeOutcome: + return TypeOutcome(target, EXACT) + + +def lossy(target: str, why: str, fix: str) -> TypeOutcome: + return TypeOutcome(target, LOSSY, why, fix) + + +def refused(why: str, fix: str) -> TypeOutcome: + return TypeOutcome(None, REFUSED, why, fix) + + +def split_type(raw: Any) -> Tuple[str, str]: + """``("decimal", "(10,2)")`` for ``"Decimal (10,2)"``; ``("array", "(string)")`` + for ``"ARRAY(string)"``. The base is lower-cased with its words single-spaced + (``timestamp with time zone``); the rest is whatever follows it, spaces dropped. + A missing type is ``string``, as the emitters have always read it.""" + text = " ".join(str(raw if raw is not None else "string").strip().lower().split()) + if not text: + text = "string" + cuts = [i for i in (text.find("("), text.find("<")) if i >= 0] + cut = min(cuts) if cuts else len(text) + return text[:cut].strip(), text[cut:].replace(" ", "") + + +def exposes_of(contract: Any) -> List[Any]: + """The contract's ``exposes`` when it is a list, else none: a schema-invalid contract + is the schema check's to report, and the type gate skips what it cannot read.""" + exposes = contract.get("exposes") if isinstance(contract, Mapping) else None + return exposes if isinstance(exposes, list) else [] + + +def schema_of(exposure: Mapping[str, Any]) -> List[Any]: + """The expose's ``contract.schema`` columns when they are a list, else none.""" + section = exposure.get("contract") + schema = section.get("schema") if isinstance(section, Mapping) else None + return schema if isinstance(schema, list) else [] + + +def column_type_findings( + expose: str, + schema: Iterable[Any], + outcome_for: Callable[[Any], TypeOutcome], + target: str, +) -> Tuple[Optional[UnsupportedBindingError], List[str]]: + """``(refusal, warnings)`` for one expose's columns on ``target`` (a table kind). + + One refusal covers every column the target cannot hold, so the author fixes + them in one pass; each lossy column is its own warning. + """ + parts: List[str] = [] + fixes: Dict[str, List[str]] = {} + warnings: List[str] = [] + for column in schema or []: + if not isinstance(column, Mapping): + continue + name = column.get("name") + declared = str(column.get("type") if column.get("type") is not None else "string") + outcome = outcome_for(column.get("type")) + if outcome.status == REFUSED: + parts.append(f"{name!r} is declared {declared!r}, {outcome.why}") + fixes.setdefault(outcome.fix, []).append(repr(name)) + elif outcome.status == LOSSY: + warnings.append( + f"expose '{expose}': column {name!r} is declared {declared!r}, {outcome.why}; " + f"it is written as {outcome.target!r}. {outcome.fix}" + ) + if not parts: + return None, warnings + refusal = UnsupportedBindingError( + "column-type", + f"expose '{expose}': {target} cannot hold its columns as declared: " + + "; ".join(parts) + + ".", + tuple( + f"{'Column' if len(names) == 1 else 'Columns'} {', '.join(names)}: {fix}" + for fix, names in fixes.items() + ), + ) + return refusal, warnings diff --git a/fluid_build/iac/governance_validation.py b/fluid_build/iac/governance_validation.py index 322c437f8..0bd0ca790 100644 --- a/fluid_build/iac/governance_validation.py +++ b/fluid_build/iac/governance_validation.py @@ -118,9 +118,15 @@ def validate_governance(contract: Mapping[str, Any]) -> Tuple[List[str], List[st """ from .providers import gcp as _gcp from .providers import gcp_governance as _gov + from .providers.aws import validate_aws_column_types errors: List[str] = [] warnings: List[str] = [] + # Column types a Glue table cannot hold: refused by the emitter, written as + # ``string`` with a warning here, or exact. + type_errors, type_warnings = validate_aws_column_types(contract) + errors.extend(type_errors) + warnings.extend(type_warnings) grants = normalize_access_grants(contract) unenforced: List[str] = [] # AWS Iceberg tables in another catalog: Lake Formation cannot reach them, diff --git a/fluid_build/iac/providers/aws.py b/fluid_build/iac/providers/aws.py index d9c04dfc1..c27cf2a52 100644 --- a/fluid_build/iac/providers/aws.py +++ b/fluid_build/iac/providers/aws.py @@ -79,7 +79,7 @@ ) from ...providers._sql_safety import quote_string_literal, validate_ident from ...providers.aws.util import warehouse as _warehouse -from .. import column_access +from .. import column_access, column_types from ..base import UnsupportedBindingError from ..governance_labels import governance_labels from ..importer import ImportBlock @@ -195,37 +195,230 @@ def _resolve_catalog_id() -> str: _CATALOG_ID_CACHE: Dict[str, str] = {} -# FLUID column type → Hive/Glue column type. -_HIVE_TYPES = { - "string": "string", - "str": "string", - "text": "string", - "integer": "int", - "int": "int", - "int32": "int", - "bigint": "bigint", - "int64": "bigint", - "long": "bigint", - "float": "float", - "float32": "float", - "double": "double", - "float64": "double", - "boolean": "boolean", - "bool": "boolean", - "date": "date", - "timestamp": "timestamp", - "datetime": "timestamp", - "binary": "binary", - "bytes": "binary", +# FLUID column type → Hive/Glue column type, keyed on the lower-cased base name +# (``varchar(255)`` is ``varchar``). Only a type Hive holds without loss is here; +# ``_hive_outcome`` classifies every other one. +# +# Where the choice is not obvious: Hive has no zone-aware timestamp, so the +# ``timestamptz`` family is ``timestamp``, which stores the UTC instant (the +# offset is not kept); ``money`` is SQL Server's ``decimal(19,4)``; ``year`` is +# MySQL's, a four-digit number, so ``smallint``; ``bit`` is the SQL Server +# 0/1 flag. Text, UUIDs and enums are ``string``: Hive has no better form, and +# every engine reading the table treats them as text. +_HIVE_TYPES = column_types.expand( + { + "string": "string, str, text, varchar, varchar2, nvarchar, char, nchar, character, clob, " + "uuid, uniqueidentifier, guid, enum", + "tinyint": "tinyint", + "smallint": "smallint, int2, int16, year", + "int": "int, integer, int4, int32, mediumint, serial", + "bigint": "bigint, int8, int64, long, longint, bigserial", + "float": "float, float4, float32, real", + "double": "double, double precision, float8, float64", + "boolean": "boolean, bool, bit", + "date": "date", + "timestamp": "timestamp, datetime, datetime2, smalldatetime, timestamptz, timestamp_tz, " + "timestamp_ltz, timestamp_ntz, timestampntz, timestamp with time zone, " + "timestamp without time zone", + "binary": "binary, bytes, varbinary, blob, bytea, raw", + "decimal(19,4)": "money", + } +) + +#: Types Hive has no form for but that keep their value as text. Written as +#: ``string``, and ``fluid validate`` warns. +_HIVE_AS_STRING = { + "json": "which Hive has no json type for", + "jsonb": "which Hive has no jsonb type for", + "variant": "which Hive has no variant type for", + "super": "which Hive has no super type for", + "time": "which Hive has no time-of-day type for", + "interval": "which Hive has no interval column type for", + "bignumeric": "whose digits beyond 38 no Hive decimal holds", } +#: Types with no Hive form and no fallback worth writing: which encoding a spatial +#: value or a sketch takes is the writer's, and a guess would be wrong half the time. +_HIVE_REFUSED = { + **{ + t: ( + "which has no Hive column type (a spatial value is stored as WKT text or WKB bytes, " + "and only the writer knows which)", + "Declare string (WKT) or binary (WKB), whichever the data holds.", + ) + for t in ("geography", "geometry", "geom", "point") + }, + "hll": ( + "which is an engine-specific sketch with no Hive column type", + "Declare binary to store the sketch's bytes.", + ), +} + +_DECIMAL_FAMILY = frozenset({"decimal", "numeric", "dec", "number"}) +# ASCII digits only: ``\d`` alone also matches Arabic-Indic and other Unicode digits. +_DECIMAL_ARGS_RE = re.compile(r"^\((\d+)(?:,(\d+))?\)$", re.ASCII) +_HIVE_DECIMAL_DEFAULT = "decimal(38,9)" +_HIVE_TYPE_FIX = ( + "Declare a FLUID column type such as string, integer, bigint, decimal(12,2), double, " + "boolean, date or timestamp." +) + + +def _hive_decimal(rest: str) -> column_types.TypeOutcome: + """``decimal(p,s)`` passes through as written; a bare one widens to a safe default.""" + if not rest: + return column_types.exact(_HIVE_DECIMAL_DEFAULT) + match = _DECIMAL_ARGS_RE.match(rest) + if match is not None: + precision, scale = int(match.group(1)), int(match.group(2) or 0) + if 1 <= precision <= 38 and scale <= precision: + return column_types.exact(f"decimal{rest}") + return column_types.refused( + f"whose precision and scale are not a Hive decimal's ({rest})", + "Declare decimal(p,s) with p from 1 to 38 and s no larger than p, or string.", + ) + + +def _hive_outcome(raw: Any) -> column_types.TypeOutcome: + """What ``raw`` becomes in a Glue table: its Hive type, exact, lossy or refused.""" + base, rest = column_types.split_type(raw) + if base in column_types.STRUCTURED: + return column_types.refused( + "which Glue holds only with its element or field types, and the contract schema " + "cannot declare them yet", + column_types.NESTED_FIX, + ) + if not rest.isascii(): + return column_types.refused( + f"whose parameters ({rest}) are not ASCII", "Declare the type with ASCII digits." + ) + if base in _DECIMAL_FAMILY: + return _hive_decimal(rest) + if base == "bit" and rest not in ("", "(1)"): + return column_types.refused( + "which is a bit string, and Hive has none", "Declare binary or an integer type." + ) + if base in _HIVE_REFUSED: + return column_types.refused(*_HIVE_REFUSED[base]) + if base in _HIVE_AS_STRING: + return column_types.lossy( + "string", + _HIVE_AS_STRING[base], + "Declare string to store the text on purpose; this warning then goes away.", + ) + hit = _HIVE_TYPES.get(base) + if hit is None: + return column_types.refused("which the Glue emitter does not map", _HIVE_TYPE_FIX) + if base == "float" and rest: + # ``float(53)`` is double precision, as in PostgreSQL and SQL Server. + digits = rest.strip("()") + if digits.isascii() and digits.isdigit() and int(digits) > 24: + return column_types.exact("double") + return column_types.exact(hit) + def _hive_type(raw: Any) -> str: - t = str(raw or "string").strip().lower() - if t.startswith(("decimal", "numeric")): - # decimal(10,2) passes through; a bare type widens to a safe default. - return t.replace("numeric", "decimal") if "(" in t else "decimal(38,9)" - return _HIVE_TYPES.get(t, "string") + """The Hive type ``raw`` is written as. Total, for ``fluid verify`` and ``fluid diff`` + to compare a live table with: a refused type reads as ``string`` here, and is + refused where the module is emitted (:func:`_refuse_glue_column_types`).""" + return _hive_outcome(raw).target or "string" + + +def _glue_column_type_findings( + expose: str, schema: Iterable[Any] +) -> Tuple[Optional[UnsupportedBindingError], List[str]]: + return column_types.column_type_findings(expose, schema, _hive_outcome, "AWS Glue") + + +def _glue_table_declared(binding: Mapping[str, Any]) -> bool: + """Does :func:`_emit_glue` write a table for ``binding``? Its three gates, for the + validate-time check to apply the column types where the emitter does.""" + loc = binding.get("location") + if not isinstance(loc, Mapping): + return False + fmt = binding.get("format") or "parquet" + return bool(loc.get("database") and loc.get("table") and _glue_cataloged(binding, fmt)) + + +def validate_aws_column_types(contract: Mapping[str, Any]) -> Tuple[List[str], List[str]]: + """``(errors, warnings)`` for the column types of the contract's Glue tables: the + refusal :func:`_emit_glue` raises, and a warning for each type it writes as a + fallback. ``fluid validate`` runs this so ``generate iac`` finds nothing new.""" + errors: List[str] = [] + warnings: List[str] = [] + for index, exposure in enumerate(column_types.exposes_of(contract)): + binding = exposure.get("binding") if isinstance(exposure, Mapping) else None + if not isinstance(binding, Mapping) or not is_cloud(binding, "aws"): + continue + if not _glue_table_declared(binding): + continue + schema = column_types.schema_of(exposure) + refusal, lossy = _glue_column_type_findings(_expose_id(exposure) or str(index), schema) + if refusal is not None: + errors.append(f"{refusal} {' '.join(refusal.remediation)}".strip()) + warnings.extend(lossy) + return errors, warnings + + +def _plan_glue_columns(state: Any) -> Dict[str, str]: + """``{column: type}`` of a planned ``aws_glue_catalog_table`` (``tofu show -json`` + renders its one ``storage_descriptor`` block as a one-item list).""" + if not isinstance(state, Mapping): + return {} + descriptor = state.get("storage_descriptor") + if isinstance(descriptor, list): + descriptor = descriptor[0] if descriptor else None + if not isinstance(descriptor, Mapping): + return {} + return { + str(c.get("name")): str(c.get("type")) + for c in descriptor.get("columns") or [] + if isinstance(c, Mapping) + } + + +def glue_column_type_changes(plan: Mapping[str, Any]) -> List[str]: + """One line per Glue table the plan updates in place with other column types. + + A column type change plans as a plain ``~`` and leaves the data files alone, but + Parquet written under the old type (Athena INSERT or CTAS, dbt: ``string``) holds + values the new type cannot read, and Athena then fails every read of those files. + So each line is data the apply puts at risk, and the engine's data-loss gate + (``plan_data_changes``) holds the apply until ``--allow-data-loss``. A difference + in spacing alone (``decimal(10, 2)`` against ``decimal(10,2)``) is no change. + Created and removed tables, and updates that keep every column's type, give none. + """ + lines: List[str] = [] + for change in plan.get("resource_changes") or []: + if not isinstance(change, Mapping) or change.get("type") != "aws_glue_catalog_table": + continue + detail = change.get("change") + if not isinstance(detail, Mapping) or list(detail.get("actions") or []) != ["update"]: + continue + before, after = detail.get("before"), detail.get("after") + old, new = _plan_glue_columns(before), _plan_glue_columns(after) + changed = [ + f"{name}: {old[name]} -> {new[name]}" + for name in old + if name in new and _unspaced(old[name]) != _unspaced(new[name]) + ] + if not changed: + continue + table = ".".join( + str(before.get(key) or "") + for key in ("database_name", "name") + if isinstance(before, Mapping) + ) + lines.append( + f"Glue table {table}: column types change in place ({'; '.join(changed)}); files " + "written under the old type may stop reading until they are rewritten, and " + "reverting the column type restores reads" + ) + return lines + + +def _unspaced(glue_type: str) -> str: + return "".join(glue_type.split()) def _columns(schema: List[Mapping[str, Any]]) -> List[Dict[str, str]]: @@ -457,6 +650,7 @@ def emit( contract=contract, placement=placement, labels=gov_labels, + expose=_expose_id(exposure) or str(index), ) _emit_s3(resources, loc, cid, tags, placement=placement) _emit_kinesis(resources, loc, cid, tags) @@ -721,6 +915,11 @@ def recorded_regions(self, resources: Iterable[Mapping[str, Any]]) -> set: """The regions existing state places resources in; see :func:`recorded_regions`.""" return recorded_regions(resources) + def plan_data_changes(self, plan: Mapping[str, Any]) -> List[str]: + """The Glue tables the saved plan gives other column types in place; each needs + ``--allow-data-loss``. See :func:`glue_column_type_changes`.""" + return glue_column_type_changes(plan) + def provider_block_for(self, contract: Mapping[str, Any]) -> Dict[str, Any]: """``provider_block``, plus the region the contract's AWS bindings name. @@ -943,6 +1142,7 @@ def _emit_glue( contract: Optional[Mapping[str, Any]] = None, placement: _Placement = _LEGACY_PLACEMENT, labels: Optional[Mapping[str, str]] = None, + expose: str = "?", ) -> None: database = loc.get("database") if not database: @@ -978,6 +1178,11 @@ def _emit_glue( # had write-once-on-create semantics and never reconciled the # parameters again. Mirrors how the Glue Terraform Registry # examples model catalog metadata + descriptions in one resource. + # A column Glue cannot hold is refused, not written as ``string``: the same + # findings ``fluid validate`` reports (:func:`validate_aws_column_types`). + refusal, _lossy = _glue_column_type_findings(expose, schema) + if refusal is not None: + raise refusal storage: Dict[str, Any] = {"columns": _columns(schema)} # Athena reads a Glue table through the Hive input format and SerDe the # table declares. Without them every query failed with HIVE_UNSUPPORTED_FORMAT diff --git a/fluid_build/iac/providers/gcp.py b/fluid_build/iac/providers/gcp.py index 197088c49..5334eebd7 100644 --- a/fluid_build/iac/providers/gcp.py +++ b/fluid_build/iac/providers/gcp.py @@ -42,9 +42,11 @@ import hashlib import json +import re from dataclasses import dataclass from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple +from .. import column_types from ..access import ( GROUP, AccessGrant, @@ -52,6 +54,7 @@ normalize_access_grants, role_grants, ) +from ..base import UnsupportedBindingError from ..importer import ImportBlock from ..naming import TofuExpr, safe_ident, tofu_ref from ..packaging import ( @@ -65,28 +68,44 @@ from ..versions import required_providers from . import gcp_governance as _gov -# FLUID column type → BigQuery type (best-effort; unknown types upper-cased). -_BQ_TYPES = { - "string": "STRING", - "str": "STRING", - "text": "STRING", - "integer": "INT64", - "int": "INT64", - "int64": "INT64", - "bigint": "INT64", - "float": "FLOAT64", - "float64": "FLOAT64", - "double": "FLOAT64", - "numeric": "NUMERIC", - "decimal": "NUMERIC", - "boolean": "BOOL", - "bool": "BOOL", - "timestamp": "TIMESTAMP", - "datetime": "DATETIME", - "date": "DATE", - "time": "TIME", - "bytes": "BYTES", - "json": "JSON", +# FLUID column type → BigQuery type, keyed on the lower-cased base name +# (``varchar(255)`` is ``varchar``). Every entry is a type a BigQuery table schema +# takes; ``_bq_outcome`` classifies the rest. A ``number`` or ``decimal`` is NUMERIC +# whatever precision it declares (precision and scale are not emitted yet), and +# ``_bq_outcome`` warns when the declared range is wider than a NUMERIC holds. +_BQ_TYPES = column_types.expand( + { + "STRING": "string, str, text, varchar, varchar2, nvarchar, char, nchar, character, clob, " + "uuid, uniqueidentifier, guid, enum", + "INT64": "integer, int, int2, int4, int8, int16, int32, int64, tinyint, smallint, " + "mediumint, bigint, long, longint, serial, bigserial, year", + "FLOAT64": "float, float4, float8, float32, float64, double, real", + "NUMERIC": "numeric, decimal, dec, number, money", + "BIGNUMERIC": "bignumeric", + "BOOL": "boolean, bool, bit", + "TIMESTAMP": "timestamp, timestamptz, timestamp_tz, timestamp_ltz", + "DATETIME": "datetime, datetime2, smalldatetime, timestamp_ntz, timestampntz", + "DATE": "date", + "TIME": "time", + "INTERVAL": "interval", + "BYTES": "bytes, binary, varbinary, blob, bytea, raw, hll", + "JSON": "json, jsonb, variant, super", + "GEOGRAPHY": "geography, geometry, geom, point", + } +) + +#: What a BigQuery NUMERIC holds, and the types that take a precision and scale: the +#: scale is not emitted, so a declared range beyond it overflows at load. +_BQ_NUMERIC_INTEGER_DIGITS = 29 +_BQ_NUMERIC_SCALE = 9 +_BQ_DECIMAL_FAMILY = frozenset({"decimal", "numeric", "dec", "number"}) +_BQ_DECIMAL_ARGS_RE = re.compile(r"^\((\d+)(?:,(\d+))?\)$", re.ASCII) + +#: Types whose BigQuery type holds a different thing. ``GEOMETRY`` is planar and +#: ``GEOGRAPHY`` is a sphere, so the same coordinates mean other shapes and distances. +_BQ_LOSSY = { + "geometry": "which is planar, and BigQuery has only the spherical GEOGRAPHY", + "geom": "which is planar, and BigQuery has only the spherical GEOGRAPHY", } # FLUID permission → BigQuery dataset access role. BigQuery dataset @@ -136,24 +155,94 @@ } -def _bq_type(raw: Any) -> str: - base = " ".join(str(raw or "STRING").strip().lower().split("(", 1)[0].split()) +def _bq_outcome(raw: Any) -> column_types.TypeOutcome: + """What ``raw`` becomes in a BigQuery table schema: exact, lossy or refused.""" + base, rest = column_types.split_type(raw) + if base in column_types.STRUCTURED: + return column_types.refused( + "which BigQuery holds only as a REPEATED column of an element type or a RECORD " + "with fields, and the contract schema cannot declare them yet", + column_types.NESTED_FIX, + ) + if not rest.isascii(): + return column_types.refused( + f"whose parameters ({rest}) are not ASCII", "Declare the type with ASCII digits." + ) + if base == "bit" and rest not in ("", "(1)"): + return column_types.refused( + "which is a bit string, and BigQuery has none", + "Declare bytes or an integer type.", + ) hit = _BQ_TYPES.get(base) or _BQ_MULTIWORD_TYPES.get(base) - if hit is not None: - return hit - # A source-native spelling (VARCHAR, SMALLINT, UUID, TIMESTAMPTZ ...) used to - # be upper-cased verbatim, which BigQuery rejects: a contract read from - # Postgres emitted ``"type": "VARCHAR"``. The ODCS physical-type table covers - # the FLUID column-type enum and agrees with every entry above, so this only - # changes the types that were invalid before. - from ...providers.odcs.mappers.types import fluid_to_physical - - physical = fluid_to_physical(base, "gcp") - if physical: - return physical + if hit is None: + return column_types.refused( + "which the BigQuery emitter does not map", + "Declare a FLUID column type such as string, integer, bigint, numeric, double, " + "boolean, date, timestamp or json.", + ) + if base in _BQ_DECIMAL_FAMILY and rest: + match = _BQ_DECIMAL_ARGS_RE.match(rest) + if match is not None: + precision, scale = int(match.group(1)), int(match.group(2) or 0) + if scale > _BQ_NUMERIC_SCALE or precision - scale > _BQ_NUMERIC_INTEGER_DIGITS: + return column_types.lossy( + hit, + f"whose range a BigQuery NUMERIC ({_BQ_NUMERIC_INTEGER_DIGITS} integer and " + f"{_BQ_NUMERIC_SCALE} fractional digits) cannot hold", + "Declare bignumeric if the values need that range.", + ) + if base in _BQ_LOSSY: + return column_types.lossy( + hit, _BQ_LOSSY[base], "Declare geography if the coordinates are on the sphere." + ) + return column_types.exact(hit) + + +def _bq_type(raw: Any) -> str: + """The BigQuery type ``raw`` is written as. Total, for ``fluid verify``, ``fluid diff`` + and the Dataform engine: a refused type reads as its upper-cased name here, and is + refused where the module is emitted (:func:`_refuse_bq_column_types`).""" + outcome = _bq_outcome(raw) + if outcome.target is not None: + return outcome.target return str(raw).upper() if raw else "STRING" +def _bq_column_type_findings( + expose: str, schema: Iterable[Any] +) -> Tuple[Optional[UnsupportedBindingError], List[str]]: + return column_types.column_type_findings(expose, schema, _bq_outcome, "BigQuery") + + +def _refuse_bq_column_types(expose: str, schema: Iterable[Any]) -> None: + """Raise for a column BigQuery cannot hold, which it would reject at apply, after + the dataset was created, or (an array) take without its element type.""" + refusal, _lossy = _bq_column_type_findings(expose, schema) + if refusal is not None: + raise refusal + + +def validate_bigquery_column_types(contract: Mapping[str, Any]) -> Tuple[List[str], List[str]]: + """``(errors, warnings)`` for the column types of the contract's BigQuery tables: + the refusal :func:`_emit_bigquery` raises, and a warning for each type it writes as + a different one. A view carries no schema. ``fluid validate`` runs this, so + ``generate iac`` finds nothing new.""" + errors: List[str] = [] + warnings: List[str] = [] + for exposure in column_types.exposes_of(contract): + binding = exposure.get("binding") if isinstance(exposure, Mapping) else None + if not isinstance(binding, Mapping) or not _gov.gcp_owned(binding): + continue + if resolve_gcp_target(binding) != BIGQUERY_TABLE: + continue + schema = column_types.schema_of(exposure) + refusal, lossy = _bq_column_type_findings(_expose_id(exposure) or "?", schema) + if refusal is not None: + errors.append(f"{refusal} {' '.join(refusal.remediation)}".strip()) + warnings.extend(lossy) + return errors, warnings + + def _bq_fields(schema: List[Mapping[str, Any]]) -> List[Dict[str, Any]]: """FLUID contract schema → BigQuery schema fields.""" return [ @@ -1167,6 +1256,10 @@ def _emit_bigquery( placement: _Placement = _LEGACY_PLACEMENT, governance: Optional[_BqGovernance] = None, ) -> None: + if not is_view: + # Refused before anything is written: the same findings ``fluid validate`` + # reports (:func:`validate_bigquery_column_types`). + _refuse_bq_column_types(_expose_id(exposure) or "?", schema) gov = governance or _NO_GOVERNANCE dataset = loc.get("dataset") or "default" table = _bq_table_name(exposure, loc) @@ -1455,7 +1548,8 @@ def validate_gcp_binding(contract: Mapping[str, Any]) -> Tuple[List[str], List[s binding (no platform) is a different contract error, and the JSON-schema check already names it. Iceberg exposes are left to ``iac.iceberg_validation``, which owns a more specific message for the - same input. Returns ``(errors, warnings)``. + same input. It also reports the column types of each BigQuery table + (:func:`validate_bigquery_column_types`). Returns ``(errors, warnings)``. """ errors: List[str] = [] warnings: List[str] = [] @@ -1496,7 +1590,10 @@ def validate_gcp_binding(contract: Mapping[str, Any]) -> Tuple[List[str], List[s errors.append(message) else: warnings.append(message) - return errors, warnings + # The column types of each BigQuery table: what the emitter refuses, or writes + # as another type. + type_errors, type_warnings = validate_bigquery_column_types(contract) + return errors + type_errors, warnings + type_warnings def _bq_table_name(exposure: Mapping[str, Any], loc: Mapping[str, Any]) -> str: diff --git a/tests/iac/test_iac_column_type_gate.py b/tests/iac/test_iac_column_type_gate.py new file mode 100644 index 000000000..3a1831ae4 --- /dev/null +++ b/tests/iac/test_iac_column_type_gate.py @@ -0,0 +1,540 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""No column type reaches AWS Glue or BigQuery silently degraded or invalid. + +Every type the bundled schemas admit is classified on both clouds: written as a +valid target type, written as a fallback and warned about, or refused. The +refusal is the same one at ``fluid validate``, ``generate iac`` and ``apply``, +and ``validate`` agrees with the emitter on which contracts pass. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import re +from pathlib import Path + +import pytest +import yaml + +import fluid_build +from fluid_build.iac import get_iac_plugin +from fluid_build.iac.base import UnsupportedBindingError +from fluid_build.iac.column_types import EXACT, LOSSY, REFUSED +from fluid_build.iac.governance_validation import validate_governance +from fluid_build.iac.providers.aws import _hive_outcome, _hive_type, validate_aws_column_types +from fluid_build.iac.providers.gcp import ( + _bq_outcome, + validate_bigquery_column_types, + validate_gcp_binding, +) + +pytestmark = [pytest.mark.unit, pytest.mark.provider] + +_SCHEMAS = Path(fluid_build.__file__).parent / "schemas" + +# Every type the BigQuery schema JSON accepts, standard and legacy spellings. +_BQ_VALID = { + "STRING", "BYTES", "INTEGER", "INT64", "FLOAT", "FLOAT64", "NUMERIC", "BIGNUMERIC", + "BOOLEAN", "BOOL", "TIMESTAMP", "DATE", "TIME", "DATETIME", "GEOGRAPHY", "JSON", + "RECORD", "STRUCT", "INTERVAL", "RANGE", +} # fmt: skip +# A Hive/Athena column type this emitter writes. +_HIVE_VALID = re.compile( + r"^(string|tinyint|smallint|int|bigint|float|double|boolean|date|timestamp|binary" + r"|decimal\((?:[1-9]|[12][0-9]|3[0-8]),\d+\))$" +) + + +def _enum(version: str) -> set: + schema = json.loads((_SCHEMAS / f"fluid-schema-{version}.json").read_text(encoding="utf-8")) + return set(schema["$defs"]["column"]["properties"]["type"]["anyOf"][0]["enum"]) + + +_ALL_TYPES = sorted(_enum("0.7.6") | _enum("0.7.7")) + +_STRUCTURED = {"array", "struct", "map", "object", "record", "row"} +_GLUE_LOSSY = {"json", "jsonb", "variant", "super", "time", "interval", "bignumeric"} +_GLUE_REFUSED = _STRUCTURED | {"geography", "geometry", "geom", "point", "hll"} +_BQ_LOSSY = {"geometry", "geom"} +_BQ_REFUSED = _STRUCTURED + + +def test_the_enums_are_the_79_types_this_gate_was_written_against(): + # A new enum member lands in the parametrised tests below and has to be + # classified; this only notes the size so a change is deliberate. + assert len(_ALL_TYPES) == 79 + + +@pytest.mark.parametrize("fluid_type", _ALL_TYPES) +def test_every_schema_type_is_valid_warned_or_refused_on_glue(fluid_type): + outcome = _hive_outcome(fluid_type) + if outcome.status == REFUSED: + assert outcome.target is None + assert outcome.why and outcome.fix + else: + assert _HIVE_VALID.match(outcome.target), (fluid_type, outcome) + assert (outcome.status == LOSSY) == (fluid_type in _GLUE_LOSSY) + assert (outcome.status == REFUSED) == (fluid_type in _GLUE_REFUSED) + + +@pytest.mark.parametrize("fluid_type", _ALL_TYPES) +def test_every_schema_type_is_valid_warned_or_refused_on_bigquery(fluid_type): + outcome = _bq_outcome(fluid_type) + if outcome.status == REFUSED: + assert outcome.target is None + assert outcome.why and outcome.fix + else: + assert outcome.target in _BQ_VALID, (fluid_type, outcome) + assert (outcome.status == LOSSY) == (fluid_type in _BQ_LOSSY) + assert (outcome.status == REFUSED) == (fluid_type in _BQ_REFUSED) + + +@pytest.mark.parametrize( + "fluid_type, expected", + [ + ("int2", "smallint"), + ("int4", "int"), + ("int8", "bigint"), + ("int16", "smallint"), + ("tinyint", "tinyint"), + ("smallint", "smallint"), + ("mediumint", "int"), + ("longint", "bigint"), + ("serial", "int"), + ("bigserial", "bigint"), + ("float4", "float"), + ("float8", "double"), + ("real", "float"), + ("dec", "decimal(38,9)"), + ("number", "decimal(38,9)"), + ("number(12,2)", "decimal(12,2)"), + ("dec(10)", "decimal(10)"), + ("money", "decimal(19,4)"), + ("bit", "boolean"), + ("datetime2", "timestamp"), + ("smalldatetime", "timestamp"), + ("timestamptz", "timestamp"), + ("timestamp_tz", "timestamp"), + ("timestamp_ltz", "timestamp"), + ("timestamp_ntz", "timestamp"), + ("timestampntz", "timestamp"), + ("TIMESTAMP WITH TIME ZONE", "timestamp"), + ("double precision", "double"), + ("varbinary", "binary"), + ("blob", "binary"), + ("bytea", "binary"), + ("raw", "binary"), + ("year", "smallint"), + ("varchar(255)", "string"), + ("float(53)", "double"), + ("float(24)", "float"), + ("decimal(10, 2)", "decimal(10,2)"), + ], +) +def test_a_scalar_with_a_hive_form_gets_it(fluid_type, expected): + outcome = _hive_outcome(fluid_type) + assert (outcome.target, outcome.status) == (expected, EXACT) + assert _hive_type(fluid_type) == expected + + +@pytest.mark.parametrize("fluid_type", ["json", "jsonb", "variant", "super", "time", "interval"]) +def test_a_type_hive_cannot_hold_is_written_as_string_and_warned(fluid_type): + outcome = _hive_outcome(fluid_type) + assert (outcome.target, outcome.status) == ("string", LOSSY) + + +@pytest.mark.parametrize( + "fluid_type", ["decimal(39,2)", "decimal(10,11)", "numeric(0)", "decimal(a,b)", "bit(8)"] +) +def test_a_type_hive_would_reject_is_refused(fluid_type): + assert _hive_outcome(fluid_type).status == REFUSED + + +@pytest.mark.parametrize( + "fluid_type", ["array", "array(string)", "ARRAY", "struct", "map(string,int)", "row(a)"] +) +def test_a_structured_type_is_refused_whatever_its_parameters(fluid_type): + assert _hive_outcome(fluid_type).status == REFUSED + assert _bq_outcome(fluid_type).status == REFUSED + + +def test_a_type_nothing_maps_is_refused_on_both_clouds(): + assert _hive_outcome("geometryz").status == REFUSED + assert _bq_outcome("geometryz").status == REFUSED + + +def test_the_lenient_views_stay_total_for_verify_and_diff(): + """``fluid verify`` and ``fluid diff`` compare with these; they never raise.""" + from fluid_build.iac.providers.gcp import _bq_type + + assert _hive_type("array") == "string" + assert _bq_type("array") == "ARRAY" + assert _bq_type("geometryz") == "GEOMETRYZ" + assert _bq_type(None) == "STRING" + + +# ── The contracts: validate, generate and apply agree ──────────────────────── + +_AWS_LOC = { + "database": "demo", + "table": "orders", + "bucket": "acme-demo", + "path": "orders/", + "region": "us-east-1", +} +_GCP_LOC = {"project": "acme", "dataset": "demo", "table": "orders"} + +_BAD = [ + ("id", "string"), + ("tags", "array"), + ("payload", "json"), + ("small", "smallint"), + ("tz", "timestamptz"), + ("addr", "struct"), +] +_GOOD = [c for c in _BAD if c[1] not in ("array", "struct")] + + +def _contract(platform, columns, binding=None): + fmt, loc = ("parquet", _AWS_LOC) if platform == "aws" else ("bigquery_table", _GCP_LOC) + loc = dict(loc) + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "demo.types", + "name": "Types demo", + "description": "column type gate", + "domain": "demo", + "metadata": {"layer": "Silver", "owner": {"team": "data", "email": "data@example.com"}}, + "exposes": [ + { + "exposeId": "orders", + "version": "1.0.0", + "kind": "table", + "binding": binding or {"platform": platform, "format": fmt, "location": loc}, + "contract": {"schema": [{"name": n, "type": t} for n, t in columns]}, + } + ], + } + + +def _gate(contract): + """What ``fluid validate`` reports for the contract's column types, both clouds.""" + gcp_errors, gcp_warnings = validate_gcp_binding(contract) + gov_errors, gov_warnings = validate_governance(contract) + return gcp_errors + gov_errors, gcp_warnings + gov_warnings + + +@pytest.mark.parametrize("platform", ["aws", "gcp"]) +def test_a_structured_column_is_refused_at_validate_and_at_generate(platform): + contract = _contract(platform, _BAD) + errors, _ = _gate(contract) + typed = [e for e in errors if "cannot hold its columns as declared" in e] + assert typed, errors + message = typed[0] + assert "expose 'orders'" in message + assert "'tags' is declared 'array'" in message + assert "'addr' is declared 'struct'" in message + assert "json or string" in message and "planned" in message + with pytest.raises(UnsupportedBindingError) as caught: + get_iac_plugin(platform).emit(contract) + assert caught.value.kind == "column-type" + assert "'tags' is declared 'array'" in str(caught.value) + assert any("json or string" in step for step in caught.value.remediation) + + +def test_the_validate_refusal_is_the_emit_refusal_word_for_word(): + for platform in ("aws", "gcp"): + contract = _contract(platform, _BAD) + errors, _ = _gate(contract) + with pytest.raises(UnsupportedBindingError) as caught: + get_iac_plugin(platform).emit(contract) + raised = f"{caught.value} {' '.join(caught.value.remediation)}" + assert raised in errors + + +def test_without_the_structured_columns_glue_writes_the_faithful_types_and_warns_on_json(): + contract = _contract("aws", _GOOD) + errors, warnings = _gate(contract) + assert errors == [] + assert len(warnings) == 1 and "'payload'" in warnings[0] and "'string'" in warnings[0] + resources = get_iac_plugin("aws").emit(contract) + columns = next(iter(resources["aws_glue_catalog_table"].values()))["storage_descriptor"][ + "columns" + ] + assert [(c["name"], c["type"]) for c in columns] == [ + ("id", "string"), + ("payload", "string"), + ("small", "smallint"), + ("tz", "timestamp"), + ] + + +def test_without_the_structured_columns_bigquery_gets_valid_types_and_no_warning(): + contract = _contract("gcp", _GOOD) + assert _gate(contract) == ([], []) + resources = get_iac_plugin("gcp").emit(contract) + schema = json.loads(next(iter(resources["google_bigquery_table"].values()))["schema"]) + assert [(f["name"], f["type"]) for f in schema] == [ + ("id", "STRING"), + ("payload", "JSON"), + ("small", "INT64"), + ("tz", "TIMESTAMP"), + ] + + +def test_a_geometry_column_on_bigquery_is_warned(): + errors, warnings = _gate(_contract("gcp", [("shape", "geometry")])) + assert errors == [] + assert len(warnings) == 1 and "planar" in warnings[0] + + +def test_a_geometry_column_on_glue_is_refused_not_guessed(): + errors, _ = _gate(_contract("aws", [("shape", "geometry")])) + assert any("'shape' is declared 'geometry'" in e and "WKT" in e for e in errors) + + +@pytest.mark.parametrize("platform", ["aws", "gcp"]) +def test_every_type_validate_lets_through_generates(platform): + """A contract that passes validate must not fail generate on a type, and the + reverse: one column per schema type, each expose judged by both.""" + for fluid_type in _ALL_TYPES: + contract = _contract(platform, [("col", fluid_type)]) + errors, _ = _gate(contract) + try: + get_iac_plugin(platform).emit(contract) + refused = False + except UnsupportedBindingError as exc: + assert exc.kind == "column-type" + refused = True + assert refused == bool(errors), (platform, fluid_type, errors) + + +def test_a_binding_with_no_glue_table_or_a_view_has_no_column_types_to_gate(): + no_table = _contract("aws", _BAD) + del no_table["exposes"][0]["binding"]["location"]["table"] + assert validate_aws_column_types(no_table) == ([], []) + view = _contract("gcp", _BAD) + view["exposes"][0]["binding"]["format"] = "bigquery_view" + view["exposes"][0]["binding"]["location"]["query"] = "SELECT 1" + assert not [e for e in _gate(view)[0] if "cannot hold its columns" in e] + get_iac_plugin("gcp").emit(view) + + +def test_another_clouds_binding_is_not_gated_by_this_ones_types(): + contract = _contract("aws", _BAD) + assert not [e for e in validate_gcp_binding(contract)[0] if "BigQuery" in e] + + +# ── fluid validate, end to end ─────────────────────────────────────────────── + + +def _validate(tmp_path, contract, *flags): + from fluid_build.cli import build_parser + from fluid_build.cli.validate import run + + path = tmp_path / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(contract, sort_keys=False), encoding="utf-8") + args = build_parser().parse_args(["validate", str(path), "--offline", *flags]) + return run(args, logging.getLogger("test")) + + +@pytest.mark.parametrize("platform", ["aws", "gcp"]) +def test_validate_strict_exits_1_for_a_structured_column(tmp_path, platform, capsys): + assert _validate(tmp_path, _contract(platform, _BAD), "--strict") == 1 + out = capsys.readouterr() + assert "cannot hold its columns as declared" in out.out + out.err + + +def test_validate_strict_exits_1_on_the_glue_json_warning_and_0_without_it(tmp_path, capsys): + assert _validate(tmp_path, _contract("aws", _GOOD), "--strict") == 1 + assert _validate(tmp_path, _contract("aws", _GOOD)) == 0 + assert ( + _validate(tmp_path, _contract("aws", [c for c in _GOOD if c[1] != "json"]), "--strict") == 0 + ) + assert _validate(tmp_path, _contract("gcp", _GOOD), "--strict") == 0 + capsys.readouterr() + + +# ── An in-place Glue column type change is held by the data-loss gate ──────── + + +def _glue_change(actions, before_types, after_types, before_raw=None): + def table(types): + if types is None: + return None + columns = [{"name": n, "type": t, "comment": ""} for n, t in types] + return { + "database_name": "demo", + "name": "orders", + "storage_descriptor": [{"columns": columns}], + } + + return { + "type": "aws_glue_catalog_table", + "address": "aws_glue_catalog_table.t", + "change": { + "actions": actions, + "before": before_raw if before_raw is not None else table(before_types), + "after": table(after_types), + }, + } + + +def _hold(*changes): + return get_iac_plugin("aws").plan_data_changes({"resource_changes": list(changes)}) + + +def test_a_glue_update_that_changes_a_column_type_is_held_with_the_whole_story(): + lines = _hold( + _glue_change( + ["update"], + [("id", "string"), ("small", "string"), ("tz", "string")], + [("id", "string"), ("small", "smallint"), ("tz", "timestamp")], + ) + ) + assert len(lines) == 1 + line = lines[0] + assert line.startswith("Glue table demo.orders: column types change in place") + assert "small: string -> smallint; tz: string -> timestamp" in line + assert "id:" not in line + assert "may stop reading until they are rewritten" in line + assert "reverting the column type restores reads" in line + + +def test_a_spacing_only_difference_a_new_column_or_a_create_is_not_held(): + same = [("id", "string"), ("d", "decimal(10, 2)")] + assert _hold(_glue_change(["update"], same, [("id", "string"), ("d", "decimal(10,2)")])) == [] + assert _hold(_glue_change(["update"], same, same + [("new", "int")])) == [] + assert _hold(_glue_change(["update"], same, same[:1])) == [] + assert _hold(_glue_change(["create"], None, same)) == [] + other = _glue_change(["update"], [("a", "string")], [("a", "int")]) + other["type"] = "aws_s3_bucket" + assert _hold(other) == [] + assert get_iac_plugin("aws").plan_data_changes({}) == [] + # A type change next to a spacing change is still held, for the type only. + held = _hold( + _glue_change( + ["update"], + [("a", "string"), ("d", "decimal(10, 2)")], + [("a", "int"), ("d", "decimal(10,2)")], + ) + ) + assert len(held) == 1 and "a: string -> int" in held[0] and "d:" not in held[0] + + +def test_a_malformed_plan_is_read_without_raising(): + assert _hold(_glue_change(["update"], None, [("a", "int")], before_raw="x")) == [] + assert _hold({"type": "aws_glue_catalog_table", "change": "x"}) == [] + + +def _enforce(lines, allow): + from fluid_build.cli import _apply_opentofu_engine as engine + + args = argparse.Namespace(allow_data_loss=allow) + changes = {"add": 0, "change": 1, "remove": 0} + engine._enforce_data_loss_gate( + args, logging.getLogger("test"), "aws", changes, {"remove": 0}, lines + ) + + +def test_the_apply_is_refused_without_the_flag_and_proceeds_with_it(monkeypatch, capsys): + from fluid_build.cli import _apply_opentofu_engine as engine + from fluid_build.cli._common import CLIError + + plan = { + "resource_changes": [ + _glue_change(["update"], [("small", "string")], [("small", "smallint")]) + ] + } + monkeypatch.setattr(engine.runner, "tofu_show_plan", lambda *a, **k: plan) + lines = engine._in_place_changes(get_iac_plugin("aws"), {"change": 1}, "wd", {}) + assert len(lines) == 1 + with pytest.raises(CLIError) as caught: + _enforce(lines, allow=False) + message = str(caught.value.context if hasattr(caught.value, "context") else caught.value) + assert "small: string -> smallint" in message and "--allow-data-loss" in message + _enforce(lines, allow=True) + assert "small: string -> smallint" in capsys.readouterr().out + + +def test_live_values_in_the_gate_list_and_the_notices_print_literally(monkeypatch, capsys): + """A live column type is data: ``string[/]`` must not be read as markup, and a + link tag must not become a hyperlink.""" + from fluid_build.cli import _apply_opentofu_engine as engine + + evil = "string[/] [link=https://evil.example]x[/link]" + lines = _hold(_glue_change(["update"], [("a", evil)], [("a", "int")])) + _enforce(lines, allow=True) + out = capsys.readouterr().out + assert "string[/] [link=https://evil.example]x[/link]" in out + + class Plugin: + def plan_notices(self, contract): + return [evil] + + engine._report_plan_notices(Plugin(), {}, logging.getLogger("test")) + assert evil in capsys.readouterr().out + + +# ── Hardening: non-ASCII parameters, the NUMERIC range, malformed bindings ─── + + +@pytest.mark.parametrize("fluid_type", ["decimal(١٠,٢)", "float(٥٣)", "number(１０)"]) +def test_non_ascii_digits_in_a_type_are_refused_on_both_clouds(fluid_type): + assert _hive_outcome(fluid_type).status == REFUSED + assert _bq_outcome(fluid_type).status == REFUSED + + +@pytest.mark.parametrize( + "fluid_type", ["decimal(38,0)", "decimal(40,10)", "numeric(38,10)", "number(30)"] +) +def test_a_decimal_wider_than_numeric_is_warned_on_bigquery(fluid_type): + outcome = _bq_outcome(fluid_type) + assert (outcome.target, outcome.status) == ("NUMERIC", LOSSY) + assert "bignumeric" in outcome.fix.lower() + errors, warnings = validate_gcp_binding(_contract("gcp", [("amt", fluid_type)])) + assert errors == [] and len(warnings) == 1 and "bignumeric" in warnings[0] + + +@pytest.mark.parametrize( + "fluid_type", ["decimal", "decimal(38,9)", "decimal(29,0)", "numeric(10,2)", "number(18)"] +) +def test_a_decimal_within_numeric_is_exact_on_bigquery(fluid_type): + assert _bq_outcome(fluid_type).status == EXACT + + +@pytest.mark.parametrize("platform", ["aws", "gcp"]) +@pytest.mark.parametrize( + "mutate", + [ + lambda e: e["binding"].update(location="s3://x"), + lambda e: e.update(contract="x"), + lambda e: e["contract"].update(schema="x"), + lambda e: e.update(binding="x"), + lambda e: e["contract"].update(schema=[1, None, "x"]), + ], +) +def test_a_schema_invalid_binding_is_skipped_not_fatal(platform, mutate): + contract = _contract(platform, _GOOD) + mutate(contract["exposes"][0]) + assert validate_aws_column_types(contract) == ([], []) + assert validate_bigquery_column_types(contract) == ([], []) + for exposes in ("x", 5, {"a": 1}): + assert validate_aws_column_types({"exposes": exposes}) == ([], []) + assert validate_bigquery_column_types({"exposes": exposes}) == ([], []) From 3a727c9c5af85926cafb6a5165d6dbc26ff42006 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:10:04 +0100 Subject: [PATCH 16/26] fix(aws): a schedule is never dropped silently, its names are valid, and its Lambda code takes contract values as literals (#794) * fix(aws): the planner keeps its action order, so the emitted module is deterministic DependencyGraph.nodes was a set, so the order of independent actions, and with it the emitted module, followed Python's hash seed. The IaC emitter interpolates a Lambda's attributes only when the function was emitted first, so a Lambda permission, schedule target, event-source mapping or S3 notification named its function by literal ARN in one run and by ${aws_lambda_function..arn} in the next, and tofu sometimes had no ordering edge to the function. nodes is now an insertion-ordered dict. Golden check on 21 AWS contracts (examples, test fixtures, synthetic scheduled ones), base under three hash seeds against this branch under two: modules without a Lambda are byte-identical; scheduled ones flipped on base and are stable here, always emitting the reference. * fix(aws): a schedule is never dropped silently, and the placeholder it emits says so The workflow Lambda was named fluid-workflow- raw, so a dotted id (the schema's example and 27 of 28 examples use dots) or one over ~49 characters failed the AWS plan validator, and native_actions swallowed that refusal at DEBUG: generate iac and apply logged "Planned EventBridge schedule" and wrote no Lambda, permission or scheduler, exit 0. - Every AWS name the schedule planner derives from a contract id (Lambda functions, permission statement ids, the schedule, event rules, the state machine, the MWAA environment) goes through bounded_aws_name: disallowed characters become '-', and past the service limit the name is cut and ends in a hash of the whole id, so long ids stay apart. Each rule cites the AWS API reference; every name starts with a fixed letter prefix. - A plan its own validator rejects raises PlanValidationError, which native_actions re-raises by type through the cause chain, like a sovereignty refusal: generate iac exits 1 and apply raises. A planner that could not run (no SDK) stays best-effort. - An AWS account that cannot be resolved is refused by name (AwsAccountUnresolvedError) before any name is derived from it, instead of surfacing as an invalid bucket "None-fluid-staging". Offline generate iac keeps its placeholder account. - The scheduled Lambda only logs its tasks and both roles it names are never created, so every plan that emits it warns schedule_placeholder_lambda, naming the roles. A real AWS scheduling path stays on Trello #875. Found by an external hands-on assessment of 0.26.0; Trello #875. * fix(aws): generated Lambda code takes contract values as literals; names never collide From the security review of this branch: - The workflow and trigger Lambdas' Python source was built by splicing contract values into string literals (TASKS = json.loads('''...'''), 'contract_id': '', FunctionName=''). A schema-valid orchestration task param such as '''+__import__('os').system('id')+''' closed the literal and ran on every schedule tick under the workflow role; an id with a trailing newline (which the id pattern admits) made a Lambda that could not import. Pre-existing on main. Every contract value now reaches the source only as a repr() literal; tests parse each generated source, walk its calls and round-trip hostile ids and params. - bounded_aws_name appends a hash of the raw name whenever sanitising changed it, so a.b and a-b no longer share one Lambda in an account. A name the service already accepts is unchanged, so nothing deployed is renamed. - The orchestration and schedule-planner failure paths raised a plain ProviderError that native_actions swallowed, still dropping the schedule with exit 0; they raise PlanValidationError now. - The S3 bucket rule allows dots as S3 does (no '..', '.-', '-.'), so a legal dotted bucket no longer fails generate iac now that validator refusals are fatal. - An unresolved account with boto3 missing says to install data-product-forge[aws] instead of blaming credentials. * test(iac): license header on the action-order test The test file split out of test_iac_aws_schedule_names.py had no Apache header, which the License Headers check requires. Added with scripts/add_license_headers.py. * fix(aws): an unresolved account refuses only a contract that needs the native plan The account check ran first in AwsProvider.plan() and refused every contract, so fluid apply of a schedule-less AWS contract failed when boto3 could not resolve the account (no [aws] extra, no STS access) although tofu resolves it itself and main applied such contracts fine; CI's test_state_backend_env_default.py caught it. Locally the tests passed because this machine's credentials resolve STS. The sovereignty check now runs first (the only place AWS enforces jurisdiction, and it needs no account). Then, with no account, a contract that declares orchestration (a schedule, triggers or tasks) is refused by name, since its Lambda, permission and schedule come from no other plan; any other contract gets a plain ProviderError, which native_actions treats as a planner that could not run, as before. * perf(aws): keep the new plan-error imports off the fluid --help path provider.py is loaded by fluid --help, and its new module-scope imports pulled _plan_validation, plan.schedule and util.config onto that budgeted path. They are now imported inside the methods that use them, so build_parser() loads exactly the fluid_build modules it does on main. --- CHANGELOG.md | 46 ++ fluid_build/cli/generate_iac.py | 37 +- fluid_build/providers/_plan_validation.py | 34 + fluid_build/providers/aws/plan/planner.py | 7 +- fluid_build/providers/aws/plan/schedule.py | 110 ++- fluid_build/providers/aws/provider.py | 68 +- fluid_build/providers/aws/util/config.py | 40 ++ .../providers/aws/util/dependencies.py | 9 +- fluid_build/providers/aws/util/names.py | 57 ++ fluid_build/providers/aws/util/validation.py | 12 +- tests/iac/test_iac_aws_action_order.py | 32 + tests/iac/test_iac_aws_schedule_names.py | 642 ++++++++++++++++++ tests/iac/test_iac_aws_unemitted_ops.py | 13 +- tests/iac/test_iac_generate_offline.py | 6 +- 14 files changed, 1052 insertions(+), 61 deletions(-) create mode 100644 fluid_build/providers/_plan_validation.py create mode 100644 tests/iac/test_iac_aws_action_order.py create mode 100644 tests/iac/test_iac_aws_schedule_names.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bafd6495..16670680e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -268,6 +268,52 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 dimension that reads the workspace's metastore summary for every table the module manages: it passes when the region matches the declaration, fails with the fix when it does not, and is skipped when the contract declares none. +- **An AWS contract's schedule was dropped without a word, and the one it got was not labelled.** + The workflow Lambda was named `fluid-workflow-` raw, so a dotted id (the schema's own + example is `gold.hr.employee_360_v1`, and 27 of 28 examples use dots) or one over about 49 + characters failed the plan validator, and `native_actions` swallowed the refusal at DEBUG: + `fluid generate iac` and `fluid apply` logged "Planned EventBridge schedule", wrote the Glue and + S3 resources and no `aws_lambda_function`, `aws_lambda_permission` or `aws_scheduler_schedule`, + exit 0. Every name the schedule planner derives from an id (Lambda functions, permission + statement ids, the schedule, event rules, the state machine, the MWAA environment) now goes + through one helper that replaces what the service refuses with `-` and, when it had to change + the name at all (a replacement, or a cut to the service limit), ends it in a hash of the whole + id, so `a.b` and `a-b` cannot share one Lambda and two long ids stay apart; a name the service + already accepts is left as written, so nothing deployed is renamed. A plan its own validator + rejects, an orchestration task the planner rejects and a schedule-planner failure are a typed + `PlanValidationError`, matched by type through the cause chain like a sovereignty refusal: + `generate iac` exits non-zero and `apply` raises, where a planner that could not run (no SDK, + no credentials) stays best-effort. This holds for every AWS plan-validator refusal, so a + contract with no schedule that the validator refuses (a bad Glue name, say) now fails + `generate iac` and `apply` too, where it used to emit the module without a word (`fluid plan` + already refused it). The scheduled Lambda only logs each + task and runs no build, and both roles it names (`fluid-workflow-execution`, + `EventBridgeSchedulerRole`) are never created, so every plan that emits it now warns + `schedule_placeholder_lambda`. When no AWS account can be resolved (no `AWS_ACCOUNT_ID`, no + credentials for `sts:GetCallerIdentity`), a contract that declares orchestration (a schedule, + triggers or tasks) is refused by name, before any name is derived from it, rather than as a Glue + or S3 name `None-fluid-staging` the validator would reject, and the message says so when the + cause is a missing boto3 (`pip install 'data-product-forge[aws]'`) rather than credentials. A + contract without orchestration needs no native plan and applies its exposes as before; the + sovereignty check still runs first either way. +- **The AWS plan validator refused dotted S3 bucket names.** S3 allows lowercase letters, digits, + dots and hyphens (3-63 characters, a letter or digit at each end, no `..`, `.-` or `-.`, not + shaped like an IP address); the validator allowed no dot, which was harmless while its refusals + were swallowed and is not now that they stop `generate iac` and `apply`. +- **A contract value could run as code in the AWS workflow Lambda.** The schedule planner spliced + task params and the contract id into the Lambda's Python source inside quote marks, so a task + param such as `'''+__import__('os').system('id')+'''` (the schema accepts any task params) + closed the literal and ran on every schedule tick under `fluid-workflow-execution`, and an id + with a trailing newline left a Lambda that was a SyntaxError. Every value now reaches the + source as a `repr()` literal. +- **The same AWS contract could emit a different module from one run to the next.** The AWS + planner ordered its actions through a `set`, so the order changed with Python's hash seed, and + with it the emitted module: a Lambda permission, schedule target, event-source mapping or S3 + notification named the function by literal ARN in one run and by + `${aws_lambda_function..arn}` in the next (the reference only when the function happened + to be planned first), so tofu sometimes had no ordering edge to the function. Actions now keep + the order they were planned in. Measured on 21 AWS contracts: modules without a Lambda are + byte-identical to before; scheduled ones now always emit the reference. ### Removed diff --git a/fluid_build/cli/generate_iac.py b/fluid_build/cli/generate_iac.py index f08cb4535..646a68828 100644 --- a/fluid_build/cli/generate_iac.py +++ b/fluid_build/cli/generate_iac.py @@ -410,12 +410,37 @@ def native_actions(contract, logger: logging.Logger, *, offline: bool = False) - # outside its declared jurisdiction logged the violation and then # emitted the module anyway, exit 0. Absence of a check and a failed # check are different things; only the first is best-effort. - if _is_sovereignty_refusal(exc): + # The same holds for a plan its own provider's validator rejected + # (``PlanValidationError``): a dotted contract id made the AWS planner + # refuse, and swallowing that left a module with no schedule, exit 0. + if _is_sovereignty_refusal(exc) or _is_plan_validation_refusal(exc): raise logger.debug("shadow: native planner unavailable: %s", exc) return [] +def _is_plan_validation_refusal(exc: BaseException) -> bool: + """True when ``exc`` (or anything it was raised *from*) is a plan-validation refusal. + + Matched on type through the ``__cause__``/``__context__`` chain, never on + message text, as :func:`_is_sovereignty_refusal` is. + """ + from fluid_build.providers._plan_validation import PlanValidationError + + return _in_cause_chain(exc, (PlanValidationError,)) + + +def _in_cause_chain(exc: BaseException, types: tuple) -> bool: + seen: set = set() + cur: BaseException | None = exc + while cur is not None and id(cur) not in seen: + if isinstance(cur, types): + return True + seen.add(id(cur)) + cur = cur.__cause__ or cur.__context__ + return False + + def _is_sovereignty_refusal(exc: BaseException) -> bool: """True when ``exc`` (or anything it was raised *from*) is a sovereignty veto. @@ -432,15 +457,7 @@ def _is_sovereignty_refusal(exc: BaseException) -> bool: from fluid_build._errors import ResidencyViolationError from fluid_build.cli._errors import SovereigntyViolationError - vetoes = (SovereigntyViolationError, ResidencyViolationError) - seen: set = set() - cur: BaseException | None = exc - while cur is not None and id(cur) not in seen: - if isinstance(cur, vetoes): - return True - seen.add(id(cur)) - cur = cur.__cause__ or cur.__context__ - return False + return _in_cause_chain(exc, (SovereigntyViolationError, ResidencyViolationError)) def _drop_referenced_container_actions(contract, actions: list, logger: logging.Logger) -> list: diff --git a/fluid_build/providers/_plan_validation.py b/fluid_build/providers/_plan_validation.py new file mode 100644 index 000000000..c2d82dd3d --- /dev/null +++ b/fluid_build/providers/_plan_validation.py @@ -0,0 +1,34 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The typed refusal a provider raises when its own plan fails validation. + +Kept apart from any one provider so ``cli/generate_iac.py`` can recognise it by +type without importing a provider package (which would register every provider +on a path that may be about another cloud). +""" + +from __future__ import annotations + +from fluid_build.providers.base import ProviderError + + +class PlanValidationError(ProviderError): + """A provider planned actions its own validator rejected. + + A planner that ran and refused, unlike one that could not run (no SDK, no + credentials): ``generate iac`` and ``apply`` treat the first as fatal and + only the second as best-effort. Typed so callers match on the class, never + on the message. + """ diff --git a/fluid_build/providers/aws/plan/planner.py b/fluid_build/providers/aws/plan/planner.py index 9f78013d2..e8cdda816 100644 --- a/fluid_build/providers/aws/plan/planner.py +++ b/fluid_build/providers/aws/plan/planner.py @@ -31,7 +31,7 @@ from ..util import warehouse as _warehouse from ..util.logging import format_event -from ..util.names import normalize_database_name +from ..util.names import EVENTBRIDGE_RULE_NAME, bounded_aws_name, normalize_database_name def _resolve_env_templates(value: str) -> str: @@ -740,7 +740,10 @@ def _plan_scheduling( { "op": "events.ensure_schedule", "id": "schedule_rule", - "name": f"{contract.get('id', 'contract').replace('.', '-')}-schedule", + "name": bounded_aws_name( + f"{contract.get('id', 'contract').replace('.', '-')}-schedule", + EVENTBRIDGE_RULE_NAME, + ), "schedule": eventbridge_schedule, "description": f"Schedule for {contract.get('name')}", "tags": _get_resource_tags(contract), diff --git a/fluid_build/providers/aws/plan/schedule.py b/fluid_build/providers/aws/plan/schedule.py index 6a3913fb4..f710417b1 100644 --- a/fluid_build/providers/aws/plan/schedule.py +++ b/fluid_build/providers/aws/plan/schedule.py @@ -28,8 +28,23 @@ import logging from typing import Any, Dict, List, Optional +from ..util.names import ( + EVENTBRIDGE_RULE_NAME, + LAMBDA_FUNCTION_NAME, + LAMBDA_STATEMENT_ID, + MWAA_ENVIRONMENT_NAME, + SCHEDULER_SCHEDULE_NAME, + STATE_MACHINE_NAME, + bounded_aws_name, +) + logger = logging.getLogger(__name__) +#: The IAM roles the planned Lambdas and schedules reference by literal ARN. The +#: planner emits no ``aws_iam_role``: both must already exist in the account. +WORKFLOW_EXECUTION_ROLE = "fluid-workflow-execution" +SCHEDULER_INVOKE_ROLE = "EventBridgeSchedulerRole" + class SchedulePlanner: """ @@ -52,6 +67,17 @@ def __init__(self, account_id: str, region: str, logger: Optional[logging.Logger self.region = region self.logger = logger or logging.getLogger(__name__) + @staticmethod + def workflow_function_name(contract_id: Any) -> str: + """The workflow Lambda's name: the one every trigger Lambda invokes. + + Every AWS name derived from a contract id goes through + :func:`bounded_aws_name`. A dotted id (``gold.hr.employee_360_v1``, the + schema's own example) or a long one used to reach the plan validator raw, + which refused the whole plan, so the schedule went missing from the module. + """ + return bounded_aws_name(f"fluid-workflow-{contract_id}", LAMBDA_FUNCTION_NAME) + def plan_schedule_actions(self, contract: Dict[str, Any]) -> List[Dict[str, Any]]: """ Generate AWS scheduling actions from contract. @@ -124,7 +150,7 @@ def _plan_mwaa_schedule( actions = [] contract_id = contract.get("id", "unknown") - environment_name = f"mwaa-{contract_id}" + environment_name = bounded_aws_name(f"mwaa-{contract_id}", MWAA_ENVIRONMENT_NAME) # Check if MWAA environment already specified mwaa_config = orchestration.get("mwaa", {}) @@ -174,7 +200,7 @@ def _plan_eventbridge_schedule( timezone = orchestration.get("timezone", "UTC") # 1. Create Lambda function for workflow execution - function_name = f"fluid-workflow-{contract_id}" + function_name = self.workflow_function_name(contract_id) actions.append( { @@ -183,7 +209,7 @@ def _plan_eventbridge_schedule( "function_name": function_name, "runtime": "python3.11", "handler": "index.handler", - "role": f"arn:aws:iam::{self.account_id}:role/fluid-workflow-execution", + "role": f"arn:aws:iam::{self.account_id}:role/{WORKFLOW_EXECUTION_ROLE}", "code": self._generate_workflow_lambda_code(contract, orchestration), "timeout": orchestration.get("timeout", 300), "memory_size": orchestration.get("memory", 256), @@ -197,7 +223,7 @@ def _plan_eventbridge_schedule( ) # 2. Create EventBridge Scheduler rule - schedule_name = f"fluid-schedule-{contract_id}" + schedule_name = bounded_aws_name(f"fluid-schedule-{contract_id}", SCHEDULER_SCHEDULE_NAME) actions.append( { @@ -208,7 +234,7 @@ def _plan_eventbridge_schedule( "timezone": timezone, "target": { "arn": f"arn:aws:lambda:{self.region}:{self.account_id}:function:{function_name}", - "role_arn": f"arn:aws:iam::{self.account_id}:role/EventBridgeSchedulerRole", + "role_arn": f"arn:aws:iam::{self.account_id}:role/{SCHEDULER_INVOKE_ROLE}", "input": json.dumps( {"contract_id": contract_id, "execution_type": "scheduled"} ), @@ -226,7 +252,9 @@ def _plan_eventbridge_schedule( "id": f"lambda_permission_{contract_id}", "op": "lambda.add_permission", "function_name": function_name, - "statement_id": f"AllowEventBridgeInvoke-{contract_id}", + "statement_id": bounded_aws_name( + f"AllowEventBridgeInvoke-{contract_id}", LAMBDA_STATEMENT_ID + ), "action": "lambda:InvokeFunction", "principal": "scheduler.amazonaws.com", "source_arn": f"arn:aws:scheduler:{self.region}:{self.account_id}:schedule/default/{schedule_name}", @@ -277,7 +305,9 @@ def _plan_s3_trigger( actions = [] contract_id = contract.get("id", "unknown") - function_name = f"fluid-s3-trigger-{contract_id}-{trigger_id}" + function_name = bounded_aws_name( + f"fluid-s3-trigger-{contract_id}-{trigger_id}", LAMBDA_FUNCTION_NAME + ) # 1. Create Lambda function actions.append( @@ -287,7 +317,7 @@ def _plan_s3_trigger( "function_name": function_name, "runtime": "python3.11", "handler": "index.handler", - "role": f"arn:aws:iam::{self.account_id}:role/fluid-workflow-execution", + "role": f"arn:aws:iam::{self.account_id}:role/{WORKFLOW_EXECUTION_ROLE}", "code": self._generate_s3_trigger_lambda_code(contract, trigger), "timeout": 300, "tags": self._get_schedule_tags(contract), @@ -318,7 +348,9 @@ def _plan_s3_trigger( "id": f"lambda_permission_s3_{trigger_id}", "op": "lambda.add_permission", "function_name": function_name, - "statement_id": f"AllowS3Invoke-{trigger_id}", + "statement_id": bounded_aws_name( + f"AllowS3Invoke-{trigger_id}", LAMBDA_STATEMENT_ID + ), "action": "lambda:InvokeFunction", "principal": "s3.amazonaws.com", "source_arn": f"arn:aws:s3:::{bucket}", @@ -334,7 +366,9 @@ def _plan_dynamodb_trigger( actions = [] contract_id = contract.get("id", "unknown") - function_name = f"fluid-dynamodb-trigger-{contract_id}-{trigger_id}" + function_name = bounded_aws_name( + f"fluid-dynamodb-trigger-{contract_id}-{trigger_id}", LAMBDA_FUNCTION_NAME + ) # 1. Create Lambda function actions.append( @@ -344,7 +378,7 @@ def _plan_dynamodb_trigger( "function_name": function_name, "runtime": "python3.11", "handler": "index.handler", - "role": f"arn:aws:iam::{self.account_id}:role/fluid-workflow-execution", + "role": f"arn:aws:iam::{self.account_id}:role/{WORKFLOW_EXECUTION_ROLE}", "code": self._generate_dynamodb_trigger_lambda_code(contract, trigger), "timeout": 300, "tags": self._get_schedule_tags(contract), @@ -376,8 +410,12 @@ def _plan_eventbridge_trigger( actions = [] contract_id = contract.get("id", "unknown") - rule_name = f"fluid-event-{contract_id}-{trigger_id}" - function_name = f"fluid-event-trigger-{contract_id}-{trigger_id}" + rule_name = bounded_aws_name( + f"fluid-event-{contract_id}-{trigger_id}", EVENTBRIDGE_RULE_NAME + ) + function_name = bounded_aws_name( + f"fluid-event-trigger-{contract_id}-{trigger_id}", LAMBDA_FUNCTION_NAME + ) # 1. Create Lambda function actions.append( @@ -387,7 +425,7 @@ def _plan_eventbridge_trigger( "function_name": function_name, "runtime": "python3.11", "handler": "index.handler", - "role": f"arn:aws:iam::{self.account_id}:role/fluid-workflow-execution", + "role": f"arn:aws:iam::{self.account_id}:role/{WORKFLOW_EXECUTION_ROLE}", "code": self._generate_event_trigger_lambda_code(contract, trigger), "timeout": 300, "tags": self._get_schedule_tags(contract), @@ -420,7 +458,9 @@ def _plan_eventbridge_trigger( "id": f"lambda_permission_eventbridge_{trigger_id}", "op": "lambda.add_permission", "function_name": function_name, - "statement_id": f"AllowEventBridgeInvoke-{trigger_id}", + "statement_id": bounded_aws_name( + f"AllowEventBridgeInvoke-{trigger_id}", LAMBDA_STATEMENT_ID + ), "action": "lambda:InvokeFunction", "principal": "events.amazonaws.com", "source_arn": f"arn:aws:events:{self.region}:{self.account_id}:rule/{rule_name}", @@ -436,7 +476,9 @@ def _plan_sqs_trigger( actions = [] contract_id = contract.get("id", "unknown") - function_name = f"fluid-sqs-trigger-{contract_id}-{trigger_id}" + function_name = bounded_aws_name( + f"fluid-sqs-trigger-{contract_id}-{trigger_id}", LAMBDA_FUNCTION_NAME + ) # 1. Create Lambda function actions.append( @@ -446,7 +488,7 @@ def _plan_sqs_trigger( "function_name": function_name, "runtime": "python3.11", "handler": "index.handler", - "role": f"arn:aws:iam::{self.account_id}:role/fluid-workflow-execution", + "role": f"arn:aws:iam::{self.account_id}:role/{WORKFLOW_EXECUTION_ROLE}", "code": self._generate_sqs_trigger_lambda_code(contract, trigger), "timeout": 300, "tags": self._get_schedule_tags(contract), @@ -480,7 +522,7 @@ def _plan_step_functions( actions = [] contract_id = contract.get("id", "unknown") - state_machine_name = f"fluid-workflow-{contract_id}" + state_machine_name = bounded_aws_name(f"fluid-workflow-{contract_id}", STATE_MACHINE_NAME) # Generate state machine definition from tasks state_machine_def = self._generate_state_machine_definition(contract, orchestration) @@ -518,6 +560,10 @@ def _generate_workflow_lambda_code( if t.get("type") == "provider_action" ] ) + # Contract values reach this source only as ``repr()`` literals (``!r``): + # spliced inside quotes, a task param or an id carrying a quote closed the + # literal and ran as code on every tick (or left a SyntaxError behind). + default_id = str(contract.get("id")) return { "ZipFile": f""" import json @@ -527,10 +573,10 @@ def _generate_workflow_lambda_code( logger = logging.getLogger() logger.setLevel(logging.INFO) -TASKS = json.loads('''{task_entries}''') +TASKS = json.loads({task_entries!r}) def handler(event, context): - contract_id = event.get('contract_id', '{contract.get("id")}') + contract_id = event.get('contract_id', {default_id!r}) logger.info(f"Executing FLUID workflow for contract: {{contract_id}}") completed = set() @@ -563,7 +609,7 @@ def _generate_s3_trigger_lambda_code( ) -> Dict[str, Any]: """Generate Lambda code for S3 trigger.""" contract_id = contract.get("id", "unknown") - workflow_fn = f"fluid-workflow-{contract_id}" + workflow_fn = self.workflow_function_name(contract_id) return { "ZipFile": f""" import json @@ -580,10 +626,10 @@ def handler(event, context): key = record['s3']['object']['key'] logger.info(f"S3 trigger: s3://{{bucket}}/{{key}}") lam.invoke( - FunctionName='{workflow_fn}', + FunctionName={workflow_fn!r}, InvocationType='Event', Payload=json.dumps({{ - 'contract_id': '{contract_id}', + 'contract_id': {contract_id!r}, 'trigger': 's3', 'bucket': bucket, 'key': key, @@ -598,7 +644,7 @@ def _generate_dynamodb_trigger_lambda_code( ) -> Dict[str, Any]: """Generate Lambda code for DynamoDB trigger.""" contract_id = contract.get("id", "unknown") - workflow_fn = f"fluid-workflow-{contract_id}" + workflow_fn = self.workflow_function_name(contract_id) return { "ZipFile": f""" import json @@ -614,10 +660,10 @@ def handler(event, context): event_name = record.get('eventName') logger.info(f"DynamoDB trigger: {{event_name}}") lam.invoke( - FunctionName='{workflow_fn}', + FunctionName={workflow_fn!r}, InvocationType='Event', Payload=json.dumps({{ - 'contract_id': '{contract_id}', + 'contract_id': {contract_id!r}, 'trigger': 'dynamodb', 'event_name': event_name, 'keys': record.get('dynamodb', {{}}).get('Keys', {{}}), @@ -632,7 +678,7 @@ def _generate_event_trigger_lambda_code( ) -> Dict[str, Any]: """Generate Lambda code for EventBridge trigger.""" contract_id = contract.get("id", "unknown") - workflow_fn = f"fluid-workflow-{contract_id}" + workflow_fn = self.workflow_function_name(contract_id) return { "ZipFile": f""" import json @@ -646,10 +692,10 @@ def handler(event, context): logger.info(f"EventBridge trigger: {{json.dumps(event)}}") lam = boto3.client('lambda') lam.invoke( - FunctionName='{workflow_fn}', + FunctionName={workflow_fn!r}, InvocationType='Event', Payload=json.dumps({{ - 'contract_id': '{contract_id}', + 'contract_id': {contract_id!r}, 'trigger': 'eventbridge', 'detail_type': event.get('detail-type', ''), 'detail': event.get('detail', {{}}), @@ -664,7 +710,7 @@ def _generate_sqs_trigger_lambda_code( ) -> Dict[str, Any]: """Generate Lambda code for SQS trigger.""" contract_id = contract.get("id", "unknown") - workflow_fn = f"fluid-workflow-{contract_id}" + workflow_fn = self.workflow_function_name(contract_id) return { "ZipFile": f""" import json @@ -680,10 +726,10 @@ def handler(event, context): body = json.loads(record['body']) logger.info(f"SQS trigger: {{body}}") lam.invoke( - FunctionName='{workflow_fn}', + FunctionName={workflow_fn!r}, InvocationType='Event', Payload=json.dumps({{ - 'contract_id': '{contract_id}', + 'contract_id': {contract_id!r}, 'trigger': 'sqs', 'body': body, }}), diff --git a/fluid_build/providers/aws/provider.py b/fluid_build/providers/aws/provider.py index 389cb2ae2..3cd4a2c08 100644 --- a/fluid_build/providers/aws/provider.py +++ b/fluid_build/providers/aws/provider.py @@ -25,7 +25,7 @@ import time from collections.abc import Mapping -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, NoReturn, Optional from fluid_build.providers.base import ( ApplyResult, @@ -166,6 +166,10 @@ def plan( Athena queries / workgroups, Redshift schemas, Lambda functions, EventBridge rules / schedules, IAM roles / policies. """ + # Imported here, not at module scope: this module is loaded by + # ``fluid --help``, whose module count is budgeted. + from fluid_build.providers._plan_validation import PlanValidationError + self.debug_kv( event="plan_started", contract_id=contract.get("id"), @@ -174,9 +178,13 @@ def plan( ) try: - # Validate sovereignty constraints (FLUID 0.7.1) + # Validate sovereignty constraints (FLUID 0.7.1). First, and with no + # account needed: this is the only place AWS enforces jurisdiction. self._validate_sovereignty(contract) + if not self.account_id: + self._no_plan_without_account(contract) + # ``plan_actions`` constructs an :class:`AwsPlanner` and # calls :meth:`BasePlanner.plan` which already runs the # 6-phase scaffold (infrastructure / IAM / replace-snapshots @@ -202,7 +210,7 @@ def plan( if not validation_result["valid"]: error_msg = "Plan validation failed:\n" + "\n".join(validation_result["errors"]) - raise ProviderError(error_msg) + raise PlanValidationError(error_msg) # Log warnings if any for warning in validation_result.get("warnings", []): @@ -316,6 +324,28 @@ def _validate_sovereignty(self, contract: Mapping[str, Any]) -> None: self.err_kv(event="sovereignty_violation", error=str(e)) raise ProviderError(str(e)) from e + def _no_plan_without_account(self, contract: Mapping[str, Any]) -> NoReturn: + """No account resolved, so no name can be derived: stop before one is. + + With ``None`` the planner builds ``None-fluid-staging`` and the validator + refuses the plan with a message about a bucket, hiding the cause. A + contract that declares orchestration (a schedule, triggers or tasks) + needs this plan, since its Lambda, permission and schedule come from + nowhere else, so the gap is refused by name. Any other contract's + resources come from its exposes, which tofu applies with its own + credentials: the planner reports that it could not run, and + ``native_actions`` treats that as best-effort, as before. ``generate + iac`` passes a placeholder account, so it never reaches this. + """ + from .util.config import AwsAccountUnresolvedError, account_unresolved_message + + orchestration = contract.get("orchestration") + if isinstance(orchestration, Mapping) and any( + orchestration.get(key) for key in ("schedule", "triggers", "tasks") + ): + raise AwsAccountUnresolvedError(account_unresolved_message()) + raise ProviderError(f"native AWS plan skipped: {account_unresolved_message()}") + def _plan_orchestration(self, contract: Mapping[str, Any]) -> List[Dict[str, Any]]: """ Plan orchestration tasks from contract (FLUID 0.7.1). @@ -327,6 +357,8 @@ def _plan_orchestration(self, contract: Mapping[str, Any]) -> List[Dict[str, Any if not orchestration: return [] + from fluid_build.providers._plan_validation import PlanValidationError + from .plan.orchestration import OrchestrationError, plan_orchestration_tasks try: @@ -343,7 +375,7 @@ def _plan_orchestration(self, contract: Mapping[str, Any]) -> List[Dict[str, Any except OrchestrationError as e: self.err_kv(event="orchestration_planning_failed", error=str(e)) - raise ProviderError(f"Orchestration planning failed: {e}") from e + raise PlanValidationError(f"Orchestration planning failed: {e}") from e def _plan_schedule(self, contract: Mapping[str, Any]) -> List[Dict[str, Any]]: """ @@ -360,11 +392,35 @@ def _plan_schedule(self, contract: Mapping[str, Any]) -> List[Dict[str, Any]]: if not orchestration.get("schedule") and not orchestration.get("triggers"): return [] - from .plan.schedule import plan_schedule_actions + from fluid_build.providers._plan_validation import PlanValidationError + + from .plan.schedule import ( + SCHEDULER_INVOKE_ROLE, + WORKFLOW_EXECUTION_ROLE, + plan_schedule_actions, + ) try: actions = plan_schedule_actions(contract, self.account_id, self.region, self.logger) + if any(a.get("op") == "eventbridge.ensure_schedule" for a in actions): + # The schedule fires a Lambda whose body only logs each task and + # records it ok; it runs no build. Its two roles are named by + # literal ARN and nothing here creates them. Say so on every + # plan that emits it, rather than let a green apply imply a + # working pipeline. + self.warn_kv( + event="schedule_placeholder_lambda", + message=( + "The scheduled Lambda is a placeholder: it logs each task and does " + "not run this product's builds. The IAM roles " + f"arn:aws:iam::{self.account_id}:role/{WORKFLOW_EXECUTION_ROLE} " + f"(Lambda) and arn:aws:iam::{self.account_id}:role/" + f"{SCHEDULER_INVOKE_ROLE} (EventBridge Scheduler) are referenced by " + "name and not created: they must already exist in the account." + ), + contract_id=contract.get("id"), + ) if actions: self.info_kv( event="schedule_planned", @@ -377,7 +433,7 @@ def _plan_schedule(self, contract: Mapping[str, Any]) -> List[Dict[str, Any]]: except Exception as e: self.err_kv(event="schedule_planning_failed", error=str(e)) - raise ProviderError(f"Schedule planning failed: {e}") from e + raise PlanValidationError(f"Schedule planning failed: {e}") from e def _utc_timestamp(self) -> str: """Generate UTC timestamp string.""" diff --git a/fluid_build/providers/aws/util/config.py b/fluid_build/providers/aws/util/config.py index cde828f8d..0ac9fc5a9 100644 --- a/fluid_build/providers/aws/util/config.py +++ b/fluid_build/providers/aws/util/config.py @@ -17,9 +17,49 @@ AWS configuration resolution utilities. """ +import importlib.util import os from typing import Optional, Tuple +from fluid_build.providers._plan_validation import PlanValidationError + + +class AwsAccountUnresolvedError(PlanValidationError): + """No AWS account id could be resolved, so no resource name can be planned. + + A :class:`PlanValidationError` so ``native_actions`` re-raises it (matched by + type) rather than treating the planner as one that could not run: the + account feeds names (``{account}-fluid-data``) and every role ARN, and a + plan built on ``None`` is refused by the validator with a message about a + bucket name, not the cause. + """ + + +def _boto3_missing() -> bool: + """True when boto3 (the ``[aws]`` extra) is not installed.""" + return importlib.util.find_spec("boto3") is None + + +def account_unresolved_message() -> str: + """Why no account resolved, and what fixes it. + + ``resolve_account_and_region`` swallows the ``ImportError`` for a missing + boto3 along with every STS failure, so the cause is re-derived here: + blaming credentials for a package that is not installed sends the user to + the wrong place. + """ + if _boto3_missing(): + return ( + "The AWS account id could not be resolved: boto3 is not installed, so " + "sts:GetCallerIdentity cannot be called. Install it with " + "pip install 'data-product-forge[aws]', or set AWS_ACCOUNT_ID." + ) + return ( + "The AWS account id could not be resolved. Set AWS_ACCOUNT_ID, or " + "configure AWS credentials (environment, profile or role) so that " + "sts:GetCallerIdentity succeeds." + ) + def resolve_account_and_region( account_id: Optional[str] = None, region: Optional[str] = None diff --git a/fluid_build/providers/aws/util/dependencies.py b/fluid_build/providers/aws/util/dependencies.py index 3ebf351a9..89341ec5f 100644 --- a/fluid_build/providers/aws/util/dependencies.py +++ b/fluid_build/providers/aws/util/dependencies.py @@ -31,7 +31,12 @@ class DependencyGraph: def __init__(self): """Initialize empty dependency graph.""" - self.nodes: Set[str] = set() + # Insertion-ordered (a dict, not a set): ``topological_sort`` walks it, so a + # set made the action order, and with it the emitted module, differ between + # runs. The IaC emitter interpolates a Lambda's attributes only when the + # function was emitted first, so a schedule or permission planned for it + # came out as a literal ARN or a reference depending on the hash seed. + self.nodes: Dict[str, None] = {} self.edges: Dict[str, Set[str]] = defaultdict(set) # node -> dependencies self.action_map: Dict[str, Dict[str, Any]] = {} @@ -46,7 +51,7 @@ def add_action(self, action: Dict[str, Any]) -> str: Action ID """ action_id = action.get("id", self._generate_id(action)) - self.nodes.add(action_id) + self.nodes[action_id] = None self.action_map[action_id] = action return action_id diff --git a/fluid_build/providers/aws/util/names.py b/fluid_build/providers/aws/util/names.py index dbba04283..5c15c1f46 100644 --- a/fluid_build/providers/aws/util/names.py +++ b/fluid_build/providers/aws/util/names.py @@ -17,6 +17,7 @@ AWS resource name normalization utilities. """ +import hashlib import re from typing import Optional @@ -155,3 +156,59 @@ def normalize_lambda_name(name: str) -> str: # Truncate to 64 characters return normalized[:64] + + +#: Per-service naming rules for names the planner derives from a contract id. +#: Each is ``(characters NOT allowed, maximum length)``; pinned from the AWS API +#: references so the planner's output is valid before ``validate_actions`` sees +#: it: +#: +#: * Lambda function name, ``CreateFunction`` ``FunctionName``: letters, digits, +#: ``-`` and ``_``; 64 characters when only the name is given (the API's longer +#: limit applies to a full ARN). +#: https://docs.aws.amazon.com/lambda/latest/api/API_CreateFunction.html +#: * Lambda permission statement id, ``AddPermission`` ``StatementId``: letters, +#: digits, ``-`` and ``_``; 1-100 characters. +#: https://docs.aws.amazon.com/lambda/latest/api/API_AddPermission.html +#: * EventBridge Scheduler schedule name, ``CreateSchedule`` ``Name``: letters, +#: digits, ``-``, ``_`` and ``.``; 1-64 characters. +#: https://docs.aws.amazon.com/scheduler/latest/APIReference/API_CreateSchedule.html +#: * EventBridge rule name, ``PutRule`` ``Name``: letters, digits, ``-``, ``_`` +#: and ``.``; 1-64 characters. +#: https://docs.aws.amazon.com/eventbridge/latest/APIReference/API_PutRule.html +#: * MWAA environment name, ``CreateEnvironment`` ``Name``: starts with a letter, +#: then letters, digits, ``-`` and ``_``; 1-80 characters (every name built +#: here starts with the literal ``mwaa-``). +#: https://docs.aws.amazon.com/mwaa/latest/API/API_CreateEnvironment.html +#: * Step Functions state machine name, ``CreateStateMachine`` ``name``: 1-80 +#: characters, no whitespace, brackets, wildcards or ``" # % \ ^ | ~ ` $ & , ; : /``. +#: https://docs.aws.amazon.com/step-functions/latest/apireference/API_CreateStateMachine.html +LAMBDA_FUNCTION_NAME = (re.compile(r"[^a-zA-Z0-9\-_]"), 64) +LAMBDA_STATEMENT_ID = (re.compile(r"[^a-zA-Z0-9\-_]"), 100) +SCHEDULER_SCHEDULE_NAME = (re.compile(r"[^a-zA-Z0-9\-_.]"), 64) +EVENTBRIDGE_RULE_NAME = (re.compile(r"[^a-zA-Z0-9\-_.]"), 64) +MWAA_ENVIRONMENT_NAME = (re.compile(r"[^a-zA-Z0-9\-_]"), 80) +STATE_MACHINE_NAME = (re.compile(r"[\s<>{}\[\]?*\"#%\\^|~`$&,;:/\x00-\x1f\x7f-\x9f]"), 80) + +#: Hex digits of the digest appended to a name that had to be shortened. +_NAME_HASH_LEN = 8 + + +def bounded_aws_name(raw: str, rule: "tuple[re.Pattern[str], int]") -> str: + """``raw`` as a name ``rule`` accepts, deterministic and collision-safe. + + Characters outside the rule's allowed set become ``-``. A name that had to + change at all (a replacement, or a cut to the service limit) ends in + ``-``, the hash taken from the *whole raw* name: ``a.b`` and ``a-b`` + would otherwise both be ``fluid-workflow-a-b``, two products in one account + fighting over one Lambda, and two long ids sharing a prefix would collide + after the cut. A name the rule already accepts is returned as written, so + nothing already deployed under it is renamed. + """ + pattern, limit = rule + name = pattern.sub("-", raw) + if name == raw and len(name) <= limit: + return name + digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:_NAME_HASH_LEN] + keep = limit - _NAME_HASH_LEN - 1 + return f"{name[:keep]}-{digest}" diff --git a/fluid_build/providers/aws/util/validation.py b/fluid_build/providers/aws/util/validation.py index 38a7bb770..03ab32ee7 100644 --- a/fluid_build/providers/aws/util/validation.py +++ b/fluid_build/providers/aws/util/validation.py @@ -37,7 +37,7 @@ class ResourceValidator: """Validates AWS resource configurations before execution.""" # AWS naming rules - S3_BUCKET_PATTERN = re.compile(r"^[a-z0-9][a-z0-9\-]{1,61}[a-z0-9]\Z") + S3_BUCKET_PATTERN = re.compile(r"^[a-z0-9][a-z0-9.\-]{1,61}[a-z0-9]\Z") GLUE_DATABASE_PATTERN = re.compile(r"^[a-z0-9_]+\Z") GLUE_TABLE_PATTERN = re.compile(r"^[a-z0-9_]+\Z") LAMBDA_FUNCTION_PATTERN = re.compile(r"^[a-zA-Z0-9\-_]+\Z") @@ -69,9 +69,10 @@ def validate_s3_bucket_name(self, bucket: str) -> Tuple[bool, Optional[str]]: Rules: - 3-63 characters - - Lowercase letters, numbers, hyphens + - Lowercase letters, numbers, dots, hyphens (a dot-separated name is a + legal bucket name) - Must start and end with letter or number - - No consecutive hyphens + - No consecutive dots, no dot next to a hyphen, no consecutive hyphens - No IP address format Args: @@ -92,13 +93,16 @@ def validate_s3_bucket_name(self, bucket: str) -> Tuple[bool, Optional[str]]: if not self.S3_BUCKET_PATTERN.match(bucket): return False, ( f"Bucket name {bucket!r} is invalid. " - "Must contain only lowercase letters, numbers, and hyphens. " + "Must contain only lowercase letters, numbers, dots, and hyphens. " "Must start and end with letter or number." ) if "--" in bucket: return False, "Bucket name cannot contain consecutive hyphens" + if ".." in bucket or ".-" in bucket or "-." in bucket: + return False, "Bucket name cannot contain '..', '.-' or '-.'" + # Check for IP address format (e.g., 192.168.0.1) if re.match(r"^\d+\.\d+\.\d+\.\d+$", bucket): return False, "Bucket name cannot be formatted as an IP address" diff --git a/tests/iac/test_iac_aws_action_order.py b/tests/iac/test_iac_aws_action_order.py new file mode 100644 index 000000000..b243a5f77 --- /dev/null +++ b/tests/iac/test_iac_aws_action_order.py @@ -0,0 +1,32 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The AWS planner keeps the order it planned actions in. + +``DependencyGraph.nodes`` was a ``set``, so the order of independent actions, and with it +the emitted module, followed Python's hash seed: a Lambda permission or schedule target +named its function by literal ARN in one run and by reference in the next. +""" + +from __future__ import annotations + + +class TestActionOrderIsStable: + def test_independent_actions_keep_the_order_they_were_planned_in(self): + from fluid_build.providers.aws.util.dependencies import order_actions_by_dependencies + + actions = [{"id": f"a{i}", "op": "x.y"} for i in range(40)] + assert [a["id"] for a in order_actions_by_dependencies(actions)] == [ + a["id"] for a in actions + ] diff --git a/tests/iac/test_iac_aws_schedule_names.py b/tests/iac/test_iac_aws_schedule_names.py new file mode 100644 index 000000000..eac68a908 --- /dev/null +++ b/tests/iac/test_iac_aws_schedule_names.py @@ -0,0 +1,642 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An AWS contract's schedule is never dropped silently, and is honestly labelled. + +``fluid generate iac`` on a contract whose id carried a dot (the schema's own +example is ``gold.hr.employee_360_v1``) logged "Planned EventBridge schedule" +and wrote a module with no ``aws_lambda_function``, ``aws_lambda_permission`` or +``aws_scheduler_schedule``, exit 0: the planner named the Lambda +``fluid-workflow-`` raw, the plan validator refused the dot, and +``native_actions`` swallowed the refusal at DEBUG. ``fluid apply`` shares that +function, so it omitted the schedule the same way. +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import logging +import re +import sys +import types +from pathlib import Path +from typing import Any, Dict + +import pytest +import yaml + +from fluid_build.cli import generate_iac +from fluid_build.cli._apply_opentofu_engine import emit_module +from fluid_build.cli._common import CLIError +from fluid_build.providers._plan_validation import PlanValidationError +from fluid_build.providers.aws.plan.schedule import SchedulePlanner +from fluid_build.providers.aws.util.config import AwsAccountUnresolvedError +from fluid_build.providers.aws.util.names import ( + EVENTBRIDGE_RULE_NAME, + LAMBDA_FUNCTION_NAME, + LAMBDA_STATEMENT_ID, + MWAA_ENVIRONMENT_NAME, + SCHEDULER_SCHEDULE_NAME, + STATE_MACHINE_NAME, + bounded_aws_name, +) +from fluid_build.providers.base import ProviderError + +pytestmark = [pytest.mark.unit, pytest.mark.provider] + +ACCOUNT = "123456789012" +REGION = "us-east-1" +DOTTED = "sales.orders_daily" +LONG = "gold.finance." + "quarterly_revenue_recognition_" * 4 + "v1" + +_LAMBDA_NAME = re.compile(r"[a-zA-Z0-9\-_]{1,64}") +_STATEMENT_ID = re.compile(r"[a-zA-Z0-9\-_]{1,100}") +_SCHEDULE_NAME = re.compile(r"[a-zA-Z0-9\-_.]{1,64}") + + +def _contract(contract_id: str = DOTTED, **orchestration: Any) -> Dict[str, Any]: + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": contract_id, + "name": "Orders", + "metadata": {"layer": "Silver", "owner": {"team": "data", "email": "d@example.com"}}, + "orchestration": {"engine": "none", "schedule": "cron(0 6 * * ? *)", **orchestration}, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": { + "platform": "aws", + "format": "parquet", + "location": { + "bucket": "offline-lake", + "path": "orders/", + "database": "sales", + "table": "orders", + "region": REGION, + }, + }, + "contract": {"schema": [{"name": "id", "type": "string"}]}, + } + ], + } + + +def _args(contract: Dict[str, Any], tmp_path: Path) -> argparse.Namespace: + path = tmp_path / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(contract, sort_keys=False), encoding="utf-8") + return argparse.Namespace( + contract=str(path), provider="auto", out=str(tmp_path / "out"), env=None + ) + + +@pytest.fixture(autouse=True) +def _account(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AWS_ACCOUNT_ID", ACCOUNT) + + +class TestBoundedAwsName: + @staticmethod + def _digest(raw: str) -> str: + return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:8] + + def test_a_name_the_rule_accepts_is_returned_as_written(self): + # Nothing already deployed under such a name is renamed. + assert bounded_aws_name("fluid-workflow-sales_orders", LAMBDA_FUNCTION_NAME) == ( + "fluid-workflow-sales_orders" + ) + + def test_a_name_that_was_sanitised_ends_in_the_hash_of_the_raw_name(self): + raw = "fluid-workflow-sales.orders" + assert bounded_aws_name(raw, LAMBDA_FUNCTION_NAME) == ( + f"fluid-workflow-sales-orders-{self._digest(raw)}" + ) + + def test_two_ids_that_sanitise_alike_do_not_collide(self): + dotted = bounded_aws_name("fluid-workflow-a.b", LAMBDA_FUNCTION_NAME) + dashed = bounded_aws_name("fluid-workflow-a-b", LAMBDA_FUNCTION_NAME) + assert dashed == "fluid-workflow-a-b" + assert dotted != dashed + + def test_each_service_keeps_the_characters_it_allows(self): + raw = "fluid-x.y_z" + assert bounded_aws_name(raw, SCHEDULER_SCHEDULE_NAME) == raw # `.` is legal here + assert bounded_aws_name(raw, EVENTBRIDGE_RULE_NAME) == raw + assert bounded_aws_name(raw, LAMBDA_FUNCTION_NAME) == f"fluid-x-y_z-{self._digest(raw)}" + assert bounded_aws_name("a:b/c d", STATE_MACHINE_NAME) == ( + f"a-b-c-d-{self._digest('a:b/c d')}" + ) + assert bounded_aws_name("mwaa-a.b", MWAA_ENVIRONMENT_NAME) == ( + f"mwaa-a-b-{self._digest('mwaa-a.b')}" + ) + + @pytest.mark.parametrize( + "rule", + [ + LAMBDA_FUNCTION_NAME, + LAMBDA_STATEMENT_ID, + SCHEDULER_SCHEDULE_NAME, + EVENTBRIDGE_RULE_NAME, + MWAA_ENVIRONMENT_NAME, + STATE_MACHINE_NAME, + ], + ) + def test_a_long_name_is_cut_to_the_limit_deterministically(self, rule): + limit = rule[1] + raw = "fluid-workflow-" + "x" * 300 + name = bounded_aws_name(raw, rule) + assert len(name) == limit + assert bounded_aws_name(raw, rule) == name + + def test_two_long_names_sharing_a_prefix_do_not_collide(self): + a = bounded_aws_name("fluid-workflow-" + "x" * 80 + "-a", LAMBDA_FUNCTION_NAME) + b = bounded_aws_name("fluid-workflow-" + "x" * 80 + "-b", LAMBDA_FUNCTION_NAME) + assert len(a) == len(b) == 64 + assert a != b + + +class TestPlannedScheduleNames: + @pytest.mark.parametrize("contract_id", [DOTTED, LONG], ids=["dotted", "long"]) + def test_every_derived_name_is_valid_and_the_target_matches(self, contract_id): + actions = SchedulePlanner(ACCOUNT, REGION).plan_schedule_actions(_contract(contract_id)) + by_op = {a["op"]: a for a in actions} + function = by_op["lambda.ensure_function"] + schedule = by_op["eventbridge.ensure_schedule"] + permission = by_op["lambda.add_permission"] + + assert _LAMBDA_NAME.fullmatch(function["function_name"]) + assert _SCHEDULE_NAME.fullmatch(schedule["schedule_name"]) + assert _STATEMENT_ID.fullmatch(permission["statement_id"]) + assert schedule["target"]["arn"].endswith(f":function:{function['function_name']}") + assert permission["function_name"] == function["function_name"] + assert permission["source_arn"].endswith(f"/default/{schedule['schedule_name']}") + + def test_the_names_are_deterministic(self): + planner = SchedulePlanner(ACCOUNT, REGION) + assert planner.plan_schedule_actions(_contract(LONG)) == planner.plan_schedule_actions( + _contract(LONG) + ) + + def test_two_long_ids_with_a_shared_prefix_get_distinct_functions(self): + planner = SchedulePlanner(ACCOUNT, REGION) + a = planner.plan_schedule_actions(_contract(LONG + "_a"))[0]["function_name"] + b = planner.plan_schedule_actions(_contract(LONG + "_b"))[0]["function_name"] + assert a != b + + def test_trigger_lambdas_invoke_the_sanitised_workflow_function(self): + contract = _contract( + LONG, + triggers=[ + {"type": "s3", "id": "land.csv", "bucket": "b"}, + {"type": "sqs", "id": "q", "queue": "q"}, + {"type": "eventbridge", "id": "evt.1", "eventPattern": {"source": ["x"]}}, + ], + ) + actions = SchedulePlanner(ACCOUNT, REGION).plan_schedule_actions(contract) + workflow = next(a for a in actions if a["id"].startswith("lambda_workflow_")) + for action in actions: + if action["op"] == "lambda.ensure_function": + assert _LAMBDA_NAME.fullmatch(action["function_name"]) + if action["op"] == "lambda.add_permission": + assert _STATEMENT_ID.fullmatch(action["statement_id"]) + if action["id"].startswith("lambda_") and "_trigger_" in action["id"]: + assert f"FunctionName='{workflow['function_name']}'" in action["code"]["ZipFile"] + + @pytest.mark.parametrize("contract_id", [DOTTED, LONG], ids=["dotted", "long"]) + def test_generate_iac_emits_the_schedule(self, contract_id, tmp_path): + args = _args(_contract(contract_id), tmp_path) + assert generate_iac.run(args, logging.getLogger("t")) == 0 + module = json.loads((tmp_path / "out" / "main.tf.json").read_text(encoding="utf-8")) + resources = module["resource"] + ((fn_key, function),) = resources["aws_lambda_function"].items() + ((_, schedule),) = resources["aws_scheduler_schedule"].items() + ((_, permission),) = resources["aws_lambda_permission"].items() + + assert _LAMBDA_NAME.fullmatch(function["function_name"]) + assert _SCHEDULE_NAME.fullmatch(schedule["name"]) + assert _STATEMENT_ID.fullmatch(permission["statement_id"]) + # The schedule and the permission reference the emitted function, not a literal. + assert schedule["target"]["arn"] == f"${{aws_lambda_function.{fn_key}.arn}}" + assert permission["function_name"] == f"${{aws_lambda_function.{fn_key}.function_name}}" + + +class TestEveryNameStartsWithALetter: + """MWAA requires a leading letter; every name here carries a literal prefix + (``fluid-``, ``mwaa-``, ``Allow...``) that survives both the replacement and + the cut, whatever the id starts with.""" + + HOSTILE = "9 weird/id:" + "v" * 120 + + def test_names_built_from_a_hostile_id(self): + planner = SchedulePlanner(ACCOUNT, REGION) + triggers = [ + {"type": "s3", "id": "1.x", "bucket": "b"}, + {"type": "sqs", "id": "2.y", "queue": "q"}, + {"type": "eventbridge", "id": "3.z", "eventPattern": {}}, + ] + tasks = [{"taskId": "a", "type": "provider_action", "action": "x"}] + actions = planner.plan_schedule_actions( + _contract(self.HOSTILE, triggers=triggers, engine="step-functions", tasks=tasks) + ) + actions += planner.plan_schedule_actions(_contract(self.HOSTILE, engine="mwaa")) + limits = { + "function_name": 64, + "statement_id": 100, + "schedule_name": 64, + "rule_name": 64, + "state_machine_name": 80, + "environment_name": 80, + } + names = [] + for a in actions: + for key, limit in limits.items(): + if key in a: + assert len(a[key]) <= limit, (key, a[key]) + names.append(a[key]) + assert len(names) >= 10 + assert all(n[0].isalpha() for n in names), names + # MWAA: a letter first, then letters, digits, ``-`` and ``_``, up to 80. + env = [a["environment_name"] for a in actions if "environment_name" in a] + assert env and all(re.fullmatch(r"[a-zA-Z][0-9a-zA-Z\-_]{0,79}", n) for n in env) + + +class _Refuses: + """A stand-in native provider whose ``plan`` raises ``exc``.""" + + def __init__(self, exc: BaseException): + self._exc = exc + + def plan(self, contract, **_): + raise self._exc + + +@pytest.fixture +def planner_raises(monkeypatch: pytest.MonkeyPatch): + def install(exc: BaseException) -> None: + monkeypatch.setattr("fluid_build.cli._common.build_provider", lambda *a, **k: _Refuses(exc)) + + return install + + +class TestAPlannerThatRanAndRefusedIsLoud: + def test_a_plan_validation_failure_stops_native_actions(self, planner_raises): + planner_raises(PlanValidationError("Plan validation failed:\nAction 1: bad name")) + with pytest.raises(PlanValidationError): + generate_iac.native_actions(_contract(), logging.getLogger("t"), offline=True) + + def test_it_is_recognised_through_the_cause_chain(self, planner_raises): + try: + raise PlanValidationError("nope") + except PlanValidationError as inner: + try: + raise RuntimeError("wrapped") from inner + except RuntimeError as outer: + assert generate_iac._is_plan_validation_refusal(outer) is True + + def test_the_message_is_not_what_is_matched(self): + assert ( + generate_iac._is_plan_validation_refusal(ProviderError("Plan validation failed: ...")) + is False + ) + + def test_generate_iac_exits_non_zero(self, planner_raises, tmp_path): + planner_raises(PlanValidationError("Plan validation failed:\nAction 1: bad name")) + with pytest.raises(CLIError) as excinfo: + generate_iac.run(_args(_contract(), tmp_path), logging.getLogger("t")) + assert excinfo.value.event == "generate_iac_failed" + assert "Plan validation failed" in excinfo.value.context["error"] + assert not (tmp_path / "out" / "main.tf.json").exists() + + def test_the_apply_native_path_raises(self, planner_raises): + planner_raises(PlanValidationError("Plan validation failed:\nAction 1: bad name")) + with pytest.raises(PlanValidationError): + emit_module(object(), _contract(), object(), logging.getLogger("t")) + + def test_a_real_planner_refusal_stops_generate_iac(self, tmp_path): + contract = _contract() + contract["exposes"][0]["binding"]["location"]["bucket"] = "Not_A_Bucket" + with pytest.raises(CLIError) as excinfo: + generate_iac.run(_args(contract, tmp_path), logging.getLogger("t")) + assert excinfo.value.event == "generate_iac_failed" + assert "Plan validation failed" in excinfo.value.context["error"] + + def test_a_sovereignty_refusal_still_propagates(self, planner_raises): + from fluid_build.cli._errors import SovereigntyViolationError + + planner_raises( + SovereigntyViolationError.for_connector(connector="aws:us-east-1", jurisdiction="EU") + ) + with pytest.raises(SovereigntyViolationError): + generate_iac.native_actions(_contract(), logging.getLogger("t"), offline=True) + + @pytest.mark.parametrize("exc", [ImportError("no boto3"), RuntimeError("no credentials")]) + def test_a_planner_that_could_not_run_is_still_best_effort(self, planner_raises, exc): + planner_raises(exc) + assert generate_iac.native_actions(_contract(), logging.getLogger("t")) == [] + + +class TestThePlaceholderLambdaIsLabelled: + @staticmethod + def _warnings(caplog) -> list: + events = [] + for record in caplog.records: + if record.levelno == logging.WARNING and "schedule_placeholder_lambda" in ( + record.getMessage() + ): + events.append(json.loads(record.getMessage())) + return events + + def test_a_schedule_warns_that_the_lambda_is_a_placeholder(self, caplog): + logger = logging.getLogger("t") + with caplog.at_level(logging.WARNING, logger="t"): + generate_iac.native_actions(_contract(), logger, offline=True) + (event,) = self._warnings(caplog) + assert event["contract_id"] == DOTTED + assert "placeholder" in event["message"] + assert "does not run" in event["message"] + assert f"role/{'fluid-workflow-execution'}" in event["message"] + assert "role/EventBridgeSchedulerRole" in event["message"] + assert "must already exist" in event["message"] + + def test_apply_native_path_warns_too(self, caplog): + """``fluid apply`` plans through ``native_actions`` without ``offline``.""" + with caplog.at_level(logging.WARNING, logger="t"): + generate_iac.native_actions(_contract(), logging.getLogger("t")) + assert len(self._warnings(caplog)) == 1 + + def test_generate_iac_shows_it(self, caplog, tmp_path): + with caplog.at_level(logging.WARNING): + generate_iac.run(_args(_contract(), tmp_path), logging.getLogger("t")) + assert len(self._warnings(caplog)) == 1 + + def test_no_schedule_no_warning(self, caplog): + contract = _contract() + del contract["orchestration"] + with caplog.at_level(logging.WARNING, logger="t"): + generate_iac.native_actions(contract, logging.getLogger("t"), offline=True) + assert self._warnings(caplog) == [] + + def test_an_mwaa_schedule_plans_no_placeholder_lambda(self, caplog): + with caplog.at_level(logging.WARNING, logger="t"): + generate_iac.native_actions( + _contract(engine="mwaa"), logging.getLogger("t"), offline=True + ) + assert self._warnings(caplog) == [] + + +class TestContractValuesNeverBecomeGeneratedLambdaCode: + """A contract value reaches the generated Lambda source only as a literal. + + Spliced inside quotes (``'{contract_id}'``, ``json.loads('''{tasks}''')``), a + task param such as ``'''+__import__('os').system('id')+'''`` closed the + literal and ran on every schedule tick under ``fluid-workflow-execution``, + and an id with a trailing newline left a Lambda that was a SyntaxError. + """ + + EVIL = "'''+__import__('os').system('id')+'''" + HOSTILE_IDS = [ + EVIL, + "a'b\"c'''d\"\"\"e", + "back\\slash\\'", + "line\nbreak", + "trailing.newline\n", + "${aws_lambda_function.x.arn}{braces}{{x}}", + ] + #: Calls the generated sources are expected to make, by final name. + EXPECTED_CALLS = { + "loads", + "dumps", + "getLogger", + "setLevel", + "get", + "info", + "set", + "list", + "all", + "add", + "remove", + "client", + "invoke", + "RuntimeError", + "len", + } + + @staticmethod + def _sources(contract_id: str, param: str) -> Dict[str, str]: + planner = SchedulePlanner(ACCOUNT, REGION) + task = { + "taskId": param, + "type": "provider_action", + "action": "aws.glue.run_job", + "params": {"k": param}, + "dependsOn": [], + } + contract = {"id": contract_id, "name": "n"} + orchestration = {"tasks": [task]} + return { + "workflow": planner._generate_workflow_lambda_code(contract, orchestration)["ZipFile"], + "s3": planner._generate_s3_trigger_lambda_code(contract, {})["ZipFile"], + "dynamodb": planner._generate_dynamodb_trigger_lambda_code(contract, {})["ZipFile"], + "event": planner._generate_event_trigger_lambda_code(contract, {})["ZipFile"], + "sqs": planner._generate_sqs_trigger_lambda_code(contract, {})["ZipFile"], + } + + @staticmethod + def _callee(call: ast.Call) -> str: + func = call.func + return func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", "?") + + @pytest.mark.parametrize("contract_id", HOSTILE_IDS, ids=range(len(HOSTILE_IDS))) + @pytest.mark.parametrize("param", [EVIL, "p'\"\\\n${x}"], ids=["inject", "punctuation"]) + def test_the_source_parses_calls_only_what_it_should_and_round_trips(self, contract_id, param): + for kind, source in self._sources(contract_id, param).items(): + tree = ast.parse(source) # a SyntaxError here is the trailing-newline defect + callees = {self._callee(n) for n in ast.walk(tree) if isinstance(n, ast.Call)} + assert callees <= self.EXPECTED_CALLS, (kind, callees - self.EXPECTED_CALLS) + + strings = set() + for node in ast.walk(tree): + if isinstance(node, ast.Dict): + for key, value in zip(node.keys, node.values, strict=True): + if ( + isinstance(key, ast.Constant) + and key.value == "contract_id" + and isinstance(value, ast.Constant) + ): + strings.add(value.value) + if kind != "workflow": + assert strings == {contract_id}, kind + functions = [ + ast.literal_eval(kw.value) + for n in ast.walk(tree) + if isinstance(n, ast.Call) + for kw in n.keywords + if kw.arg == "FunctionName" + ] + assert functions == [SchedulePlanner.workflow_function_name(contract_id)] + + @pytest.mark.parametrize("contract_id", HOSTILE_IDS, ids=range(len(HOSTILE_IDS))) + def test_the_workflow_lambda_round_trips_its_tasks_and_default_id(self, contract_id): + param = self.EVIL + source = self._sources(contract_id, param)["workflow"] + boto3 = types.ModuleType("boto3") + namespace: Dict[str, Any] = {} + sys.modules["boto3"], saved = boto3, sys.modules.get("boto3") + try: + exec(compile(source, "", "exec"), namespace) # noqa: S102 + finally: + if saved is None: + del sys.modules["boto3"] + else: + sys.modules["boto3"] = saved + assert namespace["TASKS"] == [ + {"taskId": param, "action": "aws.glue.run_job", "params": {"k": param}, "dependsOn": []} + ] + # The handler's fallback id is the exact id, and running it runs no payload. + handler_defaults = [ + ast.literal_eval(n.args[1]) + for n in ast.walk(ast.parse(source)) + if isinstance(n, ast.Call) + and self._callee(n) == "get" + and len(n.args) == 2 + and isinstance(n.args[0], ast.Constant) + and n.args[0].value == "contract_id" + ] + assert handler_defaults == [contract_id] + assert namespace["handler"]({}, None)["statusCode"] == 200 + + +class TestEveryPlanningFailureIsARefusalNotADrop: + def test_an_orchestration_task_the_planner_rejects_stops_generate_iac(self, tmp_path): + contract = _contract( + engine="airflow", + tasks=[ + {"taskId": "a", "type": "provider_action", "action": "gcp.bq.run", "params": {}} + ], + ) + with pytest.raises(CLIError) as excinfo: + generate_iac.run(_args(contract, tmp_path), logging.getLogger("t")) + assert excinfo.value.event == "generate_iac_failed" + assert "Orchestration planning failed" in excinfo.value.context["error"] + + def test_a_schedule_planner_failure_is_a_plan_validation_error(self, monkeypatch): + from fluid_build.providers.aws.provider import AwsProvider + + def boom(*args, **kwargs): + raise ValueError("bad schedule") + + monkeypatch.setattr("fluid_build.providers.aws.plan.schedule.plan_schedule_actions", boom) + provider = AwsProvider(account_id=ACCOUNT, region=REGION) + with pytest.raises(PlanValidationError, match="Schedule planning failed"): + provider._plan_schedule(_contract()) + + +class TestS3BucketNames: + @pytest.fixture + def validate(self): + from fluid_build.providers.aws.util.validation import ResourceValidator + + return ResourceValidator(ACCOUNT, REGION).validate_s3_bucket_name + + @pytest.mark.parametrize( + "bucket", ["acme.orders.lake", "my-bucket.v2", "a1b", "data.2026-lake", "x" * 63] + ) + def test_a_name_s3_allows_is_accepted(self, validate, bucket): + assert validate(bucket) == (True, None) + + @pytest.mark.parametrize( + "bucket", + [ + "a..b", + "a.-b", + "a-.b", + ".abc", + "abc.", + "-abc", + "192.168.0.1", + "Upper.case", + "has_underscore", + "ab", + "x" * 64, + "xn--bucket", + ], + ) + def test_a_name_s3_refuses_is_refused(self, validate, bucket): + assert validate(bucket)[0] is False + + def test_generate_iac_accepts_a_dotted_bucket(self, tmp_path): + contract = _contract() + contract["exposes"][0]["binding"]["location"]["bucket"] = "acme.orders.lake" + assert generate_iac.run(_args(contract, tmp_path), logging.getLogger("t")) == 0 + + +class TestTheUnresolvedAccountMessage: + @staticmethod + def _provider_without_account(monkeypatch): + from fluid_build.providers.aws.provider import AwsProvider + + monkeypatch.setattr( + "fluid_build.providers.aws.util.config.resolve_account_and_region", + lambda account_id=None, region=None: (None, REGION), + ) + return AwsProvider(region=REGION) + + def test_missing_boto3_is_named_with_the_install_command(self, monkeypatch): + provider = self._provider_without_account(monkeypatch) + monkeypatch.setattr("fluid_build.providers.aws.util.config._boto3_missing", lambda: True) + with pytest.raises(AwsAccountUnresolvedError) as excinfo: + provider.plan(_contract()) + message = str(excinfo.value) + assert "boto3 is not installed" in message + assert "pip install 'data-product-forge[aws]'" in message + assert "AWS_ACCOUNT_ID" in message + + def test_with_boto3_present_it_points_at_credentials(self, monkeypatch): + provider = self._provider_without_account(monkeypatch) + monkeypatch.setattr("fluid_build.providers.aws.util.config._boto3_missing", lambda: False) + with pytest.raises(AwsAccountUnresolvedError) as excinfo: + provider.plan(_contract()) + message = str(excinfo.value) + assert "sts:GetCallerIdentity" in message + assert "boto3" not in message + + def test_a_contract_without_orchestration_still_applies_its_exposes(self, monkeypatch): + """Nothing in it needs the native plan, and tofu resolves the account itself, + so the planner only reports that it could not run, as on main.""" + provider = self._provider_without_account(monkeypatch) + contract = _contract() + del contract["orchestration"] + with pytest.raises(ProviderError) as excinfo: + provider.plan(contract) + assert not isinstance(excinfo.value, AwsAccountUnresolvedError) + monkeypatch.setattr("fluid_build.cli._common.build_provider", lambda *a, **k: provider) + assert generate_iac.native_actions(contract, logging.getLogger("t")) == [] + + def test_sovereignty_is_checked_before_the_account(self, monkeypatch): + provider = self._provider_without_account(monkeypatch) + contract = _contract() + del contract["orchestration"] + contract["sovereignty"] = { + "jurisdiction": "EU", + "allowedRegions": ["eu-central-1"], + "enforcementMode": "strict", + } + monkeypatch.setattr("fluid_build.cli._common.build_provider", lambda *a, **k: provider) + with pytest.raises(Exception) as excinfo: + generate_iac.native_actions(contract, logging.getLogger("t")) + assert generate_iac._is_sovereignty_refusal(excinfo.value) diff --git a/tests/iac/test_iac_aws_unemitted_ops.py b/tests/iac/test_iac_aws_unemitted_ops.py index 89343330b..f9757db83 100644 --- a/tests/iac/test_iac_aws_unemitted_ops.py +++ b/tests/iac/test_iac_aws_unemitted_ops.py @@ -300,9 +300,9 @@ def _cyclic_tasks() -> Dict[str, Any]: class TestThePlannerFailing: """The planner failing on another field does not let the environment drop. - ``native_actions`` returns ``[]`` when the native planner raises (it logs - that at DEBUG), so the emitter saw no ``mwaa.ensure_environment`` and wrote - the data plane alone, exit 0. + ``native_actions`` returns ``[]`` when the native planner raises something + that is not a refusal (it logs that at DEBUG), so the emitter saw no + ``mwaa.ensure_environment`` and wrote the data plane alone, exit 0. """ @pytest.mark.parametrize("make", [_upper_case_table, _cyclic_tasks], ids=["glue-name", "cycle"]) @@ -322,7 +322,12 @@ def test_generate_iac_is_refused(self, no_aws, monkeypatch, tmp_path): generate_iac.run( _args(_write(tmp_path, _upper_case_table()), out), logging.getLogger("t") ) - assert excinfo.value.context["kind"] == "mwaa-environment-not-emitted" + # The planner ran and refused its own plan, so ``native_actions`` raises + # it (``PlanValidationError``) before the emitter's backstop is reached; + # the cycle case, a planner failure that is not a refusal of the plan, + # still gets that backstop (above). + assert excinfo.value.event == "generate_iac_failed" + assert "Plan validation failed" in excinfo.value.context["error"] assert not out.exists() @pytest.mark.parametrize( diff --git a/tests/iac/test_iac_generate_offline.py b/tests/iac/test_iac_generate_offline.py index d432c5a95..3dc3a2f34 100644 --- a/tests/iac/test_iac_generate_offline.py +++ b/tests/iac/test_iac_generate_offline.py @@ -43,6 +43,7 @@ from fluid_build.cli import generate_iac, main from fluid_build.cli._common import CLIError +from fluid_build.providers.aws.util.config import AwsAccountUnresolvedError pytestmark = [pytest.mark.unit, pytest.mark.provider] @@ -202,7 +203,10 @@ class TestApplyStillResolvesTheAccount: (``cli/_apply_opentofu_engine.py``); only ``generate iac`` passes ``offline``.""" def test_the_apply_call_asks_sts_for_a_missing_account(self, clients): - generate_iac.native_actions(_scheduled(), logging.getLogger("t")) + # The ``clients`` guard makes the lookup fail, so no account resolves: + # refused by name, before any resource name is derived from ``None``. + with pytest.raises(AwsAccountUnresolvedError, match="AWS_ACCOUNT_ID"): + generate_iac.native_actions(_scheduled(), logging.getLogger("t")) assert clients == ["sts"] def test_the_offline_call_plans_without_asking(self, clients): From bfea38279b4630cacc536e4068693123745a07cc Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:34:32 +0100 Subject: [PATCH 17/26] security: schema validation let a value with a trailing newline through a pattern (#799) * security: schema validation let a value with a trailing newline through a pattern JSON Schema writes `pattern` in ECMA-262, where `$` matches only at the end of the value. jsonschema runs it with Python's re.search, whose `$` also matches before a final newline, so `fluid validate` accepted `id: "orders\n"` (what an `id: |` block scalar produces) under every bundled schema, 0.7.1 to 0.7.7. An editor validating the same schema in JavaScript refuses it. schema_manager.schema_validator(schema) extends the declared dialect's validator with a `pattern` keyword that rewrites each `$` anchor to `\Z`. Contract and fragment validation (_validate_with_jsonschema) and bundle validation (forge.core.validators.validate_json_schema) use it. The schemas are unchanged, so cached and fetched ones are covered too, and forge_docs, the fluid spec and Command Center's models need no change. When the line break is the only problem, the error says so. patternProperties keys keep jsonschema's re.search, as its own additionalProperties and _locate_error read them. No FLUID schema uses the keyword, and a test fails if one starts to. The catalogAuth token check, `not: {pattern: ...(?:_|$)}`, now reads `$` as ECMA-262 does too; the name pattern beside it already refuses any value that ends in a newline, pinned in test_catalog_auth_schema.py. Borrowed: the keyword override from check-jsonschema (jsonschema.validators.extend), and the `$` to `\Z` rewrite from fastjsonschema 2.21.2 and js-regex 1.0.1. Their (? block keeps + and |- or >- drops)`. A `sink.catalogAuth` field gets no hint, since its value is never shown. + The schemas themselves are unchanged. + ## [0.26.0] — 2026-11-01 ### Added diff --git a/fluid_build/forge/core/validators.py b/fluid_build/forge/core/validators.py index 83472ccf4..2bab29bca 100644 --- a/fluid_build/forge/core/validators.py +++ b/fluid_build/forge/core/validators.py @@ -390,9 +390,14 @@ def validate_json_schema( ) ] schema = schema_manager.get_schema(version) - try: - import jsonschema # type: ignore - except ImportError: # pragma: no cover — jsonschema is a hard dep upstream + from fluid_build.schema_manager import ( + JSONSCHEMA_AVAILABLE, + format_schema_error, + schema_validator, + ) + + # The flag schema_validator needs: a package that is found can still fail to import. + if not JSONSCHEMA_AVAILABLE: return [ ValidationIssue( file=path, @@ -405,9 +410,7 @@ def validate_json_schema( # Honour the schema's declared $schema; Draft 7 silently ignores # newer keywords, which drops the constraints they express. - from fluid_build.schema_manager import format_schema_error - - validator = jsonschema.validators.validator_for(schema)(schema) + validator = schema_validator(schema) errors = sorted(validator.iter_errors(doc), key=lambda e: list(e.absolute_path)) for err in errors: loc = "/".join(str(p) for p in err.absolute_path) or "" diff --git a/fluid_build/schema_manager.py b/fluid_build/schema_manager.py index a757b8775..1b992d7ee 100644 --- a/fluid_build/schema_manager.py +++ b/fluid_build/schema_manager.py @@ -21,13 +21,14 @@ from __future__ import annotations +import functools import json import logging import re from dataclasses import dataclass, field from datetime import datetime, timedelta from pathlib import Path -from typing import Any, Callable, Dict, FrozenSet, List, Optional, Union +from typing import Any, Callable, Dict, FrozenSet, Iterator, List, Optional, Union import yaml @@ -110,6 +111,7 @@ def _locate_error(error: Any, locate: Callable[..., Optional[str]]) -> Optional[ known = schema.get("properties") or {} patterns = list((schema.get("patternProperties") or {}).keys()) for key in error.instance: + # re.search, as jsonschema's additionalProperties reads these keys. if key not in known and not any(re.search(p, str(key)) for p in patterns): return locate([*path, key]) if error.validator == "required": @@ -132,6 +134,72 @@ def _error_path(error: Any, locate: Optional[Callable[..., Optional[str]]] = Non return f"{where} ({path})" if where else path +def _end_anchored(pattern: str) -> str: + r"""``pattern`` with each ``$`` anchor written ``\Z``, the end of the value. + + JSON Schema writes ``pattern`` in ECMA-262, where ``$`` matches only at the + end of the value. Python's ``$`` also matches before a final ``\n``, so + jsonschema's ``re.search`` let ``"orders\n"`` through ``^[a-z_]+$``. + fastjsonschema and js-regex make the same rewrite with a ``(? re.Pattern[str]: + return re.compile(_end_anchored(pattern)) + + +def _pattern_keyword(validator: Any, pattern: str, instance: Any, schema: Any) -> Iterator[Any]: + """jsonschema's ``pattern`` keyword, with ``$`` read as ECMA-262 reads it.""" + if not validator.is_type(instance, "string") or _compiled_pattern(pattern).search(instance): + return + message = f"{instance!r} does not match {pattern!r}" + if re.search(pattern, instance): # only the final line break fails + message += " (it ends in a line break, which a YAML | or > block keeps and |- or >- drops)" + yield jsonschema.ValidationError(message) + + +@functools.lru_cache(maxsize=None) +def _end_anchored_validator(validator_class: Any) -> Any: + return jsonschema.validators.extend(validator_class, {"pattern": _pattern_keyword}) + + +def schema_validator(schema: Dict[str, Any]) -> Any: + """A jsonschema validator for ``schema``, in the dialect it declares, whose + ``pattern`` ends at the end of the value (:func:`_end_anchored`). Validate + against a FLUID schema through this, never ``validator_for`` directly. + + The keyword is replaced the way check-jsonschema replaces it, with + ``jsonschema.validators.extend``. Two places keep jsonschema's own + ``re.search``: ``patternProperties`` keys, as its ``additionalProperties`` + and :func:`_locate_error` read them, and a subschema that declares its own + ``$schema`` (or a ``$ref`` to the root, which does), for which jsonschema + picks that dialect's own class. No FLUID schema has either; a test fails if + one does. + """ + return _end_anchored_validator(jsonschema.validators.validator_for(schema))(schema) + + def _schema_version_sort_key(version: str) -> tuple[int, int, int]: match = VERSION_PATTERN.match(version) if not match: @@ -736,7 +804,7 @@ def _validate_with_jsonschema( # 2020-12; validating them with Draft 7 worked only because none # has yet used a 2020-12-only keyword. The first one to do so # would have been accepted without error. - validator = jsonschema.validators.validator_for(schema)(schema) + validator = schema_validator(schema) # Validate and collect errors errors = sorted(validator.iter_errors(contract), key=lambda e: e.path) diff --git a/tests/test_catalog_auth_schema.py b/tests/test_catalog_auth_schema.py index 30a9527b0..06275bb65 100644 --- a/tests/test_catalog_auth_schema.py +++ b/tests/test_catalog_auth_schema.py @@ -164,6 +164,8 @@ def test_ga_075_refuses_the_key(): ({**_OAUTH2, "credentialEnv": "LK_CRED\n"}, "credentialEnv: does not match"), ({"type": "bearer", "tokenEnv": "UC_TOKEN\n"}, "tokenEnv: does not match"), ({"type": "bearer", "tokenEnv": _PAT}, "tokenEnv: must not match"), + # The token check's "$" no longer matches before the "\n"; the name pattern refuses it. + ({"type": "bearer", "tokenEnv": _PAT + "\n"}, "tokenEnv: does not match"), ( {**_OAUTH2, "credentialEnv": "AKIAIOSFODNN7EXAMPLE"}, # pragma: allowlist secret "credentialEnv: must not match", @@ -193,9 +195,10 @@ def test_076_refuses_a_malformed_catalog_auth(catalog_auth, expected): ({"type": "bearer", "tokenEnv": "T", "credentialEnv": _SECRET}, _SECRET), ({"type": _SECRET}, _SECRET), ({"type": "bearer", "tokenEnv": _PAT}, _PAT), + ({"type": "bearer", "tokenEnv": _PAT + "\n"}, _PAT), ({**_OAUTH2, "oauth2ServerUri": _USERINFO_URI}, "2OR3eRvYfSZ"), ], - ids=["credentialEnv", "tokenEnv", "wrong-type-field", "type", "pat", "userinfo"], + ids=["credentialEnv", "tokenEnv", "wrong-type-field", "type", "pat", "pat-lf", "userinfo"], ) def test_a_schema_error_never_quotes_the_value(catalog_auth, secret): errors = _errors("0.7.6", catalog_auth) @@ -224,6 +227,7 @@ def test_schema_and_parser_share_the_variable_name_rule(): {**_OAUTH2, "credentialEnv": "LK_CRED\n"}, {"type": "bearer", "tokenEnv": "UC_TOKEN\n"}, {"type": "bearer", "tokenEnv": _PAT}, + {"type": "bearer", "tokenEnv": _PAT + "\n"}, {"type": "bearer", "tokenEnv": "S3_CREDENTIAL_V2"}, {"type": "bearer", "tokenEnv": "LAKEKEEPERCATALOGCREDENTIALPROD"}, {**_OAUTH2, "oauth2ServerUri": _USERINFO_URI}, diff --git a/tests/test_schema_pattern_end_anchor.py b/tests/test_schema_pattern_end_anchor.py new file mode 100644 index 000000000..6b12793da --- /dev/null +++ b/tests/test_schema_pattern_end_anchor.py @@ -0,0 +1,267 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A schema ``pattern`` ends where the value ends, as ECMA-262 reads ``$``. + +JSON Schema writes ``pattern`` in the ECMA-262 dialect, where ``$`` (no ``m`` +flag) matches only at the end of the input. jsonschema runs it with Python's +``re.search``, and Python's ``$`` also matches before a final ``\\n``. So +``fluid validate`` accepted ``id: "orders\\n"``, which a YAML ``|`` block +scalar produces, while an editor validating against the same schema in +JavaScript refused it. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from re import sub +from typing import Any, Dict, Iterator, List + +import pytest +import yaml + +import fluid_build +from fluid_build.schema_manager import FluidSchemaManager, _end_anchored, schema_validator + +try: # Python 3.11+ + from re import _parser as sre_parse +except ImportError: # Python 3.10 + import sre_parse # type: ignore[no-redef] + +pytestmark = [pytest.mark.unit] + +_VERSIONS = list(FluidSchemaManager.BUNDLED_VERSIONS) +_SCHEMA_DIR = Path(fluid_build.__file__).parent / "schemas" +_LINE_BREAK_HINT = " (it ends in a line break, which a YAML | or > block keeps and |- or >- drops)" + + +def _expose(expose_id: str = "orders") -> Dict[str, Any]: + return { + "exposeId": expose_id, + "kind": "table", + "binding": { + "platform": "local", + "format": "csv", + "location": {"path": "data/orders.csv"}, + }, + "contract": {"schema": [{"name": "id", "type": "integer"}]}, + } + + +def _contract(version: str, product_id: str = "orders") -> Dict[str, Any]: + return { + "fluidVersion": version, + "kind": "DataProduct", + "id": product_id, + "name": "Orders", + "domain": "sales", + "metadata": {"layer": "Bronze", "owner": {"team": "data"}}, + "exposes": [_expose()], + } + + +def _errors(contract: Dict[str, Any]) -> List[str]: + version = contract["fluidVersion"] + result = FluidSchemaManager().validate_contract(contract, version, offline_only=True) + return [] if result.is_valid else list(result.errors) + + +def test_every_bundled_version_is_checked() -> None: + """Guard the guard: an empty version list would pass every case below.""" + assert len(_VERSIONS) >= 7, _VERSIONS + + +@pytest.mark.parametrize("version", _VERSIONS) +def test_an_id_that_ends_in_a_newline_is_refused(version: str) -> None: + assert _errors(_contract(version)) == [] + errors = _errors(_contract(version, "orders\n")) + assert any(e.startswith("id: ") for e in errors), errors + + +def test_fluid_validate_refuses_a_block_scalar_id(tmp_path: Path, capsys: Any) -> None: + """``id: |`` keeps the line break, so the value is ``"orders\\n"``.""" + from fluid_build.cli import main + + body = yaml.safe_dump({k: v for k, v in _contract("0.7.6").items() if k != "id"}) + contract = tmp_path / "contract.fluid.yaml" + contract.write_text(f"id: |\n orders\n{body}", encoding="utf-8") + assert yaml.safe_load(contract.read_text(encoding="utf-8"))["id"] == "orders\n" + + rc = main(["validate", str(contract), "--offline", "--cache-dir", str(tmp_path / "cache")]) + + captured = capsys.readouterr() + text = captured.out + captured.err + assert rc == 1, text + assert "id" in text and "does not match" in text, text + + +def test_fluid_validate_refuses_a_fragment_whose_expose_id_ends_in_a_newline( + tmp_path: Path, capsys: Any +) -> None: + from fluid_build.cli import main + + fragment = tmp_path / "fragments" / "orders.yaml" + fragment.parent.mkdir(parents=True) + fragment.write_text(yaml.safe_dump(_expose("orders\n")), encoding="utf-8") + argv = ["validate", "--offline", "--cache-dir", str(tmp_path / "cache")] + argv += ["--schema-version", "0.7.6", "--fragment", "expose", str(fragment)] + + rc = main(argv) + + captured = capsys.readouterr() + text = captured.out + captured.err + assert rc == 1, text + assert "exposeId" in text and "does not match" in text, text + + +def test_bundle_validation_refuses_an_id_that_ends_in_a_newline() -> None: + """``fluid validate .tgz`` checks the resolved contract through + ``forge.core.validators``, not ``FluidSchemaManager.validate_contract``.""" + from fluid_build.forge.core.validators import validate_json_schema + + def issues(product_id: str) -> List[str]: + found = validate_json_schema( + "contract.resolved.yaml", + _contract("0.7.6", product_id), + schema_manager=FluidSchemaManager(), + ) + return [i.message for i in found] + + assert issues("orders") == [] + messages = issues("orders\n") + assert any(m.startswith("at id: ") for m in messages), messages + + +def test_bundle_validation_without_jsonschema_is_an_error(monkeypatch: Any) -> None: + """The check reads the flag ``schema_validator`` needs: a jsonschema that is found but + fails to import is the library-missing error, not a skipped check.""" + from fluid_build.forge.core.validators import validate_json_schema + + monkeypatch.setattr("fluid_build.schema_manager.JSONSCHEMA_AVAILABLE", False) + issues = validate_json_schema( + "contract.resolved.yaml", _contract("0.7.6"), schema_manager=FluidSchemaManager() + ) + assert [(i.code, i.severity) for i in issues] == [("SCHEMA-LIB-MISSING", "error")] + + +def test_the_error_says_the_value_ends_in_a_line_break() -> None: + errors = _errors(_contract("0.7.6", "orders\n")) + assert len(errors) == 1, errors + assert errors[0].startswith("id: 'orders\\n' does not match ") + assert errors[0].endswith(_LINE_BREAK_HINT) + + +@pytest.mark.parametrize("product_id", ["orders!", "orders!\n"]) +def test_a_value_wrong_for_another_reason_gets_no_line_break_hint(product_id: str) -> None: + errors = _errors(_contract("0.7.6", product_id)) + assert errors and not any("line break" in e for e in errors), errors + + +def test_the_json_schema_test_suite_case() -> None: + """JSON-Schema-Test-Suite, ``tests/draft2020-12/optional/ecmascript-regex.json``: + "ECMA 262 regex $ does not match trailing newline".""" + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^abc$", + } + assert schema_validator(schema).is_valid("abc") + assert not schema_validator(schema).is_valid("abc\n") + + +def test_a_not_pattern_reads_the_end_anchor_the_same_way() -> None: + """``not: {pattern: ...}`` now matches fewer values, as in ECMA-262: a deny rule that + has to catch a line break must name one (catalogAuth refuses ``[\\x00-\\x1f\\x7f]``).""" + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": {"pattern": "^x$"}, + } + assert not schema_validator(schema).is_valid("x") + assert schema_validator(schema).is_valid("x\n") + + +# ── the rewrite ───────────────────────────────────────────────────────────── + + +def _tree(pattern: str) -> str: + """Python's own parse: ``[(AT, AT_BEGINNING), (LITERAL, 97), (AT, AT_END)]``.""" + return repr(sre_parse.parse(pattern)) + + +def _only_end_anchors_changed(pattern: str) -> bool: + """Python's parser reads the rewrite as ``pattern`` with each ``$`` (``AT_END``) + made ``\\Z`` (``AT_END_STRING``), and nothing else changed.""" + expected = sub(r"\bAT_END\b", "AT_END_STRING", _tree(pattern)) + return _tree(_end_anchored(pattern)) == expected + + +@pytest.mark.parametrize( + "pattern, rewritten", + [ + (r"^abc$", r"^abc\Z"), + (r"^a$|^b$", r"^a\Z|^b\Z"), + (r"(?:^|_)[A-Za-z0-9]{20,}(?:_|$)", r"(?:^|_)[A-Za-z0-9]{20,}(?:_|\Z)"), + (r"^P(?!$)(\d+D)?$", r"^P(?!\Z)(\d+D)?\Z"), + (r"^\$\{$", r"^\$\{\Z"), # an escaped "$" is a dollar sign + (r"^a\\$", r"^a\\\Z"), # after an escaped backslash it is the anchor + (r"^[$]+$", r"^[$]+\Z"), # in a set it is a dollar sign + (r"^[\]$]+$", r"^[\]$]+\Z"), + (r"^[]$]+$", r"^[]$]+\Z"), # a "]" first in a set does not close it + (r"^[^]$]+$", r"^[^]$]+\Z"), + (r"^[a-]$", r"^[a-]\Z"), + ], +) +def test_the_rewrite_changes_only_the_end_anchors(pattern: str, rewritten: str) -> None: + assert _end_anchored(pattern) == rewritten + assert _only_end_anchors_changed(pattern) + + +def _nodes(node: Any) -> Iterator[Dict[str, Any]]: + """Every object in a schema document, the root first.""" + if isinstance(node, dict): + yield node + for value in node.values(): + yield from _nodes(value) + elif isinstance(node, list): + for value in node: + yield from _nodes(value) + + +def test_every_bundled_schema_pattern_is_rewritten_anchor_for_anchor() -> None: + """The scan reads escapes and sets as Python does, checked against Python's own + parser for every ``pattern`` the bundled schemas hold.""" + patterns = { + node["pattern"] + for path in _SCHEMA_DIR.glob("fluid-*schema-*.json") + for node in _nodes(json.loads(path.read_text(encoding="utf-8"))) + if isinstance(node.get("pattern"), str) + } + assert len(patterns) >= 20, patterns # the glob found the schemas + assert [p for p in patterns if not _only_end_anchors_changed(p)] == [] + + +@pytest.mark.parametrize( + "path", sorted(_SCHEMA_DIR.glob("fluid-schema-*.json")), ids=lambda path: path.name +) +def test_every_pattern_of_the_schema_is_checked_with_the_rewrite(path: Path) -> None: + """jsonschema picks the validator class again for each subschema it enters, by + that subschema's ``$schema``. One that declares it, or a ``$ref`` to the root, + which does, would be checked with jsonschema's own ``pattern`` again. + ``patternProperties`` keys are not rewritten (``schema_validator`` says why), so + a schema that starts to use the keyword has to come back here.""" + below_root = list(_nodes(json.loads(path.read_text(encoding="utf-8"))))[1:] + assert [n for n in below_root if isinstance(n.get("$schema"), str)] == [] + assert [n for n in below_root if n.get("$ref") == "#"] == [] + assert [n for n in below_root if "patternProperties" in n] == [] From fe0d711c3ad1e1658b820f1a8e54588cd58761d6 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:45:13 +0100 Subject: [PATCH 18/26] fix(welcome-scan): cap the scan at its budget and keep cold imports off the clock (#800) run_welcome_scan documented a 50ms scan that stays within _TOTAL_TIMEOUT_S, but the budget only decided which results were kept: the ThreadPoolExecutor context joined every probe thread. Run from a directory that holds many repos, the specialization probe's rglob took 177s and the whole scan with it. Each probe's wait window also started when the collector reached it, so a cold first import counted against it. On a loaded machine the AI probe (a lazy pydantic import inside the AI-config load) missed its window and the scan reported "AI not configured" although it is. - Probes run on daemon threads, one Future each. The scan returns at _TOTAL_TIMEOUT_S and abandons stragglers. An executor is deliberately not used: its non-daemon workers are joined at interpreter exit, so an abandoned probe would keep the CLI alive. - _warm_probe_imports imports the modules the probes load lazily while the probes run, then the budget starts. A test fails if a probe imports something that is not listed in _WARM_MODULES. - _PROBE_TIMEOUT_S is removed: once the probes start together it can never bind before the total. _TOTAL_TIMEOUT_S (1.0s) is the single budget. The dbt probe gets its own _DBT_VERSION_TIMEOUT_S (0.32s, was 0.4 * 0.8), which must stay below the budget so the child is killed by its own timeout rather than orphaned. - scan_duration_ms uses a monotonic clock and covers the whole call. - The "<50ms" claims (module docstring, forge_modes comment, CLAUDE.md) are corrected, and the deterministic_welcome_scan fixture drops the removed constant. - tests/test_welcome_scan_budget.py pins the cap, the exit behaviour of an abandoned probe, the cold-import accounting, the warm list and the dbt cap. - Review follow-up: _spawn_probe records only Exception; SystemExit and KeyboardInterrupt end the probe thread and are dropped at the deadline. --- CLAUDE.md | 11 +- fluid_build/cli/_welcome_scan.py | 136 +++++++++++---- fluid_build/cli/forge_modes.py | 6 +- tests/conftest.py | 24 ++- tests/test_forge_mode_picker.py | 4 +- tests/test_welcome_scan.py | 2 +- tests/test_welcome_scan_budget.py | 266 ++++++++++++++++++++++++++++++ 7 files changed, 398 insertions(+), 51 deletions(-) create mode 100644 tests/test_welcome_scan_budget.py diff --git a/CLAUDE.md b/CLAUDE.md index 3ec3c630e..551f490d7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -118,7 +118,7 @@ The `feat/cli-pipeline-and-publish-hardening` branch (15 commits) landed the **1 A multi-phase push landed comprehensive UX + architecture upgrades on `feat/source-aligned-acquisition`. Key surfaces and where they live: - **Mode picker (`fluid forge`).** Bare invocation surfaces a 5-mode menu (AI / Compose / Refine / Template / Blank) instead of dropping into AI. Lives at `cli/_forge_mode_picker.py`. Skipped via `FLUID_FORGE_NO_PICKER=1`. Picker default is highlighted by the welcome scan; user always sees the menu. -- **Welcome scan.** Detect-first parallel scan runs in <50ms before any prompt — workspace state, AI configured, CLIs installed, sample data, cloud creds, return-user state. Lives at `cli/_welcome_scan.py`. `~/.fluid/usage.json` carries `forge_count`. +- **Welcome scan.** Detect-first parallel scan runs before any prompt (budget: see the 2026-11-03 section) — workspace state, AI configured, CLIs installed, sample data, cloud creds, return-user state. Lives at `cli/_welcome_scan.py`. `~/.fluid/usage.json` carries `forge_count`. - **Mode-aware interviews.** `forge_copilot_interview.py::run_adaptive_copilot_interview` short-circuits to `_run_compose_interview` (3 questions max for `--from-product`) or `_run_refine_interview` (1 question for `--refine`) when the runtime resolved upstreams or loaded an existing contract. Standard mode runs the world-class bootstrap (below). - **World-class bootstrap.** New default fresh-product interview at `cli/_world_class_interview.py`. Inferences first (`InterviewSignals`), examples in every prompt, productType-first, `:auto` escape, schema-coverage gate. Toggle via `FLUID_INTERVIEW_LEGACY=1` to revert to the legacy bootstrap. - **Slash commands inside the interview.** `cli/_interview_slash_commands.py` — `:ai-setup`, `:override`, `:show-work`, `:doctor`, `:help`, `:quit`. Wrapped into `forge_dialogs.ask_friendly_text` so every prompt accepts them. @@ -416,6 +416,15 @@ would replace; when building it, **depend on the official `agent-client-protocol PyPI SDK** (Pydantic models + async base classes + stdio JSON-RPC) rather than hand-rolling the protocol. +## Recent architectural changes worth knowing (2026-11-03, welcome-scan budget) + +`cli/_welcome_scan.py::run_welcome_scan` now has a budget that means what it says: + +- **One wall-clock cap.** `_TOTAL_TIMEOUT_S` (1.0 s) is the only scan budget; the old per-probe `_PROBE_TIMEOUT_S` is gone. Probes run on daemon threads (`_spawn_probe`: one `Future` each, deliberately not a `ThreadPoolExecutor`, whose non-daemon workers are joined at interpreter exit and would hold the CLI open for an abandoned probe). The scan returns at the deadline and drops stragglers. Python cannot kill a thread, so an abandoned probe keeps running in the background until it finishes; the one that can run long is `_probe_specialization_suggestion` (unbounded `rglob`), and the picker, the interview signals and the welcome panel each start a scan, so up to three walkers can overlap on a huge cwd. Probes must not write to stdout/stderr (a daemon thread caught mid-write at interpreter exit can abort the process). +- **Cold imports are off the clock.** `_warm_probe_imports` imports `_WARM_MODULES` (the AI-config pydantic chain, the dbt runner) from the main thread while the probes run, then the budget starts. Probe imports stay function-local: if a probe starts importing something new, list it in `_WARM_MODULES` or `tests/test_welcome_scan_budget.py::test_probes_import_nothing_new_after_the_warm_up` fails. NB `build_parser()` already imports `ai_setup`, the dbt runner and pydantic, so in a real CLI process only `copilot.unified_config` is still cold; the warm-up matters for tests and library callers. +- **dbt has its own cap.** `_DBT_VERSION_TIMEOUT_S` (0.32 s) sizes the `dbt --version` subprocess and is not derived from the scan budget and must stay below it, so the child is killed by its own timeout instead of being orphaned. With Python dbt-core on PATH it usually waits out that cap, which is the scan's floor (~0.35 s). +- `scan_duration_ms` is the real wall-clock of the call, imports included. Pins: `tests/test_welcome_scan_budget.py`. + ## Working style - **Goal-driven execution.** For non-trivial tasks, state success criteria before implementing and loop until verified. Transform imperatives into tests: "add redactor coverage" → "new pattern is masked by a test assertion"; "fix the bug" → "reproducing test passes first, then fix makes it green". diff --git a/fluid_build/cli/_welcome_scan.py b/fluid_build/cli/_welcome_scan.py index ecacdb7e2..4c2f58e14 100644 --- a/fluid_build/cli/_welcome_scan.py +++ b/fluid_build/cli/_welcome_scan.py @@ -29,28 +29,51 @@ un-inferred bits. Recurring users (≥5 forges) skip the welcome entirely and land straight in the interview. -50ms scan budget — every probe is best-effort, never blocking. +Every probe is best-effort and never raises. Budget: once the one-time +module imports are done, the probes share ``_TOTAL_TIMEOUT_S`` of wall-clock +and the scan returns by then, dropping any probe still running. The dbt probe +sets the floor: with Python dbt-core on PATH it usually waits out +``_DBT_VERSION_TIMEOUT_S`` and reports nothing. ``scan_duration_ms`` is the +measured wall-clock of the whole call, imports included. """ from __future__ import annotations +import importlib import json import logging import os import shutil +import threading import time -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future, wait from dataclasses import asdict, dataclass, field from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Tuple LOG = logging.getLogger(__name__) -# Probe budgets — every individual probe must finish under this. -_PROBE_TIMEOUT_S = 0.4 -# Aggregate scan budget — shut down probes that overrun. +# Wall-clock budget for the whole scan, counted once the cold imports are done +# (see ``_warm_probe_imports``). A probe still running at the deadline is dropped. _TOTAL_TIMEOUT_S = 1.0 +# ``dbt --version`` cap. Fusion answers in milliseconds, Python dbt-core in +# seconds, so a slow binary simply reports nothing. Independent of the scan +# budget (it sizes one subprocess, not the scan) but it must stay below it: the +# subprocess is then killed by its own timeout before the scan can abandon it. +_DBT_VERSION_TIMEOUT_S = 0.32 +# Modules the probes import lazily, paid up front by ``_warm_probe_imports``. +# They stay function-local in the probes, not imported here, so this module is +# cheap to import. ``tests/test_welcome_scan_budget.py`` fails if a probe starts +# importing something that is not listed here. +_WARM_MODULES = ( + "fluid_build.cli.workspace_config", # _probe_workspace + "fluid_build.cli.ai_setup", # _probe_ai_credentials + # ``_load_ai_config`` imports it lazily; it pulls in pydantic and is most of + # the AI probe's cold cost (~0.1s CPU, several times that on a loaded host). + "fluid_build.copilot.unified_config", + "fluid_build.build_runners.dbt.runner", # _probe_dbt_engine +) # Repeat-user threshold — at this many forges we stop showing the panel. _REPEAT_USER_THRESHOLD = 5 # File extensions we recognise as candidate sample data. @@ -244,11 +267,11 @@ def _probe_dbt_engine() -> Dict[str, Any]: """Detect the dbt engine flavor (Python dbt-core v1 vs Fusion / v2). Fusion (Rust) answers ``dbt --version`` in milliseconds; Python - dbt-core takes seconds, so the subprocess gets a sub-budget timeout - and a slow binary simply reports nothing — the CLIs row then shows a - plain ``dbt`` exactly as before. The detection cache is keyed on the - timeout too, so this short-budget miss never poisons the build - runner's full-budget probe. + dbt-core takes seconds, so the subprocess gets its own short timeout + (``_DBT_VERSION_TIMEOUT_S``) and a slow binary simply reports nothing — + the CLIs row then shows a plain ``dbt`` exactly as before. The detection + cache is keyed on the timeout too, so this short-timeout miss never + poisons the build runner's full-timeout probe. """ try: from fluid_build.build_runners.dbt.runner import ( @@ -259,7 +282,7 @@ def _probe_dbt_engine() -> Dict[str, Any]: dbt_bin = _resolve_dbt_executable() if not dbt_bin: return {} - flavor, version = _detect_dbt_engine(dbt_bin, timeout=_PROBE_TIMEOUT_S * 0.8) + flavor, version = _detect_dbt_engine(dbt_bin, timeout=_DBT_VERSION_TIMEOUT_S) if flavor == "unknown": return {} return {"dbt_engine": f"{flavor} {version}".strip()} @@ -381,18 +404,68 @@ def _probe_specialization_suggestion(start: Path) -> Dict[str, Any]: # --------------------------------------------------------------------------- +def _warm_probe_imports() -> None: + """Import every module in ``_WARM_MODULES`` (best-effort, idempotent). + + A cold import is CPU work, not the I/O wait the scan budget exists for, and + a loaded machine stretches it several times over. Paying it here, before + the budget clock starts, keeps the first scan of a process from dropping + results that a warm scan returns. + """ + for name in _WARM_MODULES: + try: + importlib.import_module(name) + except Exception as exc: # noqa: BLE001 — best-effort; the probe reports its own failure + LOG.debug("welcome_scan_warm_failed: %s — %s", name, exc) + + +def _spawn_probe( + name: str, fn: Callable[..., Dict[str, Any]], args: Tuple[Any, ...] +) -> Future[Dict[str, Any]]: + """Run ``fn(*args)`` on a daemon thread and return a Future for its result. + + Not a ``ThreadPoolExecutor``: its workers are non-daemon and joined at + interpreter exit (documented in ``concurrent.futures``), so a probe the scan + gave up on would keep the CLI alive until it finished. Same idiom as the + non-SIGALRM path of ``ProcessManager.run_with_timeout`` (``cli/security.py``), + with a Future per probe so the caller can wait on all of them against one + deadline. + + Probes must not write to stdout/stderr: a daemon thread caught mid-write + when the interpreter exits can abort the process. Only ``Exception`` is + recorded; a probe that ends its thread with ``SystemExit`` or + ``KeyboardInterrupt`` is simply dropped at the deadline. + """ + future: Future[Dict[str, Any]] = Future() + + def _run() -> None: + try: + future.set_result(fn(*args)) + except Exception as exc: # noqa: BLE001 — recorded on the Future, never raised here + future.set_exception(exc) + + threading.Thread(target=_run, name=f"welcome-scan-{name}", daemon=True).start() + return future + + def run_welcome_scan(*, start: Optional[Path] = None) -> WelcomeFindings: """Run every probe in parallel and return a :class:`WelcomeFindings`. - Total wall-clock <= ``_TOTAL_TIMEOUT_S``. Each individual probe runs - under ``_PROBE_TIMEOUT_S``. A probe that times out is silently - dropped — its fields stay at their defaults. + The probes start together on daemon threads. The cold imports they trigger + finish first (``_warm_probe_imports``, overlapping the probes' own work, + the dbt subprocess included); the probes then get ``_TOTAL_TIMEOUT_S`` of + wall-clock. A probe still running at the deadline is dropped — its fields + stay at their defaults — and abandoned, so a hung probe delays neither the + prompt nor the process exit. + + Wall-clock is the cold imports plus at most ``_TOTAL_TIMEOUT_S``; + ``scan_duration_ms`` reports all of it. """ target = (start or Path.cwd()).resolve() findings = WelcomeFindings() - started = time.time() + started = time.monotonic() - probes = { + probes: Dict[str, Tuple[Callable[..., Dict[str, Any]], Tuple[Any, ...]]] = { "workspace": (_probe_workspace, (target,)), "ai": (_probe_ai_credentials, ()), "clis": (_probe_clis, ()), @@ -404,21 +477,22 @@ def run_welcome_scan(*, start: Optional[Path] = None) -> WelcomeFindings: "specialization": (_probe_specialization_suggestion, (target,)), } - with ThreadPoolExecutor(max_workers=len(probes)) as pool: - futures = {name: pool.submit(fn, *args) for name, (fn, args) in probes.items()} - deadline = started + _TOTAL_TIMEOUT_S - for name, fut in futures.items(): - remaining = max(0.0, deadline - time.time()) - try: - result = fut.result(timeout=min(_PROBE_TIMEOUT_S, remaining)) - except Exception as exc: # noqa: BLE001 — concurrent.futures.TimeoutError + others - LOG.debug("welcome_scan_probe_failed: %s — %s", name, exc) - continue - for key, value in (result or {}).items(): - if hasattr(findings, key): - setattr(findings, key, value) + futures = {name: _spawn_probe(name, fn, args) for name, (fn, args) in probes.items()} + _warm_probe_imports() + done, _ = wait(futures.values(), timeout=_TOTAL_TIMEOUT_S) + for name, future in futures.items(): + if future not in done: + LOG.debug("welcome_scan_probe_timeout: %s", name) + continue + exc = future.exception() + if exc is not None: + LOG.debug("welcome_scan_probe_failed: %s — %s", name, exc) + continue + for key, value in (future.result() or {}).items(): + if hasattr(findings, key): + setattr(findings, key, value) - findings.scan_duration_ms = int((time.time() - started) * 1000) + findings.scan_duration_ms = int((time.monotonic() - started) * 1000) return findings diff --git a/fluid_build/cli/forge_modes.py b/fluid_build/cli/forge_modes.py index 9d9f32cc9..007e29793 100644 --- a/fluid_build/cli/forge_modes.py +++ b/fluid_build/cli/forge_modes.py @@ -1139,9 +1139,9 @@ def _render_ai_copilot_error(console: Any, exc: BaseException) -> None: copilot = copilot_class() is_non_interactive = bool(get_cli_arg_fn(args, "non_interactive", False)) - # Phase 0.2 — detect-first welcome scan. Runs in <50ms (parallel - # probes) and renders a populated panel so the user sees what we - # already know before any prompt. Skipped silently in + # Phase 0.2 — detect-first welcome scan. Parallel probes under a + # wall-clock cap (see ``_welcome_scan``) render a populated panel so + # the user sees what we already know before any prompt. Skipped silently in # non-interactive runs and for return users (forge_count >= 5). # Set FLUID_FORGE_NO_WELCOME=1 to suppress for tests / CI. if ( diff --git a/tests/conftest.py b/tests/conftest.py index cfa6b6c4d..0eaf60cbb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -208,22 +208,20 @@ def _disable_copilot_self_eval(monkeypatch): def deterministic_welcome_scan(monkeypatch): """Make ``run_welcome_scan`` return every probe's result, however slow the runner. - The scan runs its probes in threads and discards any result that misses a - 0.4s per-probe / 1.0s total budget: right for a CLI prompt, wrong for a test - that asserts on what a probe reports (workspace products, sample data, ...). - On a loaded runner (coverage, ``-n auto``) the probe overruns, its field - keeps the "nothing detected" default, and the mode picker falls back to - ``ai`` where the test expects ``from_product`` or ``refine``. - - The budget only decides which results are kept, never how long the scan runs - (it still joins every probe thread), so wall-clock assertions are unaffected. - The dbt probe sizes its ``dbt --version`` subprocess from the same budget, so - it is pointed at "no dbt installed" instead of waiting out a slow dbt-core; - tests of the dbt probe patch ``_resolve_dbt_executable`` themselves. + The scan runs its probes in threads and discards any result that misses its + 1.0s wall-clock budget: right for a CLI prompt, wrong for a test that asserts + on what a probe reports (workspace products, sample data, ...). On a loaded + runner (coverage, ``-n auto``) a probe can overrun, its field keeps the + "nothing detected" default, and the mode picker falls back to ``ai`` where + the test expects ``from_product`` or ``refine``. + + The budget is lifted so nothing is dropped. The scan still returns as soon + as every probe has finished, so wall-clock assertions are unaffected. The dbt + probe is pointed at "no dbt installed" so the result does not depend on the + host's dbt; tests of the dbt probe patch ``_resolve_dbt_executable`` themselves. Opt in with ``pytestmark = pytest.mark.usefixtures("deterministic_welcome_scan")``. """ - monkeypatch.setattr("fluid_build.cli._welcome_scan._PROBE_TIMEOUT_S", 30.0) monkeypatch.setattr("fluid_build.cli._welcome_scan._TOTAL_TIMEOUT_S", 60.0) monkeypatch.setattr( "fluid_build.build_runners.dbt.runner._resolve_dbt_executable", lambda: None diff --git a/tests/test_forge_mode_picker.py b/tests/test_forge_mode_picker.py index 27a3c37a6..63ade618d 100644 --- a/tests/test_forge_mode_picker.py +++ b/tests/test_forge_mode_picker.py @@ -181,7 +181,7 @@ def test_pick_mode_default_picks_from_product_when_workspace_has_products(tmp_pa def test_pick_mode_default_survives_a_slow_workspace_probe(tmp_path, silent_console, monkeypatch): - """A workspace probe slower than the scan's default 0.4s budget keeps its result. + """A workspace probe slower than the scan's default 1.0s budget keeps its result. Deterministic stand-in for a loaded CI runner, where the probe overran and the default fell back to ``ai`` (``assert 'ai' == 'from_product'``). Holds only while @@ -194,7 +194,7 @@ def test_pick_mode_default_survives_a_slow_workspace_probe(tmp_path, silent_cons real_probe = _welcome_scan._probe_workspace def _slow_probe(start): - time.sleep(0.6) # past the default 0.4s _PROBE_TIMEOUT_S + time.sleep(1.2) # past the default 1.0s _TOTAL_TIMEOUT_S return real_probe(start) monkeypatch.setattr(_welcome_scan, "_probe_workspace", _slow_probe) diff --git a/tests/test_welcome_scan.py b/tests/test_welcome_scan.py index 89bc946f8..da07f8843 100644 --- a/tests/test_welcome_scan.py +++ b/tests/test_welcome_scan.py @@ -46,7 +46,7 @@ def isolate_home(tmp_path, monkeypatch): def test_welcome_scan_returns_findings_under_one_second(tmp_path: Path): findings = run_welcome_scan(start=tmp_path) assert isinstance(findings, WelcomeFindings) - assert findings.scan_duration_ms < 1500 # 50ms budget + slack for CI + assert findings.scan_duration_ms < 1500 # budget is lifted by the fixture: real probe latency def test_welcome_scan_detects_workspace(tmp_path: Path): diff --git a/tests/test_welcome_scan_budget.py b/tests/test_welcome_scan_budget.py new file mode 100644 index 000000000..d194a4c02 --- /dev/null +++ b/tests/test_welcome_scan_budget.py @@ -0,0 +1,266 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Welcome-scan budget semantics. + +Two promises, both measured against a real clock rather than a mocked one: + +* the cold imports the probes trigger are paid before the budget starts, so the + first scan of a process returns what a warm scan returns (on a loaded machine + the AI probe's pydantic import used to blow the old per-probe window); +* ``_TOTAL_TIMEOUT_S`` is a real cap: the scan returns on time and a probe it + gave up on cannot keep the process alive. +""" + +from __future__ import annotations + +import importlib +import json +import os +import subprocess +import sys +import textwrap +import threading +import time +from pathlib import Path + +import pytest + +from fluid_build.cli import _welcome_scan as ws + +pytestmark = pytest.mark.unit + + +@pytest.fixture(autouse=True) +def isolate_home(tmp_path, monkeypatch): + """Sandbox ``~/.fluid`` so a real usage.json can't turn the run into a return user.""" + fake_home = tmp_path / "_home" + fake_home.mkdir(exist_ok=True) + monkeypatch.setattr("fluid_build.cli._welcome_scan.Path.home", lambda: fake_home) + return fake_home + + +def _write_orders_csv(root: Path) -> None: + (root / "data").mkdir(exist_ok=True) + (root / "data" / "orders.csv").write_text("id\n1\n") + + +def _child_env(home: Path) -> dict: + """Environment for a probe subprocess: no AI selector, no dbt, throwaway home. + + ``PYTHONPATH`` is seeded from this process's ``sys.path`` so the child imports + the tree under test whether fluid is pip-installed (CI) or run from a source + checkout (local worktree); same convention as ``test_ai_config_shared.py``. + """ + env = {k: v for k, v in os.environ.items() if k != "FLUID_LLM_PROVIDER"} + env.update( + HOME=str(home), + USERPROFILE=str(home), + DBT_EXECUTABLE="/nonexistent/dbt", + PYTHONPATH=os.pathsep.join(p for p in sys.path if p), + ) + return env + + +# --------------------------------------------------------------------------- +# The cap is real +# --------------------------------------------------------------------------- + + +def test_scan_returns_at_the_cap_while_a_probe_is_still_running(tmp_path, monkeypatch): + """A hung probe is dropped and the scan returns; it does not wait for the probe.""" + release = threading.Event() + + def hung_probe(): + release.wait(timeout=10) # joining this thread would cost the full 10s + return {"cloud_credentials": ["too-late"]} + + monkeypatch.setattr(ws, "_probe_cloud_creds", hung_probe) + monkeypatch.setattr(ws, "_TOTAL_TIMEOUT_S", 0.3) + _write_orders_csv(tmp_path) + ws._warm_probe_imports() # keep this process's first-import cost out of the timing below + + try: + t0 = time.monotonic() + findings = ws.run_welcome_scan(start=tmp_path) + elapsed = time.monotonic() - t0 + finally: + release.set() + + assert elapsed < 5.0, f"scan waited for a hung probe: {elapsed:.1f}s" + assert findings.cloud_credentials == [] # the straggler's result is dropped + assert any("orders.csv" in c for c in findings.sample_data_candidates) # the rest is kept + assert 250 <= findings.scan_duration_ms < 5000 # it did use the budget, and says so + + +def test_abandoned_probe_does_not_keep_the_process_alive(tmp_path): + """The thread is a daemon: a probe the scan gave up on must not delay interpreter exit. + + Measured in a child because the effect is at exit. A non-daemon straggler (a + ThreadPoolExecutor worker, joined at exit) would add its remaining ~10s here. + """ + script = textwrap.dedent( + """ + import time + from fluid_build.cli import _welcome_scan as ws + + ws._TOTAL_TIMEOUT_S = 0.2 + ws._warm_probe_imports() + ws._probe_cloud_creds = lambda: time.sleep(10) or {} + ws.run_welcome_scan() + print(time.time(), flush=True) # wall-clock stamp: the scan has returned + """ + ) + home = tmp_path / "_child_home" + home.mkdir() + proc = subprocess.run( + [sys.executable, "-c", script], + cwd=tmp_path, + env=_child_env(home), + capture_output=True, + text=True, + timeout=120, + ) + exited = time.time() + assert proc.returncode == 0, proc.stderr + scan_returned = float(proc.stdout.strip().splitlines()[-1]) + assert exited - scan_returned < 5.0, ( + f"process lingered {exited - scan_returned:.1f}s after the scan returned; " + "an abandoned probe thread is keeping it alive" + ) + + +# --------------------------------------------------------------------------- +# Cold imports are off the clock +# --------------------------------------------------------------------------- + + +def test_cold_import_is_not_counted_against_the_budget(tmp_path, monkeypatch): + """A probe whose first import outlasts the budget still reports its result. + + Stand-in for the AI probe's lazy pydantic import: slow to import, instant to run. + """ + module = "welcome_scan_slow_import_fixture" + (tmp_path / f"{module}.py").write_text("import time\ntime.sleep(1.2)\nVALUE = ['imported']\n") + monkeypatch.syspath_prepend(str(tmp_path)) + sys.modules.pop(module, None) + + def probe(): + return {"cloud_credentials": importlib.import_module(module).VALUE} + + monkeypatch.setattr(ws, "_probe_cloud_creds", probe) + monkeypatch.setattr(ws, "_WARM_MODULES", (module,)) + monkeypatch.setattr(ws, "_TOTAL_TIMEOUT_S", 0.6) # half the import time + + try: + findings = ws.run_welcome_scan(start=tmp_path) + finally: + sys.modules.pop(module, None) + + assert findings.cloud_credentials == ["imported"] + assert findings.scan_duration_ms >= 1000 # the import is in the reported wall-clock + + +def test_probes_import_nothing_new_after_the_warm_up(tmp_path): + """``_WARM_MODULES`` covers everything the probes load lazily. + + A fresh interpreter: warm up, run every probe once, and no module outside the + standard library may appear in ``sys.modules`` afterwards. If this fails, a + probe (or something it calls, e.g. ``_load_ai_config``) started importing a + module that would land back on the budget clock; add it to ``_WARM_MODULES``. + """ + (tmp_path / "contract.fluid.yaml").write_text("metadata:\n productType: SDP\n") + script = textwrap.dedent( + """ + import json, sys + from pathlib import Path + from fluid_build.cli import _welcome_scan as ws + + ws._warm_probe_imports() + before = set(sys.modules) + start = Path.cwd() + for fn, args in ( + (ws._probe_workspace, (start,)), + (ws._probe_ai_credentials, ()), + (ws._probe_clis, ()), + (ws._probe_dbt_engine, ()), + (ws._probe_coding_agents, ()), + (ws._probe_sample_data, (start,)), + (ws._probe_cloud_creds, ()), + (ws._probe_return_user, ()), + (ws._probe_specialization_suggestion, (start,)), + ): + fn(*args) + stdlib = sys.stdlib_module_names # lazy stdlib helpers (heapq, ...) are noise + new = set(sys.modules) - before + print(json.dumps(sorted(m for m in new if m.split(".")[0] not in stdlib))) + """ + ) + home = tmp_path / "_child_home" + home.mkdir() + proc = subprocess.run( + [sys.executable, "-c", script], + cwd=tmp_path, + env=_child_env(home), + capture_output=True, + text=True, + timeout=120, + ) + assert proc.returncode == 0, proc.stderr + assert json.loads(proc.stdout.strip().splitlines()[-1]) == [] + + +# --------------------------------------------------------------------------- +# The rest of the contract +# --------------------------------------------------------------------------- + + +def test_a_raising_probe_is_dropped_and_the_rest_survive(tmp_path, monkeypatch): + def boom(): + raise RuntimeError("probe blew up") + + monkeypatch.setattr(ws, "_probe_clis", boom) + monkeypatch.setattr(ws, "_TOTAL_TIMEOUT_S", 30.0) # healthy probes finish at once + _write_orders_csv(tmp_path) + + findings = ws.run_welcome_scan(start=tmp_path) + + assert findings.installed_clis == [] + assert any("orders.csv" in c for c in findings.sample_data_candidates) + + +def test_dbt_version_timeout_does_not_follow_the_scan_budget(monkeypatch): + """The ``dbt --version`` cap sizes one subprocess; the scan budget is not its input.""" + seen = [] + + def fake_detect(exe, timeout=10.0): + seen.append(timeout) + return ("fusion", "2.0.0") + + monkeypatch.setattr( + "fluid_build.build_runners.dbt.runner._resolve_dbt_executable", lambda: "/opt/fusion/dbt" + ) + monkeypatch.setattr("fluid_build.build_runners.dbt.runner._detect_dbt_engine", fake_detect) + + for budget in (0.5, 60.0): + monkeypatch.setattr(ws, "_TOTAL_TIMEOUT_S", budget) + assert ws._probe_dbt_engine() == {"dbt_engine": "fusion 2.0.0"} + + assert seen == [ws._DBT_VERSION_TIMEOUT_S] * 2 + + +def test_dbt_version_cap_fits_inside_the_scan_budget(): + """The ``dbt --version`` subprocess has to hit its own timeout, which kills it, before + the scan can abandon it; an abandoned child would outlive the CLI.""" + assert ws._DBT_VERSION_TIMEOUT_S < ws._TOTAL_TIMEOUT_S From 875590624ab34b99e962c2036f1b60ae8d2c3123 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:15:34 +0100 Subject: [PATCH 19/26] fix(apply): refuse an unusable build id before tofu apply (#803) * fix(apply): refuse an unusable build id before tofu apply `fluid apply --mode amend-and-build` (and replace-and-build) checked contract.id and builds[].id only in the build runner. On the OpenTofu engine the runner starts after `tofu apply`, so an id the schema admits but the build phase refuses (1orders, more than 128 characters, a trailing newline) was refused after the product was provisioned, and rendered as "Unexpected error". The runner's rule is now one function, build_runners/_ids.py validate_build_ids, and apply runs it right after the schema gate in a build mode, ahead of the apply hooks, tofu and the runner, on both engines. The refusal is a typed CLIError, apply_build_identifier_invalid, that names the field and the value and says nothing was applied. The runner keeps its own check as the backstop. * docs(changelog): build ids are checked before tofu apply * test(apply): import fluid_build.cli one way in the build-id test --- CHANGELOG.md | 13 + fluid_build/_error_catalog.py | 11 + fluid_build/build_runners/_ids.py | 39 +- fluid_build/build_runners/base.py | 12 +- fluid_build/cli/apply.py | 43 +++ tests/cli/test_apply_build_ids_before_tofu.py | 357 ++++++++++++++++++ 6 files changed, 467 insertions(+), 8 deletions(-) create mode 100644 tests/cli/test_apply_build_ids_before_tofu.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ff28c6574..f08f25036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -132,6 +132,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`fluid apply --mode amend-and-build` provisioned a product, then refused its id.** The build + phase keeps run records, cursors and watermarks under `.fluid/runs///`, + so its ids must start with a letter or underscore, use only letters, digits, `_`, `.` and `-`, + and be at most 128 characters. The contract schema allows more (`1orders`, or a longer id), so + `fluid validate` and `fluid plan` passed such an id. Only the build runner checked it, and on the + OpenTofu engine (AWS, GCP, Snowflake, Confluent, Databricks and IaC plugin clouds) the runner + starts after `tofu apply`: the infrastructure was created, and the run then stopped with + "Unexpected error". A build mode (`amend-and-build`, `replace-and-build`) now checks + `contract.id` and every `builds[].id` before the apply hooks, `tofu` and the runner, on both + engines, and refuses with `ERR_APPLY_BUILD_IDENTIFIER_INVALID`, which names the id, says nothing + was applied and suggests a rename. A mode without a build phase, and a contract without builds, + are not checked, as before. + - **`fluid diff --baseline` reported nothing for column changes in a 0.7.x contract.** It read columns from `exposes[].schema`, but a 0.7.x contract keeps them in `exposes[].contract.schema` (as does every `fluid init` template), so a removed column, a type change or a new NOT NULL diff --git a/fluid_build/_error_catalog.py b/fluid_build/_error_catalog.py index db85b031a..8918e49da 100644 --- a/fluid_build/_error_catalog.py +++ b/fluid_build/_error_catalog.py @@ -265,6 +265,17 @@ def slug_for(event: str) -> str: ["Run 'fluid validate ' first to rule out a contract problem"], None, ), + "apply_build_identifier_invalid": ( + [ + "Rename the id to fit, for example orders_v1 instead of 1orders, and apply again", + ( + "A renamed id starts over: its builds get new cursors and watermarks under " + ".fluid/runs/, and a renamed contract.id gets a new OpenTofu workdir under " + ".fluid/iac//" + ), + ], + None, + ), "output_write_failed": ( ["Check that the --out path is writable and the parent directory exists"], None, diff --git a/fluid_build/build_runners/_ids.py b/fluid_build/build_runners/_ids.py index 831263a63..5fdb0f241 100644 --- a/fluid_build/build_runners/_ids.py +++ b/fluid_build/build_runners/_ids.py @@ -37,6 +37,7 @@ from __future__ import annotations import re +from typing import Any, Mapping, Optional # A permissive regex, identical to ``validate_ident`` in # ``providers/_sql_safety``: alphanumerics + dots + dashes + underscores, @@ -48,7 +49,17 @@ class IdentifierViolation(ValueError): - """Raised when a contract identifier fails validation.""" + """Raised when a contract identifier fails validation. + + ``kind`` and ``value`` name the id that failed when the raiser knows it + (:func:`validate_identifier` always does), so a caller can report it + without parsing the message. + """ + + def __init__(self, message: str, *, kind: Optional[str] = None, value: Any = None) -> None: + super().__init__(message) + self.kind = kind + self.value = value def validate_identifier(value: str, *, kind: str) -> str: @@ -65,6 +76,30 @@ def validate_identifier(value: str, *, kind: str) -> str: f"{kind} {value!r} is not a valid identifier " "(must match ^[A-Za-z_][A-Za-z0-9_.\\-]{0,127}$). This guard " "blocks template-injection / path-traversal attacks via " - "attacker-controlled contract metadata." + "attacker-controlled contract metadata.", + kind=kind, + value=value, ) return value + + +def validate_build_ids(contract: Mapping[str, Any]) -> None: + """Validate the ids a contract's build phase uses: ``contract.id`` and each ``builds[].id``. + + The build runner (``base.run_builds_from_args``) runs this before any + build, and ``fluid apply`` runs it before it applies anything, so the two + refuse the same ids. A contract with no builds has no build phase and is + not checked. An empty or absent id is skipped: it cannot traverse, and the + state store falls back to a safe default. A ``builds`` entry that is not a + mapping is left to the schema gate. + + Raises :class:`IdentifierViolation` for the first id that fails. + """ + builds = contract.get("builds") or [] + if not builds: + return + if contract.get("id"): + validate_identifier(contract["id"], kind="contract.id") + for build in builds: + if isinstance(build, Mapping) and build.get("id"): + validate_identifier(build["id"], kind="build.id") diff --git a/fluid_build/build_runners/base.py b/fluid_build/build_runners/base.py index 36db5499c..3fd4e3afc 100644 --- a/fluid_build/build_runners/base.py +++ b/fluid_build/build_runners/base.py @@ -1393,17 +1393,17 @@ def run_builds_from_args( # rather than traversing the filesystem. The parallel pipeline # (``cli/_acquisition_stage_ext``) already validates the same fields; # this closes the one-sided-guard gap on the runtime path. - from ._ids import validate_identifier + from ._ids import validate_build_ids # Only non-empty ids are checked: an absent/empty id cannot traverse # (``_build_dir`` falls back to a safe "product" default and # ``FileStateStore._confine()`` is the backstop), while a non-empty id # like ``../../../../tmp/escape`` is rejected here before any path is made. - if contract.get("id"): - validate_identifier(contract["id"], kind="contract.id") - for _b in builds: - if _b.get("id"): - validate_identifier(_b["id"], kind="build.id") + # ``fluid apply`` runs the same check before it applies anything + # (``cli/apply.py::_gate_build_ids_for_apply``), so on the OpenTofu engine + # an id is not refused here after ``tofu apply``; this one stays as the + # backstop for any caller that skips it. + validate_build_ids(contract) # The running ``fluid apply``'s run report, if any: each build is recorded on it. from fluid_build.observability.apply_run import current_apply_run diff --git a/fluid_build/cli/apply.py b/fluid_build/cli/apply.py index a4c8d45da..40b806005 100644 --- a/fluid_build/cli/apply.py +++ b/fluid_build/cli/apply.py @@ -308,6 +308,44 @@ def _gate_contract_for_apply(contract: Dict[str, Any], logger: logging.Logger) - _gate_contract_for_plan_or_apply(contract, logger, command="apply") +def _gate_build_ids_for_apply(contract: Dict[str, Any]) -> None: + """Refuse, before anything is applied, an id the build phase would refuse. + + The build phase turns ``contract.id`` and ``builds[].id`` into directory + names, so it takes a narrower grammar than the schema: ``1orders``, or an + id of 129 characters, passes ``fluid validate`` and ``fluid plan``. The + build runner checks the ids itself, but on the OpenTofu engine it runs + after ``tofu apply``, so such an id was refused once the product was + provisioned, as an "Unexpected error". This is the runner's own check + (``build_runners._ids.validate_build_ids``), run ahead of the hooks, + ``tofu`` and the runner on both engines. + + Raises: + CLIError: ``apply_build_identifier_invalid``. + """ + from fluid_build.build_runners._ids import IdentifierViolation, validate_build_ids + + try: + validate_build_ids(contract) + except IdentifierViolation as exc: + raise CLIError( + 1, + "apply_build_identifier_invalid", + { + "field": exc.kind, + "value": repr(exc.value), + "hint": ( + "The build phase keeps each build's run records, cursors and " + "watermarks under .fluid/runs///, so both ids " + "must start with a letter or underscore, use only letters, digits, " + "'_', '.' and '-', and be at most 128 characters. The contract schema " + "allows more, so fluid validate and fluid plan accept it. Nothing was " + "applied." + ), + }, + ) from exc + + def _log_apply_failure(logger: logging.Logger, start_time: float, error: Exception) -> None: """Record a pre-dispatch ``apply_contract`` failure for the structured log.""" log_operation_failure(logger, "apply_contract", error=error, duration=time.time() - start_time) @@ -1923,8 +1961,13 @@ def run(args, logger: logging.Logger) -> int: # IaC plugin clouds) and both engines' ``needs_build`` returns used to # skip it, so a schema-invalid contract reached ``tofu`` or the build # runner and could exit 0. One call per run, on the one loaded copy. + # A build mode also checks the ids the build phase turns into paths, which + # the schema admits more of: on the OpenTofu engine the runner would only + # refuse one after ``tofu apply``. try: _gate_contract_for_apply(source_contract, logger) + if needs_build(resolved_mode): + _gate_build_ids_for_apply(source_contract) except CLIError as exc: _log_apply_failure(logger, start_time, exc) raise diff --git a/tests/cli/test_apply_build_ids_before_tofu.py b/tests/cli/test_apply_build_ids_before_tofu.py new file mode 100644 index 000000000..a68f329db --- /dev/null +++ b/tests/cli/test_apply_build_ids_before_tofu.py @@ -0,0 +1,357 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fluid apply`` refuses an id the build phase cannot use before it applies anything. + +The build phase turns ``contract.id`` and ``builds[].id`` into directory names and +scheduler code, so it takes a narrower grammar (``build_runners._ids``) than the +schema's ``$defs.identifier``: a leading digit (``1orders``) or an id longer than +128 characters passes ``fluid validate`` and ``fluid plan``. Only the build runner +checked these ids, and on the OpenTofu engine it runs after ``tofu apply``. So +``--mode amend-and-build`` provisioned the product and then failed with +"Unexpected error", because ``IdentifierViolation`` had no typed mapping. + +These tests drive ``run()`` through the real argparse surface. ``tofu`` and the +apply hooks are recorders; the build runner is the real one, behind a spy, so +"refused first" is asserted as "``tofu``, the hooks and the runner never ran". +""" + +from __future__ import annotations + +import copy +import json +import logging +from pathlib import Path +from typing import Any, Dict, Iterator +from unittest import mock + +import pytest +import yaml + +import fluid_build.cli as cli_mod +from fluid_build.build_runners import _ids +from fluid_build.build_runners import base as runner_base +from fluid_build.build_runners._ids import IdentifierViolation +from fluid_build.cli._common import CLIError +from fluid_build.cli.apply import _gate_contract_for_apply +from fluid_build.cli.apply import run as apply_run +from fluid_build.forge.core.plan_digest import inject_digests + +LOG = logging.getLogger("test.apply_build_ids_before_tofu") + +_CONTRACT: Dict[str, Any] = { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "orders", + "name": "Orders", + "description": "Inline SQL build.", + "domain": "Demo", + "metadata": {"layer": "Bronze", "owner": {"team": "dp", "email": "dp@example.com"}}, + "builds": [ + { + "id": "make_rows", + "description": "Inline rows.", + "pattern": "embedded-logic", + "engine": "sql", + "properties": {"sql": "SELECT 1 AS id"}, + "outputs": ["rows"], + } + ], + "exposes": [ + { + "exposeId": "rows", + "kind": "table", + "binding": { + "platform": "local", + "format": "parquet", + "location": {"path": "./out/rows.parquet"}, + }, + "contract": {"schema": [{"name": "id", "type": "INTEGER", "required": True}]}, + } + ], +} + +# Ids the schema admits and the build phase refuses: (field, value). +_UNUSABLE = [ + pytest.param("contract.id", "1orders", id="contract-id-leading-digit"), + pytest.param("contract.id", "o" * 129, id="contract-id-129-chars"), + pytest.param("build.id", "1make_rows", id="build-id-leading-digit"), + pytest.param("build.id", "b" * 129, id="build-id-129-chars"), +] + +_BUILD_MODES = [ + pytest.param("amend-and-build", (), id="amend-and-build"), + pytest.param("replace-and-build", ("--allow-data-loss",), id="replace-and-build"), +] + + +def _with(field: str, value: str) -> Dict[str, Any]: + """``_CONTRACT`` with ``value`` as its ``contract.id`` or its build's id.""" + contract = copy.deepcopy(_CONTRACT) + if field == "contract.id": + contract["id"] = value + else: + contract["builds"][0]["id"] = value + return contract + + +@pytest.fixture +def workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("FLUID_ENV", raising=False) + (tmp_path / "runtime").mkdir() + return tmp_path + + +def _source(form: str, contract: Dict[str, Any], *, mode: str) -> str: + """Write ``contract`` as the YAML file an operator applies, or as a + digest-bound ``plan.json`` that embeds it.""" + if form == "yaml": + path, text = Path("contract.fluid.yaml"), yaml.safe_dump(contract, sort_keys=False) + else: + plan = {"format_version": "0.7.5", "mode": mode, "actions": [], "contract": contract} + path, text = Path("runtime/plan.json"), json.dumps(inject_digests(plan, bundle_path=None)) + path.write_text(text, encoding="utf-8") + return str(path) + + +def _apply(src: str, *flags: str) -> int: + """``fluid apply --yes`` through the real top-level parser.""" + return apply_run(cli_mod.build_parser().parse_args(["apply", src, *flags, "--yes"]), LOG) + + +@pytest.fixture +def tofu() -> Iterator[mock.Mock]: + """Route ``run()`` onto the OpenTofu engine and record the engine call.""" + with ( + mock.patch( + "fluid_build.cli._apply_opentofu_engine.resolve_apply_engine", + return_value="opentofu", + ), + mock.patch( + "fluid_build.cli._apply_opentofu_engine.apply_via_opentofu", return_value=0 + ) as engine, + ): + yield engine + + +@pytest.fixture +def native() -> Iterator[None]: + with mock.patch( + "fluid_build.cli._apply_opentofu_engine.resolve_apply_engine", return_value="native" + ): + yield + + +@pytest.fixture +def hooks() -> Iterator[mock.Mock]: + with mock.patch("fluid_build.cli.apply._run_apply_hooks", return_value=0) as recorder: + yield recorder + + +@pytest.fixture +def runner() -> Iterator[mock.Mock]: + """The real build runner, behind a spy.""" + with mock.patch( + "fluid_build.build_runners.run_builds_from_args", + wraps=runner_base.run_builds_from_args, + ) as spy: + yield spy + + +@pytest.mark.parametrize("field,value", _UNUSABLE) +def test_the_schema_admits_these_ids(field: str, value: str) -> None: + """Why the refusal came late: these ids pass the gate ``fluid validate`` and + ``fluid plan`` run. If the schema starts refusing them, the schema gate + refuses them first and the tests below need another example.""" + _gate_contract_for_apply(_with(field, value), LOG) + + +@pytest.mark.usefixtures("workspace") +class TestRefusedBeforeAnythingIsApplied: + @pytest.mark.parametrize("form", ["yaml", "plan"]) + @pytest.mark.parametrize("field,value", _UNUSABLE) + @pytest.mark.parametrize("mode,extra", _BUILD_MODES) + def test_refused_before_tofu( + self, + tofu: mock.Mock, + hooks: mock.Mock, + runner: mock.Mock, + form: str, + field: str, + value: str, + mode: str, + extra: Any, + ) -> None: + src = _source(form, _with(field, value), mode=mode) + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", mode, *extra) + tofu.assert_not_called() + hooks.assert_not_called() + runner.assert_not_called() + assert exc.value.event == "apply_build_identifier_invalid" + assert exc.value.context["field"] == field + assert exc.value.context["value"] == repr(value) + + @pytest.mark.usefixtures("native") + @pytest.mark.parametrize("field,value", _UNUSABLE) + def test_native_engine_refuses_with_the_same_typed_error( + self, runner: mock.Mock, field: str, value: str + ) -> None: + """The native engine applies nothing before its builds, but its refusal was + the same untyped ``IdentifierViolation``.""" + src = _source("yaml", _with(field, value), mode="amend-and-build") + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", "amend-and-build") + runner.assert_not_called() + assert exc.value.event == "apply_build_identifier_invalid" + + def test_a_build_the_filter_skips_is_still_checked( + self, tofu: mock.Mock, runner: mock.Mock + ) -> None: + """The runner checks every build's id before ``--build-id`` picks one, so a + check that skipped the other builds would let ``tofu`` run first.""" + contract = copy.deepcopy(_CONTRACT) + other = dict(contract["builds"][0], id="1other") + contract["builds"].append(other) + src = _source("yaml", contract, mode="amend-and-build") + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", "amend-and-build", "--build-id", "make_rows") + tofu.assert_not_called() + assert exc.value.context["value"] == repr("1other") + + def test_the_refusal_says_what_to_change(self, tofu: mock.Mock) -> None: + src = _source("yaml", _with("contract.id", "1orders"), mode="amend-and-build") + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", "amend-and-build") + hint = exc.value.context["hint"] + assert "Nothing was applied" in hint + assert "start with a letter or underscore" in hint + assert "128" in hint + assert exc.value.suggestions, "the catalog gives the event its suggestions" + + def test_the_cli_renders_it_as_a_typed_error( + self, tofu: mock.Mock, capsys: pytest.CaptureFixture[str] + ) -> None: + """Through ``main()``: the slug and the id, not "Unexpected error".""" + src = _source("yaml", _with("contract.id", "1orders"), mode="amend-and-build") + assert cli_mod.main(["apply", src, "--mode", "amend-and-build", "--yes"]) == 1 + out = capsys.readouterr().out + assert "ERR_APPLY_BUILD_IDENTIFIER_INVALID" in out + assert "'1orders'" in out + assert "Unexpected error" not in out + tofu.assert_not_called() + + @pytest.mark.parametrize( + "field,value", + [ + pytest.param("contract.id", "orders\n", id="contract-id"), + pytest.param("build.id", "make_rows\n", id="build-id"), + ], + ) + def test_a_trailing_newline_is_refused_before_tofu( + self, + tofu: mock.Mock, + hooks: mock.Mock, + runner: mock.Mock, + field: str, + value: str, + ) -> None: + """``_IDENT_RE`` ends in ``\\Z``, so the build phase refuses ``orders\\n``. The + schema's ``^...$`` pattern still lets ``re.search`` match before the newline. + This check refuses it until the schema pattern is fixed; after that the schema + gate does. Either way the refusal is typed and comes before ``tofu``.""" + src = _source("yaml", _with(field, value), mode="amend-and-build") + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", "amend-and-build") + tofu.assert_not_called() + hooks.assert_not_called() + runner.assert_not_called() + assert exc.value.event in {"apply_build_identifier_invalid", "apply_contract_invalid"} + + +@pytest.mark.usefixtures("workspace") +class TestNothingNewIsRefused: + """The check refuses exactly what the build runner would refuse.""" + + @pytest.mark.parametrize("form", ["yaml", "plan"]) + @pytest.mark.parametrize( + "field,value", + [ + pytest.param("contract.id", "orders", id="plain"), + pytest.param("contract.id", "o" * 128, id="contract-id-128-chars"), + pytest.param("build.id", "_make_rows", id="build-id-leading-underscore"), + ], + ) + def test_usable_ids_reach_tofu_then_the_runner( + self, tofu: mock.Mock, form: str, field: str, value: str + ) -> None: + with mock.patch("fluid_build.build_runners.run_builds_from_args", return_value=0) as run: + mode = "amend-and-build" + assert _apply(_source(form, _with(field, value), mode=mode), "--mode", mode) == 0 + tofu.assert_called_once() + run.assert_called_once() + + def test_a_mode_without_a_build_phase_is_not_checked( + self, tofu: mock.Mock, runner: mock.Mock + ) -> None: + src = _source("yaml", _with("contract.id", "1orders"), mode="amend") + assert _apply(src, "--mode", "amend") == 0 + tofu.assert_called_once() + runner.assert_not_called() + + def test_a_contract_without_builds_is_not_checked( + self, tofu: mock.Mock, runner: mock.Mock + ) -> None: + """With no builds the runner returns before its own check.""" + contract = _with("contract.id", "1orders") + del contract["builds"] + src = _source("yaml", contract, mode="amend-and-build") + assert _apply(src, "--mode", "amend-and-build") == 0 + tofu.assert_called_once() + runner.assert_called_once() + + +class TestOneRuleForBothCallers: + """``validate_build_ids`` is the rule the runner applies and ``fluid apply`` + runs ahead of it.""" + + @pytest.mark.parametrize("field,value", _UNUSABLE) + def test_it_names_the_id_that_failed(self, field: str, value: str) -> None: + with pytest.raises(IdentifierViolation) as exc: + _ids.validate_build_ids(_with(field, value)) + assert (exc.value.kind, exc.value.value) == (field, value) + + def test_the_runner_still_refuses_on_its_own(self, tmp_path: Path) -> None: + """Kept as the backstop for a caller that skips ``fluid apply``.""" + import argparse + + path = tmp_path / "contract.fluid.yaml" + path.write_text("{}", encoding="utf-8") + args = argparse.Namespace(contract=str(path), env=None, build_id=None) + with pytest.raises(IdentifierViolation, match="contract.id"): + runner_base.run_builds_from_args(args, LOG, contract=_with("contract.id", "1orders")) + + @pytest.mark.parametrize( + "contract", + [ + pytest.param({"id": "1orders"}, id="no-builds"), + pytest.param({"id": "1orders", "builds": []}, id="empty-builds"), + pytest.param({"id": "", "builds": [{"id": ""}]}, id="empty-ids"), + pytest.param({"builds": [{"engine": "sql"}]}, id="absent-ids"), + pytest.param({"id": "orders", "builds": ["make_rows", None]}, id="non-mapping-builds"), + ], + ) + def test_what_it_does_not_check(self, contract: Dict[str, Any]) -> None: + _ids.validate_build_ids(contract) From 61ccba55a815147b6a0454ba6d277d84b18ccc20 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:23:13 +0100 Subject: [PATCH 20/26] feat(databricks): bronze landing, retention, scoped grants and enforcement follow-ups (#801) * fix(databricks): the fail-safe lock never revokes from the deploy identity and says when the owner could not be skipped * fix(databricks): the lock names a restricted table owner as a path it cannot close * fix(databricks): the fail-safe says when the permission listing cannot show inherited grants * fix(databricks): the pause decision is made once per run, and verify flags a job paused for an unprotected table * fix(databricks): post_apply drops this module's superseded functions that no table in the schema uses * fix(databricks): a residency failure prints one line with the declared region, the live region and the fix * docs(databricks): HONESTLY_TESTED records the 2026-11-03 manual live run and what it left open * docs(databricks): CHANGELOG entries for the masks and row filter follow-ups * fix(databricks): the orphan drop leaves a function the deploy identity does not own * feat(databricks): the acquisition runner lands bronze into the Unity Catalog table A DuckDB acquisition build writing a `platform: databricks` expose no longer falls back to a local file. The runner masks at landing as before, writes Parquet, uploads it to DATABRICKS_STAGING_VOLUME through the Files API, loads it with one INSERT OVERWRITE ... SELECT FROM read_files(...) on DATABRICKS_WAREHOUSE_ID (the table's definition, grants, tags, row filter and masks stay), checks the row count, and deletes the staged files whether or not the load worked. A missing table fails naming `fluid apply`; the runner never creates or alters a table. Because the runner applies the mask before the data leaves the machine, the emitter now accepts policy.privacy.masking on an expose that only DuckDB acquisition builds write, and still refuses it for dbt and every other writer. * feat(databricks): lifecycle.expire is accepted on a full-refresh table whose Delta history fits the retention A Databricks table rebuilt in full every run holds no row older than a run except in Delta history, so expire is accepted when every build writing the expose is a full-refresh writer and lifecycle.retention covers the effective history (the larger of delta.logRetentionDuration, 30 days by default, and delta.deletedFileRetentionDuration, 7 days by default, or a dbt model's tblproperties). Full-refresh writers: a dbt build that selects a model resolving to materialized=table (SQL config(), then the model's YAML config, then dbt_project.yml, default view; anything not a plain literal is refused with the fix), and a DuckDB acquisition build, whose Databricks landing overwrites the whole table. ISO 8601 durations count years and months at their shortest, so a too-short retention never passes. New refusals: databricks-retention-incremental, -not-table, -dbt-unresolved, -too-short, -period; -unmapped stays for every other writer. fluid verify gains a retention dimension: it reads the table's properties and fails when the live history exceeds the contract, and warns when predictive optimization is off (VACUUM would then need scheduling). * fix(databricks): the landing reads only its own file and deletes it on an interrupt read_files now names the one Parquet file this run uploaded, not the run directory, so a file someone with WRITE VOLUME drops there is not loaded. Ctrl-C or SystemExit during the upload or the INSERT cancels the statement on the warehouse (SqlRunner) and still deletes the staged files before the interrupt propagates. A run killed outright cannot clean up, so each load first deletes what earlier runs of the same build left under /// and records it as facets.databricks_load.earlier_runs_staging_deleted. * fix(databricks): close the review's low findings on the landing, retention and orphan drop - the landing's error names the files it could not delete first, so a shortened message keeps them, and logs them in full - the dbt resolver does not walk a model path that reaches outside the project through a linked folder on the way - the orphan drop lists tables with include_browse, so a table the run may only browse makes the schema unreadable (nothing dropped) instead of a binding going unseen - verify's retention dimension warns when GetTable serves no properties, instead of passing on Delta's assumed defaults * docs(databricks): say why the landing uploads with one Files API PUT * refactor(databricks): the landing reuses the runner's path-part helper, and the orphan drop's per-function step is its own function * fix(databricks): close the inspection's findings on the landing, retention and docs - the INSERT OVERWRITE lists its target columns, so a live table that keeps its columns in another order (one fluid apply appended) gets each value in the column of its name, not by position - the stale-staging sweep deletes an earlier run's directory only when every file in it is over an hour old, so a concurrent run keeps its upload, and it follows next_page_token - retention accepts a DuckDB acquisition only with a source mode the landing runs (LOAD_MODES, full_refresh): another mode is databricks-retention- incremental, none databricks-retention-unmapped; the too-short message shows the declared Delta properties, not the defaults - the dbt config scan reads Jinja tags, comments and raw blocks in linear time (the lazy regexes were quadratic on unclosed openers), matching them on 200k random inputs - the runner and landing use public scrub/is_not_found names, and retention reuses util.contract.is_duckdb_acquisition_build - a DROP FUNCTION that fails is named and the apply still succeeds (tested) - CHANGELOG and HONESTLY_TESTED say what the code now does: masking at landing, orphan drop, lock identity, the second live run, counts * fix(databricks): the masking refusal starts its reason as a sentence * feat(databricks): an accessPolicy grant scoped by resources lands on the tables it names The emitter refused every grant with resources (databricks-grant-scope-unmapped), which blocked products whose grants carry $.exposes[?(@.exposeId=='')]. iac/access.py now parses resources once (AccessGrant.scope, parse_grant_scope): one expose by id (either quote, spaces allowed) or every expose ($.exposes[*], $.exposes). The Databricks emitter grants only on the matching tables. Any other JSONPath, alone or beside a good one, and an exposeId the contract does not have stay refused with a message that names the resource; covers() is false for an unevaluable scope, so a refused grant is never placed on more tables. AWS and GCP emit what they did: GCP still applies every grant to every expose. * fix(databricks): close the defect scan's and the second security pass's findings - fluid validate leaves a table name, retention or source mode that is still an {{ env.X }} placeholder to the emitter (it refused contracts fluid apply accepts), and fluid verify compiles the contract with its placeholders substituted, as apply does; a placeholder retention promises nothing - the emitter refuses what the DuckDB acquisition runner would refuse at run time after apply created the table: the source mode and sink format (databricks-landing-unsupported) and a masking rule it cannot apply (databricks-masking-invalid) - the stale-staging sweep deletes only directories named like a run id that hold nothing but Parquet files, all over an hour old; an empty directory, a subdirectory or another file means the directory is left - a blank accessPolicy resource is unevaluable, never "every expose" - _contract_key returns None for a contract JSON cannot key (mixed-type or date keys, a cycle) instead of crashing emit, and no pause decision is reused then - dbt fqn selectors match as dbt does: a prefix of the fqn with or without the project's name (shop, marts, marts.orders) - the too-short fix rounds the history up to whole days; a COUNT that fails after a successful INSERT says the load ran; the lock names an owner that is also the run's identity once * docs(databricks): CHANGELOG for the placeholder, landing, sweep and grant-scope fixes * fix(databricks): match dbt selectors as dbt-core does, and refuse an unresolved landing mode at apply - _fqn_selects ports dbt-core 1.11's is_selected_node / node_is_match (Apache-2.0): exact leaf name, else an exact fqn prefix up to the first wildcard part and one fnmatch of the rest, with and without the project's name. The per-part glob it replaces accepted ord*, ma*ts and *.core, which dbt does not select, so lifecycle.expire could be accepted for a model the build never rebuilds. Checked against dbt-core itself on 34,944 selector/fqn pairs. - _refuse_unlandable skips a placeholder mode or sink format only while validating; at apply one still unresolved (its variable unset) is refused before the table is created. * chore: refresh the secrets baseline's line numbers * test(databricks): one blank line between the license header and the docstring, as the writer makes it * refactor(databricks): scrub and is_not_found are the enforce module's public names; drop a pass-through lambda and repeated json imports --- CHANGELOG.md | 126 ++- HONESTLY_TESTED.md | 9 +- fluid_build/build_runners/_databricks_load.py | 547 +++++++++++ fluid_build/build_runners/duckdb/runner.py | 96 +- fluid_build/cli/_verify_databricks.py | 189 +++- fluid_build/cli/verify.py | 8 + fluid_build/iac/access.py | 62 +- fluid_build/iac/providers/databricks.py | 293 +++++- .../iac/providers/databricks_enforce.py | 222 ++++- .../iac/providers/databricks_retention.py | 887 ++++++++++++++++++ fluid_build/util/contract.py | 15 + .../test_duckdb_databricks_load.py | 717 ++++++++++++++ tests/cli/test_verify_databricks.py | 51 +- tests/cli/test_verify_databricks_retention.py | 263 ++++++ tests/iac/_fake_databricks.py | 351 ++++++- tests/iac/test_iac_databricks.py | 64 +- tests/iac/test_iac_databricks_enforce.py | 233 ++++- tests/iac/test_iac_databricks_fake_e2e.py | 9 +- tests/iac/test_iac_databricks_grant_scope.py | 175 ++++ .../test_iac_databricks_landing_fake_e2e.py | 207 ++++ .../test_iac_databricks_masking_at_landing.py | 286 ++++++ tests/iac/test_iac_databricks_retention.py | 856 +++++++++++++++++ 22 files changed, 5586 insertions(+), 80 deletions(-) create mode 100644 fluid_build/build_runners/_databricks_load.py create mode 100644 fluid_build/iac/providers/databricks_retention.py create mode 100644 tests/build_runners/test_duckdb_databricks_load.py create mode 100644 tests/cli/test_verify_databricks_retention.py create mode 100644 tests/iac/test_iac_databricks_grant_scope.py create mode 100644 tests/iac/test_iac_databricks_landing_fake_e2e.py create mode 100644 tests/iac/test_iac_databricks_masking_at_landing.py create mode 100644 tests/iac/test_iac_databricks_retention.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f08f25036..2973b0bc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -119,8 +119,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 against the live `row_filter` and each column's `mask` from the GetTable it already reads, and each bound function read back for its owner (this verify's identity or the table's owner), body, parameters and return type; a missing or different binding is CRITICAL with `fluid - apply` as the fix, and a contract that restricts nothing skips it. `policy.privacy.masking` - (landing-time masking), `rowLevelPolicy` and `binding.governance` are still refused. Proven + apply` as the fix, and a contract that restricts nothing skips it. `rowLevelPolicy` and + `binding.governance` are still refused, and `policy.privacy.masking` (landing-time masking) is + refused unless a DuckDB acquisition writes the expose (see the bronze landing entry). Proven against the in-process stand-in with a real `tofu` apply, and the function bodies evaluated in DuckDB; not run against a live workspace: that `CREATE FUNCTION IF NOT EXISTS`, `SET ROW FILTER` (with `ON ()` as well) and `SET MASK` behave as the stand-in models them, that `SET` @@ -130,6 +131,98 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 written, that `current_user()` is a service principal's application ID, and what a warehouse of each type enforces are unproven. +- **`lifecycle.expire` on a Databricks table is accepted when the table is rebuilt in full and + its Delta history is limited.** It used to be refused (`databricks-retention-unmapped`): + nothing on Databricks deletes rows by age. A table rewritten whole on every run holds no row + copy older than a run except in its Delta history, which `VACUUM` (run by predictive + optimization on Unity Catalog managed tables) empties after `delta.logRetentionDuration` + (30 days by default) and `delta.deletedFileRetentionDuration` (7 days). So the emitter and + `fluid validate` now accept `expire: true` when every build writing the expose is a dbt build + whose model for the table resolves to `materialized: table`, or forge-cli's DuckDB acquisition + with `properties.source.mode: full_refresh` (its Databricks landing overwrites the whole table, + and runs no other mode; another mode is `databricks-retention-incremental`, none `-unmapped`), + and `lifecycle.retention` is at least the larger of the two history properties (the model's + `tblproperties` when it sets them). The model is the one whose `alias`, else file name, is the + expose's table, with `materialized` resolved as dbt does: `config()` in the SQL, then the + model's YAML `config:`, then `dbt_project.yml` by the most specific path; the default is `view`. + The build must also run that model: its `--select` (`properties.model` / `select`, as + `build_selectors` makes it) must include the model by name, fully qualified name (dbt-core + 1.11's fqn method: an exact prefix, with or without the project's name, a wildcard globbing the + whole rest of the name) or path, else + `databricks-retention-not-selected` (graph operators on another node, tags, configs and state + cannot be evaluated here and are refused too). A table name, retention or source mode that is + still an `{{ env.X }}` placeholder is left by `fluid validate` to the emitter, which checks it + substituted, and `fluid verify` reads the contract with its placeholders substituted, as `fluid + apply` does. + Refused, with the fix: `databricks-retention-incremental` (`incremental`, `snapshot`), + `-not-table` (`view` and the rest), `-dbt-unresolved` (a Git repository, or a value that is not + a plain literal: `var()`, a conditional `config()`, a custom `schema`), `-too-short` + (a retention under the history, so under 30 days by default), `-period` (missing or not ISO 8601) + and `-unmapped` (any other writer, or none). ISO 8601 years and months are counted at their + shortest (P1Y = 365 days, P1M = 28 days), so the check never passes a retention that is too + short. `fluid verify` gains a `retention` dimension: the live table's Delta history + (`properties` and `delta_runtime_properties_kvpairs`) must not exceed the contract's retention + (CRITICAL, with the `ALTER TABLE ... SET TBLPROPERTIES`), and predictive optimization off for + the table is a warning with the fix (enable it, or schedule `VACUUM`), as is a table GetTable + serves without any properties (Delta's defaults were assumed; check `SHOW TBLPROPERTIES`); + skipped without `expire`. Not run against a live workspace: which property names Unity Catalog's GetTable + serves for a table's Delta retention, and what predictive optimization reports, are modelled + on databricks-sdk-go v0.190.0, not observed. + +- **A Databricks `accessPolicy` grant scoped by `resources` lands on the tables it names.** It used + to be refused (`databricks-grant-scope-unmapped`) whatever it named, which blocked every product + whose grants carry `resources: ["$.exposes[?(@.exposeId=='')]"]`. The emitter now evaluates + that shape (single or double quotes, spaces allowed) and `$.exposes[*]` / `$.exposes` (every + expose), and grants only on the matching tables; several resources add up. A grant without + `resources` still covers every table. Any other JSONPath (`$.exposes[0]`, a filter on another + field, `||`, a path below the expose), alone or beside a good one, and an exposeId this contract + does not have, stay `databricks-grant-scope-unmapped`, with a message that names the resource and + says the grant is refused rather than placed on more tables than it names; a blank resource + (`""`) is one of those, not "every expose". `fluid validate` and + the emitter read the one `_compile` verdict, and `fluid verify`'s catalog-access check reads the + per-table grants, so a scoped reader is checked only where it is granted. The scope is parsed + once, in `iac/access.py` (`AccessGrant.scope`); what the AWS and GCP emitters produce is + unchanged. Known gap, not fixed here: the GCP emitter applies every `accessPolicy` grant to every + expose and never reads `resources`, and the AWS emitter does not read `accessPolicy.grants`, so a + scoped grant is not scoped there. + +- **The DuckDB acquisition runner lands a Databricks expose in its table, and `policy.privacy.masking` + is accepted on one it writes.** For `platform: databricks` the runner used to write a LOCAL file + (`location.path`, else `out/`) and report success with nothing in Databricks. It now hashes (or + tokenizes, encrypts, masks) at landing as before, writes Parquet under `.fluid/staging`, uploads it + through the Files API to `DATABRICKS_STAGING_VOLUME` (`/Volumes///`, from + the environment only, like `DATABRICKS_WAREHOUSE_ID`) as `///.parquet`, + and runs one statement on the warehouse, `INSERT OVERWRITE ..
+ () SELECT FROM read_files('', format => + 'parquet')`: a full refresh, each value into the column of its name, that rewrites the rows and keeps the table's definition, grants, tags, row filter and column masks. + It reads only the file the run uploaded, never the directory, so a file someone else puts there is + not loaded. The staged files are deleted afterwards whether or not the load worked, and on Ctrl-C + too (an INSERT the warehouse has answered for is cancelled first); a file that cannot be deleted + fails the run and is named first in the error. A run killed outright cannot clean up, so each + load first deletes what earlier runs of the same build left under `///`: + only a directory named like a run id that holds nothing but Parquet files, all over an hour old + (a run still in progress keeps its upload, and a directory forge-cli did not stage is left). + Give the staging a volume of its own. The run record gains + `facets.databricks_load` (table, rows loaded, where it staged, that it deleted, and + `earlier_runs_staging_deleted`). The table must exist: a missing one fails naming + `fluid apply`, and the runner never creates or alters a table. An unset volume or warehouse, a + mode other than `full_refresh`, a sink other than Parquet and more than one stream are refused + before anything runs, and the emitter refuses the mode and the sink format already + (`databricks-landing-unsupported`), with any masking rule the runner cannot apply + (`databricks-masking-invalid`), so `fluid validate` says so before `fluid apply` creates the + table. Every other platform's landing is unchanged. The Databricks emitter now + accepts `policy.privacy.masking` on an expose when every build that writes it is a DuckDB + acquisition build (the runner applies the mask before the data leaves the machine, as on AWS), and + keeps refusing it, with the reason, for a dbt-written expose, another engine, or no build at all; + `fluid validate` and `fluid apply` read the one `_compile` verdict. `rowLevelPolicy` and + `binding.governance` are still refused. Proven against the in-process stand-in (Files API, + `INSERT OVERWRITE ... read_files` over the uploaded Parquet in DuckDB) with a real `tofu` apply + followed by the build; not run against a live workspace: what `read_files` and `INSERT OVERWRITE` + do on a managed table that carries a mask and a row filter, that `INSERT OVERWRITE` keeps them, + the Files API's limits, and whether a type the Parquet carries is accepted by the table's are + unproven. The run-time schema gate still compares a declared `string` with DuckDB's `VARCHAR` as a + type change; declare `varchar` for a column the source reads as text. + ### Fixed - **`fluid apply --mode amend-and-build` provisioned a product, then refused its id.** The build @@ -340,6 +433,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 provider". A contract's `redshift` build runtime or expose platform is served by `aws`, and plugins cannot claim either retired name. +- **A failed fail-safe no longer locks the run out, and says what it could not close.** When a + mask or row filter cannot be bound, the lock that revokes SELECT from every grantee but the + owner now also skips the identity the run is (it has to repair the table), and when the + table's owner could not be read it says the owner could not be skipped. An owner the contract + itself restricts reads its table whatever is revoked: the message now names that path as still + open, with the `ALTER TABLE ... OWNER TO` to run. When the permission listing shows no + `inherited_from_*` field at all, it says it could not determine whether a schema or catalog + grant is open, where it claimed none was. + +- **A job paused for an unprotected table is announced and found.** The apply decides once, in + `emit`, which dbt jobs to create paused, and its notice reads that decision, so a transient + read error can no longer pause a job without saying so. `fluid verify` now fails the jobs + dimension for a job that is paused while its table's masks or row filter are not in place, + with the apply that binds them and starts the job as the fix. + +- **A changed contract no longer leaves its old masks and row filter functions behind.** After + a successful apply (never in a dry run, never after a failed apply or a failed bind), the + functions in the product schema that carry the table's `fluid_rf_
_`, + `fluid_mask_
__` or deny-all name and are neither expected now nor bound to any + table in the schema (every table is read first; if one cannot be, nothing is dropped) are + dropped, each printed, and only when the run's own identity owns the function: one with + another owner, or an owner that cannot be read, is left and named. A function without the + `fluid_` prefix is never touched. + +- **A plan that fails the residency check prints one line.** The apply showed the last 800 + characters of the tofu JSON, which cut off the sentence naming the live region. It now says + the declared region, the live region (read from the diagnostic's message, or its `values`) + and the fix. + ### Security - **`validate_ident` let a name with a trailing newline through.** Python's `$` also matches just diff --git a/HONESTLY_TESTED.md b/HONESTLY_TESTED.md index def4a5a6b..a3ba82d76 100644 --- a/HONESTLY_TESTED.md +++ b/HONESTLY_TESTED.md @@ -85,12 +85,15 @@ Markers declared in `pyproject.toml`. | Stage 1 unit | `tests/iac/test_iac_databricks.py`, `test_iac_databricks_jobs.py`, `test_iac_databricks_auth.py`, `test_iac_databricks_engine_phase2.py`, `tests/test_util_cron.py`, `tests/build_runners/test_databricks_job_builds.py`, `tests/engines/test_dbt_databricks.py`, `tests/cli/test_verify_databricks.py`, `tests/test_doctor_databricks.py`, `tests/test_welcome_scan_databricks.py` | ✅ 521 tests in the Databricks files above (139 tables, 149 jobs, 56 credentials, 47 engine hooks, 45 cron, 31 runner, 40 verify, 14 doctor and welcome scan), plus the dbt profile and type tests. The emitted module: a `databricks_catalog` data source, the owned schema, a managed Delta table, per-principal grants with best-effort imports, the provider's `retry_timeout_seconds`. A declared `binding.location.region` (validated, lower-cased; `databricks-region-invalid` and `databricks-region-conflict` refused) adds `data.databricks_current_metastore` and a precondition on each owned schema; with none the module is unchanged. A contract's governance labels (`iac/governance_labels.py`, the AWS and GCP source) are one `databricks_entity_tag_assignment` per tag on each owned schema (the contract's own) and each table (the expose's too), `-` read as `_` in a key; two labels meeting at one key are `databricks-tag-collision`, more than 50 tags on a securable `databricks-tags-too-many`; a contract without labels gets the module it always did; each assignment has a best-effort import (`entity_type,entity_name,tag_key`); removing one is not data loss. Databricks is a region-placed platform for the `sovereignty` checks (`tests/policy/test_sovereignty_enforcement_mode.py`). Each dbt build on Databricks becomes a serverless job with its workspace files, mapped from the build: Quartz cron, triggers, retries, notifications, timeout and run-as. The project upload's allowlist and refusals; the escaped file names. Every refusal `fluid validate` and the emitter share. The credential policy (host, profile, auth type, blanked variables) and the SDK kwargs. The in-place data-loss and protocol-upgrade readings of a saved plan. Job and file removals left ungated; the `USE CATALOG` notice; the drift report for dbt-written tables. `fluid verify`'s Databricks inspector against a stubbed client (residency: pass, mismatch, skipped; governance tags: pass, a missing tag, a wrong value, the paginated list read to its last page with an empty page in the middle, skipped); the doctor rows and welcome-scan detection; the dbt runner leaving a Databricks job build to its job. | What Databricks accepts (the manual live row). The real databricks-sdk is neither in the dev install nor in CI, so a stubbed client stands in for it; one test answers as the live workspace did, with the privilege under the group that conveys it. | | Stage 1 `tofu validate` | `tests/iac/test_iac_tofu_validate.py` `[databricks]`, `test_a_databricks_dbt_job_passes_tofu_validate[git\|workspace]` | ✅ Both job modules (GIT with schedule, retries, notifications and run-as; WORKSPACE with a continuous trigger and files) and the dbt-written table's `ignore_changes` list validate against provider 1.140.0's schema. | Anything the provider checks only at plan or apply. | | Stage 1 real tofu, stand-in workspace: tables | `tests/iac/test_iac_databricks_fake_e2e.py` (`tests/iac/_fake_databricks.py`) | ✅ 28. The real `fluid` CLI, `tofu` and databricks/databricks 1.140.0 against an in-process stand-in for the Unity Catalog, SQL statement and permissions APIs. validate / plan / `generate iac --validate` / dry-run route to OpenTofu, and a second plan is empty. A partition change (a replace), a dropped column and a positional rename are refused without `--allow-data-loss`; a grant revoke is not. A column reorder is refused even with the flag. An added column applies in place. A table that exists with column mapping is adopted, not replaced; without it the apply is refused (with or without the flag) until its owner runs the `ALTER`, and the refused run, like a dry-run, leaves nothing it imported in state. Existing grants are adopted with their table: the adoption plan is `+0 ~1 -0` (the provider imports neither the table's properties nor its warehouse id, so the first apply writes back the values it already has), and the plan after it is empty. A privilege the contract does not name shows as an update that removes it. A declared `location.region` plans and applies when the stand-in's metastore is in it (case-insensitively) and fails the plan, naming both regions and `DATABRICKS_HOST`, before anything is written when it is not. A type change and a missing catalog fail at plan or import, and a missing host or credential fails before `tofu` starts; each names the fix before any change. `fluid diff` initialises with the committed lock. A contract's governance labels are created as tag assignments on the schema and the table in the stand-in and a second plan is empty; a changed value updates the tag in place; a removed label destroys only its assignment, without `--allow-data-loss`; an assignment already in the workspace (state lost, or set by hand with another value) is adopted and its value corrected, never a failed create, and a tag someone else set is left alone. | The stand-in models the requests provider 1.140.0 sends: SQL dialect, warehouse state and privilege names are not checked. Its metastore region is whatever the test sets; whether a real workspace's `metastore_summary` reports the region in the spelling a contract declares (`eu-central-1`, `westeurope`) is not proven: the residency check and the verify dimension were not run against a live workspace; the governance tags and their verify dimension were not run live either. The tag endpoints are modelled on the SDK's source (databricks-sdk-go v0.190.0, the one provider 1.140.0 builds on), not on a live workspace: the live tier was not run for tags, so what real Databricks accepts as a tag key or value, whether it keeps a table's tags across a dbt CREATE OR REPLACE (undocumented), and whether the import id `entity_type,entity_name,tag_key` resolves a live assignment are unproven; `fluid verify`'s governance-tags dimension is the gate for the last. It stores the table properties it was sent, so the adoption check reads `properties` / `effective_properties` as a real workspace may not fill them. | -| Stage 1 unit: Databricks column masks and row filters | `tests/iac/test_iac_databricks_access.py` (the SQL), `tests/iac/test_iac_databricks_enforce.py` (the hooks and the engine's gate), `tests/cli/test_verify_databricks.py` (21 of its 61), `tests/iac/_databricks_governed.py`, `tests/iac/_fake_databricks.py` | ✅ 97 + 84 + 21. A Customer-Profile-360-shaped contract (six denied columns, a `msisdn` mask, a consent row filter, principals mapped to `fluid-demo-lab-*` groups) compiles to the exact `CREATE FUNCTION IF NOT EXISTS` / `ALTER TABLE ... SET ROW FILTER ... ON (...)` / `ALTER COLUMN ... SET MASK` statements. The function bodies are evaluated in DuckDB (transpiled from the Databricks dialect, `is_account_group_member` stubbed): a restricted group reads only the rows its predicate keeps, a member of several filtered principals reads the union, everyone else every row, a NULL predicate hides the row from a member; `last_four` / `first_four` / `nullify` and a deny give the BigQuery-matching values, a value of four characters or fewer is its SHA-256, a NULL stays NULL, a deny beats a mask, and a mask rule belongs to the principal that names it (nullify for one group and last_four for another on one column: strictest branch first, a member of both reads NULL). Every identity is tested as a group and as a user, so a `group:` principal mapped to a service principal's application ID or a user's email is still restricted (DuckDB's own session user stands in for a service principal that is in no group). Names carry a hash of the definition: the same contract names the same functions, a changed predicate or rule names a new one. Every refusal kind, 30 predicate injection attempts (`;`, comments, subqueries, session functions, unknown or qualified columns, a backslash-escaped quote) and a principal name with a quote or a backslash. The hooks against the stand-in (statements, Tables API reads): pre_apply binds existing tables and leaves a missing table to post_apply, a dry run prints and executes nothing, a second run executes nothing, a changed predicate binds a new function and keeps the old, a removal drops only what this module bound, a statement needing a column the apply adds waits for post_apply, a failed pre_apply stops the run and revokes nothing, a failed post_apply binds a deny-all row filter for the restricted identities (SELECT untouched), and when that cannot be bound, or would replace a filter not made here, revokes SELECT and ALL PRIVILEGES from every grantee but the owner, lists the schema and catalog grants still open with the statement to run, and reports a revoke that failed or a grant listing that failed; a function pre-created under any hashed name (row filter, mask, deny-all) by another owner, with another body, parameters or return type, is never bound; a row filter or mask set by hand is never replaced, with nothing changed on any table; a Tables API read that fails for one table (503, 429) is a failed bind for it and the next table is still tried; a filter needing a column the apply adds is held by the deny-all filter in pre_apply; a dbt job (cron or continuous) writing a table that does not exist, is unbound, has a wrong function or cannot be read is created PAUSED, and a manual job or a job writing no restricted table is left alone, after a `tofu apply` that failed part way, post_apply binds what exists, drops nothing, skips a table never created without a NOT REVOKED message, and the engine prints an enforcement failure there without raising it; the SQL statement poller (running, failed, cancelled, timed out, credentials scrubbed). The engine calls the hooks in order around `tofu apply`. `fluid generate iac` refuses a module with masks or filters (emit stays pure). `fluid verify`'s row-filter-and-masks dimension, including the owner, body, parameters and return type of every bound function (pass, no filter, another function, other input columns, missing and wrong mask, case and backticks ignored, skipped, the CLI output and `--strict`). | What Databricks accepts. The stand-in executes the statements as this module sends them and models the Unity Catalog refusal of `CREATE OR REPLACE FUNCTION` on a bound function, the preserved bindings across `CREATE OR REPLACE TABLE`, and the Tables API fields `row_filter.function_name` / `input_column_names` and `columns[].mask.function_name` (databricks-sdk-go v0.190.0), all from the docs and the SDK source, not from a live workspace. DuckDB evaluates the bodies, not Databricks: `sha2`, `right`, `left` and `is_account_group_member` are Databricks' own. | -| Stage 1 real tofu, stand-in workspace: masks and row filters | `tests/iac/test_iac_databricks_fake_e2e.py` (17 of its 45; needs databricks-sdk, the `[databricks]` extra, and skips without it) | ✅ 17. The real `fluid` CLI and `tofu` apply bind the filter and masks (after the apply, on a table it created); a second apply executes no statement; a table that already exists is bound before the apply's first grant on it; `--mode dry-run` prints the statements and executes none; a changed predicate binds a new function and keeps the old; a removed restriction is dropped and said to widen access without `--allow-data-loss`; a binding that fails revokes SELECT from the restricted principal only, fails the run (`opentofu_access_enforcement_failed`), and the next apply binds before it grants again; a failure before the apply stops it with nothing applied; an invalid predicate is refused before anything runs; an apply that fails after creating the table (the grant on it refused) still binds the filter and masks and the run exits with `opentofu_apply_failed`, an apply that fails before the table exists binds nothing and revokes nothing, and a failed apply drops no binding the contract no longer asks for (it is dropped by the next apply that succeeds); a function taken under the hashed name stops the apply before it runs; a row filter set by hand stops it and is left in place; a binding that fails with no deny-all possible revokes every grantee but the owner and the next apply binds first and grants again; a failed mask leaves the deny-all filter and every grant until the next apply; a filter needing a column the apply adds is held by the deny-all filter, then replaced, in that order around the added column; a dbt job created with its restricted table is paused and the next apply unpauses it in place; `generate iac` refuses the module. | Live behaviour: that `CREATE FUNCTION IF NOT EXISTS`, `SET ROW FILTER` and `SET MASK` run on a real warehouse as modelled, that a second `SET` replaces a binding already on the table (if it does not, a changed predicate fails closed with the old binding in place), that `current_user()` of a service principal is its application ID, that a Free Edition or classic warehouse can run them, and whether the SDK's statement API answers as the stand-in does; whether `SET ROW FILTER ... ON ()` (a zero-column function) is accepted; whether FunctionInfo's `routine_definition`, `input_params` and `full_data_type` come back in the form compared (whitespace-normalised, lower-case types), whose mismatch would refuse a good function; whether `scim/v2/Me` `userName` is the owner `CREATE FUNCTION` records for a service principal; that `SET` replaces a bound filter or mask (the stand-in does; none of the fixes uses a DROP first, which would leave a window with no filter); the permissions and effective-permissions fields used to lock a table; and the pause behaviour of a job on a live workspace. Unproven for ALL_PRIVILEGES grantees on a live workspace: a revoke of ALL PRIVILEGES removes more than read access, by design of the lock. A principal that is a metastore admin or the table's owner is never restricted by these bindings. | +| Stage 1 unit: Databricks column masks and row filters | `tests/iac/test_iac_databricks_access.py` (the SQL), `tests/iac/test_iac_databricks_enforce.py` (the hooks and the engine's gate), `tests/cli/test_verify_databricks.py` (21 of its 61), `tests/iac/_databricks_governed.py`, `tests/iac/_fake_databricks.py` | ✅ 97 + 101 + the verify dimension's. The job pause is decided once, in `emit`, and the notice reads that decision; `fluid verify` fails the jobs dimension for a job paused while its table's masks or row filter are not in place; a plan that fails the residency check prints one line with the declared region, the live region and the fix (`databricks-residency-region`). A Customer-Profile-360-shaped contract (six denied columns, a `msisdn` mask, a consent row filter, principals mapped to `fluid-demo-lab-*` groups) compiles to the exact `CREATE FUNCTION IF NOT EXISTS` / `ALTER TABLE ... SET ROW FILTER ... ON (...)` / `ALTER COLUMN ... SET MASK` statements. The function bodies are evaluated in DuckDB (transpiled from the Databricks dialect, `is_account_group_member` stubbed): a restricted group reads only the rows its predicate keeps, a member of several filtered principals reads the union, everyone else every row, a NULL predicate hides the row from a member; `last_four` / `first_four` / `nullify` and a deny give the BigQuery-matching values, a value of four characters or fewer is its SHA-256, a NULL stays NULL, a deny beats a mask, and a mask rule belongs to the principal that names it (nullify for one group and last_four for another on one column: strictest branch first, a member of both reads NULL). Every identity is tested as a group and as a user, so a `group:` principal mapped to a service principal's application ID or a user's email is still restricted (DuckDB's own session user stands in for a service principal that is in no group). Names carry a hash of the definition: the same contract names the same functions, a changed predicate or rule names a new one. Every refusal kind, 30 predicate injection attempts (`;`, comments, subqueries, session functions, unknown or qualified columns, a backslash-escaped quote) and a principal name with a quote or a backslash. The hooks against the stand-in (statements, Tables API reads): pre_apply binds existing tables and leaves a missing table to post_apply, a dry run prints and executes nothing, a second run executes nothing, a changed predicate binds a new function and, after an apply that succeeds, drops the old one (only when no table in the schema uses it and this run's identity owns it; never after a failed apply or bind, nor in a schema whose tables cannot all be read; a drop that fails is named and the apply still succeeds), a removal drops only what this module bound, a statement needing a column the apply adds waits for post_apply, a failed pre_apply stops the run and revokes nothing, a failed post_apply binds a deny-all row filter for the restricted identities (SELECT untouched), and when that cannot be bound, or would replace a filter not made here, revokes SELECT and ALL PRIVILEGES from every grantee but the owner and this run's identity (naming an owner it could not read, an owner the contract restricts as a path still open, and a permission listing that shows no `inherited_from_*` field as undetermined), lists the schema and catalog grants still open with the statement to run, and reports a revoke that failed or a grant listing that failed; a function pre-created under any hashed name (row filter, mask, deny-all) by another owner, with another body, parameters or return type, is never bound; a row filter or mask set by hand is never replaced, with nothing changed on any table; a Tables API read that fails for one table (503, 429) is a failed bind for it and the next table is still tried; a filter needing a column the apply adds is held by the deny-all filter in pre_apply; a dbt job (cron or continuous) writing a table that does not exist, is unbound, has a wrong function or cannot be read is created PAUSED, and a manual job or a job writing no restricted table is left alone, after a `tofu apply` that failed part way, post_apply binds what exists, drops nothing, skips a table never created without a NOT REVOKED message, and the engine prints an enforcement failure there without raising it; the SQL statement poller (running, failed, cancelled, timed out, credentials scrubbed). The engine calls the hooks in order around `tofu apply`. `fluid generate iac` refuses a module with masks or filters (emit stays pure). `fluid verify`'s row-filter-and-masks dimension, including the owner, body, parameters and return type of every bound function (pass, no filter, another function, other input columns, missing and wrong mask, case and backticks ignored, skipped, the CLI output and `--strict`). | What Databricks accepts. The stand-in executes the statements as this module sends them and models the Unity Catalog refusal of `CREATE OR REPLACE FUNCTION` on a bound function, the preserved bindings across `CREATE OR REPLACE TABLE`, and the Tables API fields `row_filter.function_name` / `input_column_names` and `columns[].mask.function_name` (databricks-sdk-go v0.190.0), all from the docs and the SDK source, not from a live workspace. DuckDB evaluates the bodies, not Databricks: `sha2`, `right`, `left` and `is_account_group_member` are Databricks' own. | +| Stage 1 real tofu, stand-in workspace: masks and row filters | `tests/iac/test_iac_databricks_fake_e2e.py` (17 of its 45; needs databricks-sdk, the `[databricks]` extra, and skips without it) | ✅ 17. The real `fluid` CLI and `tofu` apply bind the filter and masks (after the apply, on a table it created); a second apply executes no statement; a table that already exists is bound before the apply's first grant on it; `--mode dry-run` prints the statements and executes none; a changed predicate binds a new function and the apply drops the old one; a removed restriction is dropped and said to widen access without `--allow-data-loss`; a binding that fails revokes SELECT from the restricted principal only, fails the run (`opentofu_access_enforcement_failed`), and the next apply binds before it grants again; a failure before the apply stops it with nothing applied; an invalid predicate is refused before anything runs; an apply that fails after creating the table (the grant on it refused) still binds the filter and masks and the run exits with `opentofu_apply_failed`, an apply that fails before the table exists binds nothing and revokes nothing, and a failed apply drops no binding the contract no longer asks for (it is dropped by the next apply that succeeds); a function taken under the hashed name stops the apply before it runs; a row filter set by hand stops it and is left in place; a binding that fails with no deny-all possible revokes every grantee but the owner and this run's identity and the next apply binds first and grants again; a failed mask leaves the deny-all filter and every grant until the next apply; a filter needing a column the apply adds is held by the deny-all filter, then replaced, in that order around the added column; a dbt job created with its restricted table is paused and the next apply unpauses it in place; `generate iac` refuses the module. | Live behaviour: that `CREATE FUNCTION IF NOT EXISTS`, `SET ROW FILTER` and `SET MASK` run on a real warehouse as modelled, that a second `SET` replaces a binding already on the table (if it does not, a changed predicate fails closed with the old binding in place), that `current_user()` of a service principal is its application ID, that a Free Edition or classic warehouse can run them, and whether the SDK's statement API answers as the stand-in does; whether `SET ROW FILTER ... ON ()` (a zero-column function) is accepted; whether FunctionInfo's `routine_definition`, `input_params` and `full_data_type` come back in the form compared (whitespace-normalised, lower-case types), whose mismatch would refuse a good function; whether `scim/v2/Me` `userName` is the owner `CREATE FUNCTION` records for a service principal; that `SET` replaces a bound filter or mask (the stand-in does; none of the fixes uses a DROP first, which would leave a window with no filter); the permissions and effective-permissions fields used to lock a table; and the pause behaviour of a job on a live workspace. Unproven for ALL_PRIVILEGES grantees on a live workspace: a revoke of ALL PRIVILEGES removes more than read access, by design of the lock. A principal that is a metastore admin or the table's owner is never restricted by these bindings. | +| Stage 1: Databricks bronze landing and masking at landing (live not run) | `tests/build_runners/test_duckdb_databricks_load.py` (the runner and the loader), `tests/iac/test_iac_databricks_masking_at_landing.py` (the emitter), `tests/iac/test_iac_databricks_landing_fake_e2e.py` (real `fluid` + `tofu` + databricks-sdk; skips without the `[databricks]` extra), `tests/iac/_fake_databricks.py` | ✅ A `platform: databricks` binding no longer writes a local file: the run stages Parquet under `.fluid/staging`, uploads it to `DATABRICKS_STAGING_VOLUME` through the Files API, runs one `INSERT OVERWRITE ... read_files` over the one file it uploaded (a file someone else puts in the run's directory is not loaded) and deletes the staged files (also after a failed upload, a failed or short load and a Ctrl-C, which cancels the statement first; one that cannot be deleted fails the run naming it first); each load first deletes an earlier run's staging of the same build once all of it is over an hour old, and leaves a recent one. The statement's text, quoting and column list are pinned, as are the volume's validation, the staging path (no contract id climbs out), the refusals before any work (unset volume or warehouse, mode, sink, stream count), a missing table (names `fluid apply`, uploads nothing), credential scrubbing and the matrix of when the emitter accepts `policy.privacy.masking` (every writer a DuckDB acquisition build) against when it refuses it, through both `validate` and `emit`. End to end against the stand-in: `fluid apply` creates the table with its mask and row filter, `--mode amend-and-build` loads the rows with the masked column hashed and the cleartext absent, the volume is empty, nothing but the load runs on the warehouse, the table is not replaced, a second plan is empty, and a second run overwrites the rows. | **Not run live.** The stand-in is not Databricks: it reads the uploaded Parquet with DuckDB, so what `read_files` accepts, that `INSERT OVERWRITE` keeps a managed table's row filter and column masks, implicit casts from the Parquet types to the table's, the Files API's limits and a real warehouse's permissions are unproven. The run-time schema gate reads a declared `string` as a type change against DuckDB's `VARCHAR`, so a contract declares `varchar` for a text column. | | Stage 1 real tofu, stand-in workspace: dbt jobs | `tests/iac/test_iac_databricks_jobs_fake_e2e.py` | ✅ 6. A local dbt build applies as one serverless job plus the workspace files dbt reads, each in a folder created before it (no upload meets a missing parent) (never `profiles.yml`, `target/` or a hidden file), and a second plan is empty. A changed model updates its file in place. A GIT build runs its tag on a continuous trigger and uploads nothing. Turning a job build local removes the job, its files and their folders without `--allow-data-loss`, and says so; a folder holding a file the apply did not put there is not deleted. The table a dbt build writes keeps what dbt's CREATE OR REPLACE made of it, and the apply reports the column drift instead of failing on a type change. | Whether Databricks accepts and runs the job: serverless environment version 2, the `dbt-databricks>=1.8.0,<2.0.0` range, the commands, a WORKSPACE `project_directory` under `/Workspace`, `run_as` permissions, a Git credential for a private repository, `dbt deps` with a local package (not uploaded). The manual live row proves the first four. Whether the jobs API echoes a server default the stand-in does not (the fake's builder found `performance_target` and an all-false `notification_settings` would each plan an update), which would make a second plan non-empty (live, it was empty). What dbt's CREATE OR REPLACE does to a table with column mapping (the module sets none on a dbt-written table). Not bound by a plan: an apply uploads the dbt project as it is when it runs (`planDigest` and `bundleDigest` cover the contract, not the project's files, as for a local dbt run), and a bundle does not carry the project. Known gap, by design: a job is not adopted. Its import id is a job id, which neither the contract nor `tofu` can look up by name, so an apply that lost its state creates a second job. `fluid verify` flags the duplicate (a stubbed client proves it). Workspace files need no import: the provider's create overwrites. | +| Stage 1 unit: Databricks retention (`lifecycle.expire`) | `tests/iac/test_iac_databricks_retention.py`, `tests/cli/test_verify_databricks_retention.py` | ✅ 195 + 18. ISO 8601 durations (years and months at their shortest, P1Y = 365 and P1M = 28 days; a fraction, a sign, a lower-case letter, `PT`, `P1DT`, out-of-order and over-long numbers refused), Delta `interval` properties (rounded up, months refused) and the history as the larger of the two properties with Delta's defaults. A dbt model's `materialized` resolved from `config()` in the SQL, the model's YAML `config:` and `dbt_project.yml` (the `+` prefix or none, root, project, folder and model-name levels, the most specific wins, SQL over YAML over project), the default `view`, the model found by alias or file name, commented-out and `raw` configs ignored, a `config()` inside a Jinja block, `var()`, `**`, a custom schema or database, a Python model, a versioned or twin model, a linked file and a Git repository refused as undeterminable. The build's `--select` must include the resolved model (name, `fqn`, `path:`, `*`, unions and intersections; a graph operator on another node, a tag, a config or run state is refused as not evaluable). Accepted: a dbt `table` model and a DuckDB acquisition with P30D, P365D, P6Y; refused: P7D, P29D, P4W, P1M, a history set longer than the retention, `incremental`, `snapshot`, `view`, another acquisition engine, no writer, a mixed set of writers. `fluid validate` and the emitter agree on kind and message. `fluid verify`'s retention dimension against a stubbed client and, over HTTP, the stand-in workspace (pass, fail with the `ALTER TABLE`, the runtime property over the declared one, skip, predictive optimization off as a warning). | Live behaviour: no real Databricks endpoint was called. Whether Unity Catalog's GetTable serves a table's `delta.logRetentionDuration` and `delta.deletedFileRetentionDuration` in `properties` or `delta_runtime_properties_kvpairs` (read from databricks-sdk-go v0.190.0 `model.go`, not observed), whether `effective_predictive_optimization_flag` is filled for a managed table, that predictive optimization runs `VACUUM` within the retention (a Databricks guarantee, not tested), and that dbt-databricks `table` replaces the table each run. Delta history is only half the promise: the acquisition acceptance rests on the landing being an `INSERT OVERWRITE` of the whole table in the one mode it runs (`full_refresh`, read from the landing's own `LOAD_MODES`; another source mode is refused as `incremental`, none as `unmapped`), which the bronze landing row covers against the stand-in, not live. A GetTable answer with no properties is a verify warning (Delta's defaults assumed). Embedded-logic DuckDB builds, `dlt`, `airbyte` and the other acquisition engines are refused. Selection is matched on names and paths only, not on dbt's graph: `+other+` is refused even if `other` is upstream of the model. | | Manual live, Free Edition (founder-approved) | the real `fluid` CLI, no test file: run 2026-11-01 from main dfd6f145, databricks-sdk 0.158.0, every `fluid_iactest_*` object swept after (1 schema, 1 table, 1 folder; 0 left) | ✅ validate, `generate iac --validate`, plan, dry-run and apply `+9`; a second plan is EMPTY against the real Jobs API; Databricks accepts the Quartz schedule, serverless environment 2 with `dbt-databricks>=1.8.0,<2.0.0` (dbt 1.12.6, adapter 1.13.0), the WORKSPACE `project_directory` and the commands as emitted; one `run-now` ends SUCCESS and dbt builds the table; `fluid verify` with the real SDK (columns, types, one job); `fluid diff`'s state check (its live check has no Databricks inspector); a model change `~1`; the build made local removes job, files and folders `-5` without `--allow-data-loss` (the empty `.fluid//` folders above `files/` stay: only `files/` and below are managed). It found the USE CATALOG check reading a group-conveyed privilege as none. After the fix, `fluid verify` on a fresh one-table product (`+4`) passes every dimension, catalog access included (1 schema and 1 table swept, 0 left). | A GIT source, profile or OAuth auth, a run-as service principal, table adoption and a larger project, live; the region residency check (the metastore precondition and `fluid verify`'s residency dimension), which was added after this run and not run live; dbt's CREATE OR REPLACE drops a contract's NOT NULL, so verify flags a `required` column a dbt model does not constrain itself. | +| Manual live, Free Edition (founder-approved), masks and row filters | the real `fluid` CLI, no test file: run 2026-11-03 at the tip of #785, a real workspace with an AWS us-west-2 metastore; every object swept after (0 left) | ✅ 10 steps passed. Functions are created and read back (owner is the user's email; `routine_definition` is the body without `RETURN`, backticks kept; `full_data_type` is STRING or BOOLEAN; a zero-parameter function's FunctionInfo has no `input_params`). The row filter and masks are bound and enforced on the restricted user's own SELECT: only the consented rows, `msisdn` as `XXXXX4567`, a 3-character value hashed, the denied column NULL. `SET ROW FILTER ... ON ()` is accepted and a `SET` replaces a live filter. Tags are present, `fluid verify` passes every dimension, a second apply is `+0 ~0 -0`, the residency precondition fails the plan and names both regions, `generate iac` refuses, and the sweep left 0. | Still open, as this run showed: superseded functions were left behind (orphans; the follow-up that drops them after a successful apply is not run live); `tofu destroy` of the module fails at a schema that still holds functions, so a teardown must drop them first; and the residency failure text was forge-cli's 800-character tail of the tofu JSON, which cut off the sentence naming the live region (the one-line explanation is not run live). A metastore in another cloud or region, a service principal as the run identity, and a group-conveyed grant on the live workspace are still not run. | | Stage 2 emulator (nightly, non-gating) | `tests/iac/test_iac_databricks_minilake.py` (job `nightly-databricks-minilake`) | ⚠️ first run 2026-10-30 (manual dispatch, run 41111425504, minilake v1.9.0): 1/4. Schema, table and grant creates work, and the replace gate holds. | A second plan is not empty and every in-place table update fails: minilake strips `TBLPROPERTIES` at create and its SQL parser rejects `ALTER TABLE ... SET TBLPROPERTIES`, which the provider sends before each column change. An upstream issue is drafted, not filed. Jobs and workspace files are not in this flow yet. | -| Stage 3 live | — | ❌ not automated: manual maintainer runs only, never CI. One has run: the manual row above (2026-11-01). | Everything the rows above leave to the live tier. | +| Stage 3 live | — | ❌ not automated: manual maintainer runs only, never CI. Two have run: the manual rows above (2026-11-01 and 2026-11-03). | Everything the rows above leave to the live tier. | ## What's NOT tested (deliberately deferred) diff --git a/fluid_build/build_runners/_databricks_load.py b/fluid_build/build_runners/_databricks_load.py new file mode 100644 index 000000000..969c0b7ab --- /dev/null +++ b/fluid_build/build_runners/_databricks_load.py @@ -0,0 +1,547 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Land a build's Parquet file in the Unity Catalog table ``fluid apply`` created. + +A Databricks binding is not a file destination. The acquisition runner used to treat its +``location.path`` (or ``out/``) as one: it wrote a LOCAL file and reported success with nothing in +Databricks. Now the build lands its file under ``.fluid/staging`` (after the runner's own masking, +which hashes, tokenizes, encrypts or masks columns before the data leaves the machine), and this +module moves it into the table in three steps: + +1. upload the file to a Unity Catalog volume through the Files API, + ``////.parquet``; +2. run ONE statement on ``DATABRICKS_WAREHOUSE_ID`` through the SQL Statement Execution API:: + + INSERT OVERWRITE `c`.`s`.`t` (`col`, ...) SELECT `col`, ... FROM read_files('', format => 'parquet') + + reading the one file this run uploaded, never the directory (a file someone else put there is not + loaded), a full refresh that rewrites the table's rows and nothing else: the table's definition, grants, + tags, row filter and column masks stay (``INSERT OVERWRITE`` is a data operation; ``CREATE OR + REPLACE`` is the statement that rebuilds a table); +3. delete the staged files and their directory, whether the load worked or not. The volume must never + keep landed data (masked, but still the product's data). A file that cannot be deleted fails the + run and is named, so that someone deletes it. Ctrl-C (or any other interrupt) during the upload or + the INSERT still deletes the files before it propagates. An INSERT the warehouse has already + answered for is cancelled first; one still inside its first 30-second wait runs on, and Delta + commits it whole or not at all. A run killed outright (SIGKILL, a CI runner torn down) cannot clean up, so each + load first deletes what an earlier run of the same build left under + ``///``, once all of it is over an hour old. + +The table is the IaC's: a missing table is an error that names ``fluid apply``, and this module never +creates or alters one. The volume is named by ``DATABRICKS_STAGING_VOLUME`` +(``/Volumes///``), from the environment only, like +``DATABRICKS_WAREHOUSE_ID``: a contract that named a path could aim the upload anywhere the +credential reaches. Give the staging a volume of its own: the sweep of earlier runs deletes only +directories named like a run id that hold nothing but old Parquet files, but it runs under the +deploy credential. The client and the statement runner are the ones that enforce masks and row +filters (``iac/providers/databricks_enforce.py``), on the same pinned credential; no message carries +a credential. + +Not proven without a workspace: what ``read_files`` and ``INSERT OVERWRITE`` accept on a managed table +carrying a mask and a row filter, and the Files API's limits. ``tests/iac/_fake_databricks.py`` stands +in for them. +""" + +from __future__ import annotations + +import logging +import os +import re +import time +from dataclasses import dataclass +from typing import Any, Dict, List, Mapping, Optional, Tuple +from urllib.parse import quote + +from ..iac.provider_match import is_cloud +from ..iac.providers.databricks_access import quote as quote_ident +from ..iac.providers.databricks_access import quote_table, string_literal +from ..iac.providers.databricks_enforce import ( + DATABRICKS_EXTRA_INSTALL, + WAREHOUSE_ENV, + SqlRunner, + StatementFailed, + default_client, + is_not_found, + read_table, + scrub, +) +from ..providers._sql_safety import validate_ident +from ..util.binding_paths import ENV_PLACEHOLDER_RE + +#: The Unity Catalog volume the run's file is staged in: ``/Volumes///``. +STAGING_VOLUME_ENV = "DATABRICKS_STAGING_VOLUME" + +_VOLUME_FORM = "/Volumes///" +_VOLUME_RE = re.compile(r"^/Volumes/([^/]+)/([^/]+)/([^/]+)/?$") +#: How old every file of an earlier run's staging directory must be before a load deletes it: +#: well past a load's own span (the INSERT is cancelled after 600 s), so a run of the same build +#: still in progress keeps its upload. +STALE_STAGING_SECONDS = 3600 +#: The only mode a load supports: ``INSERT OVERWRITE`` replaces the table's rows. +LOAD_MODES = ("full_refresh",) +_SINK_FORMATS = ("parquet",) + + +class DatabricksLoadError(RuntimeError): + """The landed file could not be loaded into the table, or loaded the wrong number of rows.""" + + +@dataclass(frozen=True) +class StagingVolume: + catalog: str + schema: str + volume: str + + @property + def path(self) -> str: + return f"/Volumes/{self.catalog}/{self.schema}/{self.volume}" + + +@dataclass(frozen=True) +class LoadTarget: + """The table a Databricks expose lands in, named as the emitter creates it.""" + + catalog: str + schema: str + table: str + #: The contract's columns, in table order. + columns: Tuple[str, ...] + + @property + def full_name(self) -> str: + return f"{self.catalog}.{self.schema}.{self.table}" + + +def staging_volume(env: Mapping[str, str]) -> StagingVolume: + """``DATABRICKS_STAGING_VOLUME`` as three identifiers, or a refusal that names the variable.""" + raw = str(env.get(STAGING_VOLUME_ENV) or "").strip() + if not raw: + raise DatabricksLoadError( + f"{STAGING_VOLUME_ENV} is not set. The build stages its Parquet file in a Unity Catalog " + f"volume before loading the table; set it to {_VOLUME_FORM}." + ) + match = _VOLUME_RE.match(raw) + parts = match.groups() if match else () + try: + for part in parts: + validate_ident(part) + except ValueError: + parts = () + if len(parts) != 3: + raise DatabricksLoadError( + f"{STAGING_VOLUME_ENV} is {raw[:120]!r}, which is not {_VOLUME_FORM}, with each part a " + "plain identifier (letters, digits and underscores)." + ) + return StagingVolume(*parts) + + +def run_prefix(volume: StagingVolume, product_id: Any, build_id: Any, run_id: Any) -> str: + """The directory one run stages in: ``///``.""" + from .duckdb.runner import _path_part # one safe path component, as the local staging uses + + parts = (_path_part(product_id), _path_part(build_id), _path_part(run_id)) + return "/".join((volume.path, *parts)) + + +def _resolved(value: Any) -> Any: + """``{{ env.NAME }}`` in a name, resolved as the runner resolves a bucket; unset ones stay.""" + if not isinstance(value, str) or "{{" not in value: + return value + return ENV_PLACEHOLDER_RE.sub(lambda m: os.environ.get(m.group(1), m.group(0)), value) + + +def databricks_load_target( + binding: Mapping[str, Any], expose: Mapping[str, Any] +) -> Optional[LoadTarget]: + """The table a binding loads into, or ``None`` when it is not a Databricks binding. + + Resolved with the emitter's own helpers, so the load names the catalog, schema, table and + columns ``fluid apply`` created, and every name has been through the same identifier check. + A Databricks binding that cannot be resolved raises :class:`DatabricksLoadError` with the + emitter's reasons. + """ + if not is_cloud(binding, "databricks"): + return None + from ..iac.providers import databricks as dbx + + where = f"exposes[{expose.get('exposeId')}]" + raw_loc = binding.get("location") + loc: Dict[str, Any] = dict(raw_loc) if isinstance(raw_loc, Mapping) else {} + loc.update({k: _resolved(loc.get(k)) for k in ("database", "schema", "table")}) + refusals: List[Any] = [] + names = dbx._location_names(where, loc, refusals, validating=False) + columns = dbx._columns(where, expose, refusals, validating=False) + if refusals: + raise DatabricksLoadError("; ".join(str(r) for r in refusals)) + return LoadTarget( + catalog=str(names["database"]), + schema=str(names["schema"]), + table=str(names["table"]), + columns=tuple(column["name"] for column in columns), + ) + + +def unsupported_reason(mode: str, sink_format: str, stream_count: int) -> Optional[str]: + """Why this build cannot be loaded into a Databricks table, or ``None`` if it can. + + Checked before the build runs, so an unsupported shape fails by name instead of landing + somewhere else and reporting success. + """ + if stream_count != 1: + return f"a Databricks binding loads exactly one stream; this build has {stream_count}" + if sink_format not in _SINK_FORMATS: + return f"a Databricks load takes {list(_SINK_FORMATS)}; the sink format is {sink_format!r}" + if mode not in LOAD_MODES: + return ( + f"a Databricks load is an INSERT OVERWRITE, so it supports modes {list(LOAD_MODES)}; " + f"this build is {mode!r}" + ) + return None + + +def insert_overwrite_statement(target: LoadTarget, source: str) -> str: + """The one statement that loads the run's file: every name an identifier the emitter + accepted, quoted; the path a Spark string literal (backslash escapes). + + The target columns are listed, so each value goes to the column of its name whatever order + the live table keeps them in (a column added later by ``fluid apply`` sits at the end, not + where the contract declares it); without the list Databricks inserts by position. + """ + columns = ", ".join(quote_ident(column) for column in target.columns) + return ( + f"INSERT OVERWRITE {quote_table(target.full_name)} ({columns}) SELECT {columns} " + f"FROM read_files({string_literal(source)}, format => 'parquet')" + ) + + +#: A run directory's name: the acquisition runner's run id (``01`` + 16 Crockford base32 +#: characters, ``_acquisition_common.generate_run_id``). The sweep touches nothing else. +_RUN_DIR_RE = re.compile(r"01[0-9A-HJKMNP-TV-Z]{16}") + + +def _stale_files(listed: Any, run_dir: str, cutoff_ms: float) -> Optional[List[str]]: + """The files of an earlier run's directory listing, when it looks like what a load stages + (Parquet files only, no subdirectory) and every file was last modified before + ``cutoff_ms`` (epoch milliseconds, as the Files API reports ``last_modified``); ``None`` + otherwise: a file that is newer, undated or not Parquet, a subdirectory, or an empty + directory (a run that has made it and is still uploading).""" + files: List[str] = [] + if not listed: + return None + for entry in listed: + entry = _mapping(entry) + path = str(entry.get("path") or "") + if entry.get("is_directory") or not path.startswith(run_dir + "/"): + return None + if not path.endswith(".parquet"): + return None + try: + modified = float(entry.get("last_modified")) + except (TypeError, ValueError): + return None + if modified >= cutoff_ms: + return None + files.append(path) + return files + + +def _fs(kind: str, path: str) -> str: + return f"/api/2.0/fs/{kind}{quote(path, safe='/')}" + + +def _failure(exc: BaseException) -> str: + return f"{type(exc).__name__}: {scrub(exc)}" + + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _rows_reported(doc: Mapping[str, Any]) -> Optional[Tuple[int, str]]: + """The rows an INSERT says it wrote: ``num_inserted_rows``, else ``num_affected_rows``.""" + schema = _mapping(_mapping(doc.get("manifest")).get("schema")) + names = [str(_mapping(c).get("name")) for c in schema.get("columns") or []] + data = _mapping(doc.get("result")).get("data_array") or [] + if not data or not isinstance(data[0], list): + return None + for name in ("num_inserted_rows", "num_affected_rows"): + if name in names and names.index(name) < len(data[0]): + try: + return int(data[0][names.index(name)]), name + except (TypeError, ValueError): + return None + return None + + +class Landing: + """A client, a runner and a staging volume, for loading one build's file.""" + + def __init__(self, target: LoadTarget, volume: StagingVolume, client: Any, sql: SqlRunner): + self.target = target + self.volume = volume + self._client = client + self._sql = sql + + # ── the three steps ──────────────────────────────────────────────── + + def _upload(self, path: str, prefix: str, remote: str) -> None: + # One PUT to the Files API's own endpoint (the one databricks-sdk's FilesAPI.upload + # calls), not WorkspaceClient.files.upload: from databricks-sdk 0.4x its FilesExt can + # switch to multipart uploads through presigned cloud-storage URLs, a second host the + # data would go to. The cost is the single PUT's 5 GiB limit per file. + do = self._client.api_client.do + try: + do("PUT", _fs("directories", prefix)) + except Exception as exc: # noqa: BLE001 — every transport failure is the upload's + raise DatabricksLoadError( + f"could not create {prefix} in the staging volume: {_failure(exc)}" + ) from exc + try: + with open(path, "rb") as fh: + do( + "PUT", + _fs("files", remote), + query={"overwrite": "true"}, + headers={"Content-Type": "application/octet-stream"}, + data=fh, + ) + except Exception as exc: # noqa: BLE001 + raise DatabricksLoadError(f"could not upload to {remote}: {_failure(exc)}") from exc + + def _count(self, full: str) -> int: + doc = self._sql.execute(f"SELECT COUNT(*) FROM {quote_table(full)}") + data = _mapping(doc.get("result")).get("data_array") or [] + try: + return int(data[0][0]) + except (IndexError, TypeError, ValueError) as exc: + raise DatabricksLoadError(f"counting {full} returned no readable count") from exc + + def _insert(self, remote: str) -> Tuple[int, str]: + full = self.target.full_name + try: + doc = self._sql.execute(insert_overwrite_statement(self.target, remote)) + except StatementFailed as exc: + raise DatabricksLoadError(f"loading {full} failed: {exc}") from exc + reported = _rows_reported(doc) + if reported is not None: + return reported + # A full refresh leaves exactly the file's rows, so the table's count is the load's. + try: + return self._count(full), "count_after_load" + except StatementFailed as exc: + raise DatabricksLoadError( + f"loaded {full}, but counting its rows afterwards failed: {exc}" + ) from exc + + def _delete(self, prefix: str, remote: str) -> List[str]: + """Delete the staged file and its directory; what could not be deleted, named.""" + do = self._client.api_client.do + try: + do("DELETE", _fs("files", remote)) + except Exception as exc: # noqa: BLE001 + if not is_not_found(exc): + return [f"{remote} ({_failure(exc)})"] + try: + do("DELETE", _fs("directories", prefix)) + except Exception as exc: # noqa: BLE001 + if not is_not_found(exc): + return [f"{prefix} ({_failure(exc)})"] + return [] + + def _list_dir(self, path: str) -> List[Any]: + """Every entry of a volume directory, following ``next_page_token``.""" + entries: List[Any] = [] + token: Optional[str] = None + for _ in range(100): + page = _mapping( + self._client.api_client.do( + "GET", _fs("directories", path), query={"page_token": token} if token else None + ) + ) + entries += list(page.get("contents") or []) + token = page.get("next_page_token") or None + if not token: + return entries + raise DatabricksLoadError(f"the listing of {path} never ended") + + def _sweep_stale(self, build_dir: str, current: str, logger: logging.Logger) -> List[str]: + """Delete what earlier runs of this build left in ``build_dir`` (a run killed before its + own delete); the run directories deleted. Only a run directory whose every file is older + than :data:`STALE_STAGING_SECONDS` is deleted: a run of the same build still in progress + keeps its upload. Best effort: a failure is logged, not raised, because this run's own + files are deleted by this run whatever happens here.""" + do = self._client.api_client.do + swept: List[str] = [] + cutoff_ms = (time.time() - STALE_STAGING_SECONDS) * 1000 + try: + runs = self._list_dir(build_dir) + except Exception as exc: # noqa: BLE001 + if not is_not_found(exc): + logger.warning( + "databricks.load could not list %s for earlier runs' staged files: %s", + build_dir, + _failure(exc), + ) + return swept + for run in runs: + run_dir = str(_mapping(run).get("path") or "").rstrip("/") + if not _mapping(run).get("is_directory") or run_dir == current: + continue + name = run_dir[len(build_dir) + 1 :] + if not run_dir.startswith(build_dir + "/") or not _RUN_DIR_RE.fullmatch(name): + continue # only a run directory, directly in this build's directory + try: + listed = self._list_dir(run_dir) + files = _stale_files(listed, run_dir, cutoff_ms) + if files is None: + continue # a file there is recent, or undated: perhaps a run in progress + for path in files: + do("DELETE", _fs("files", path)) + do("DELETE", _fs("directories", run_dir)) + except Exception as exc: # noqa: BLE001 + logger.warning( + "databricks.load could not delete an earlier run's staged files in %s; " + "delete them: %s", + run_dir, + _failure(exc), + ) + continue + swept.append(run_dir) + logger.warning( + "databricks.load deleted staged files an earlier run left behind: %s", run_dir + ) + return swept + + # ── the load ─────────────────────────────────────────────────────── + + def load( + self, + path: str, + *, + product_id: Any, + build_id: Any, + run_id: Any, + expected_rows: int, + logger: logging.Logger, + ) -> Dict[str, Any]: + """Upload ``path``, replace the table's rows with it, delete the upload. Returns the facts. + + The table must exist. The staged files are deleted whether or not the load worked, and + a load that wrote the wrong number of rows, or a file that cannot be deleted, fails. + """ + full = self.target.full_name + try: + live = read_table(self._client, full) + except Exception as exc: # noqa: BLE001 + raise DatabricksLoadError( + f"could not read {full} from Unity Catalog: {_failure(exc)}" + ) from exc + if live is None: + raise DatabricksLoadError( + f"{full} does not exist in Unity Catalog. The build loads the table fluid apply " + "creates and never creates one itself: run fluid apply for this contract first." + ) + prefix = run_prefix(self.volume, product_id, build_id, run_id) + remote = f"{prefix}/{os.path.basename(path)}" + swept = self._sweep_stale(prefix.rsplit("/", 1)[0], prefix, logger) + loaded = 0 + rows_from = "" + failure: Optional[DatabricksLoadError] = None + try: + self._upload(path, prefix, remote) + loaded, rows_from = self._insert(remote) + if loaded != expected_rows: + raise DatabricksLoadError( + f"loaded {loaded} rows into {full}, but the landed file holds {expected_rows}" + ) + except DatabricksLoadError as exc: + failure = exc + except Exception as exc: # noqa: BLE001 — a failure must still reach the delete below + failure = DatabricksLoadError(_failure(exc)) + except BaseException: + # Ctrl-C or SystemExit mid-load (the statement runner has cancelled the INSERT): the + # staged files still go before the interrupt propagates. + left = self._delete(prefix, remote) + if left: + logger.error( + "databricks.load interrupted, and the staged files, which hold the landed " + "data, could not be deleted; delete them: %s", + "; ".join(left), + ) + raise + left = self._delete(prefix, remote) + if left: + # the paths lead the message: a caller that shortens it must not cut them off + logger.error( + "databricks.load could not delete the staged files, which hold the landed data; " + "delete them: %s", + "; ".join(left), + ) + if failure is not None: + if left: + raise DatabricksLoadError( + f"could not delete the staged files, which hold the landed data; delete them: " + f"{'; '.join(left)}. The load had failed: {failure}" + ) from None + raise DatabricksLoadError( + f"{failure} (the staged files in {prefix} were deleted)" + ) from None + if left: + raise DatabricksLoadError( + f"loaded {loaded} rows into {full}, but could not delete the staged files, which " + f"hold the landed data; delete them: {'; '.join(left)}" + ) + logger.info( + "databricks.load table=%s rows=%d rows_from=%s staged_to=%s deleted=true", + full, + loaded, + rows_from, + prefix, + ) + return { + "table": full, + "rows": loaded, + "rows_from": rows_from, + "staged_to": prefix, + "staged_files_deleted": True, + "earlier_runs_staging_deleted": swept, + } + + +def open_landing(target: LoadTarget, *, env: Optional[Mapping[str, str]] = None) -> Landing: + """A :class:`Landing` for ``target``, or a :class:`DatabricksLoadError` naming what is missing. + + Called before the build runs, so a missing volume, warehouse, credential or SDK refuses the + run before anything is read or written. + """ + env = os.environ if env is None else env + volume = staging_volume(env) + warehouse = str(env.get(WAREHOUSE_ENV) or "").strip() + if not warehouse: + raise DatabricksLoadError( + f"{WAREHOUSE_ENV} is not set. The load runs on that SQL warehouse: set it to the id " + "of a warehouse in the workspace DATABRICKS_HOST names." + ) + try: + client = default_client() + except ImportError: + raise DatabricksLoadError( + f"loading into Databricks needs the databricks extra: {DATABRICKS_EXTRA_INSTALL}" + ) from None + except Exception as exc: # noqa: BLE001 — a bad credential must not become a traceback + raise DatabricksLoadError( + f"could not create the Databricks client: {_failure(exc)}" + ) from exc + return Landing(target, volume, client, SqlRunner(client, warehouse)) diff --git a/fluid_build/build_runners/duckdb/runner.py b/fluid_build/build_runners/duckdb/runner.py index 203add1b4..4976b8cb4 100644 --- a/fluid_build/build_runners/duckdb/runner.py +++ b/fluid_build/build_runners/duckdb/runner.py @@ -1619,6 +1619,34 @@ def _execute(ctx: RunContext, runner: DuckdbRunner) -> RunResult: ctx, streams_to_run, started_at, t_start, f"bigquery load: {reason}" ) + # A Databricks binding lands the file locally, uploads it to a Unity Catalog volume and + # loads the table fluid apply created (see ``_databricks_load``). Everything it needs is + # checked before any work runs: it must never land somewhere else and succeed. + dbx_landing = None + if _is_databricks_binding(ctx): + from .._databricks_load import ( + DatabricksLoadError, + databricks_load_target, + open_landing, + ) + from .._databricks_load import ( + unsupported_reason as databricks_unsupported_reason, + ) + + try: + expose = _build_expose(ctx) or {} + dbx_target = databricks_load_target(expose.get("binding") or {}, expose) + reason = databricks_unsupported_reason( + _mode_name(ctx), sink_format, len(streams_to_run) + ) + if reason: + raise DatabricksLoadError(reason) + if dbx_target is None: # unreachable: _is_databricks_binding said it is + raise DatabricksLoadError("the binding does not name a Databricks table") + dbx_landing = open_landing(dbx_target) + except DatabricksLoadError as exc: + return _refused_run(ctx, streams_to_run, started_at, t_start, f"databricks load: {exc}") + stream_results: List[StreamResult] = [] failures = 0 records_total = 0 @@ -1870,6 +1898,34 @@ def _execute(ctx: RunContext, runner: DuckdbRunner) -> RunResult: error=f"bigquery load: {type(exc).__name__}: {exc}", ) + # Databricks: the file is on local disk now; upload it and load the table. After the + # late-arrival split, so the rows loaded are the rows that stayed. A failed or short load + # fails the stream, and so the run and the apply. The staged files are deleted either way. + databricks_load: Dict[str, Any] = {} + if dbx_landing is not None and failures == 0 and stream_results: + from ...iac.providers.databricks_enforce import scrub as _scrub_dbx + + for i, sr in enumerate(stream_results): + if sr.state is not RunState.SUCCEEDED: + continue + staged = _resolve_destination_path(ctx, sr.name, sink_format, out_dir) + try: + databricks_load = dbx_landing.load( + staged, + product_id=ctx.product_id, + build_id=ctx.build_id, + run_id=ctx.run_id, + expected_rows=_count_file_rows(staged, sink_format), + logger=LOG, + ) + except Exception as exc: # noqa: BLE001 + failures += 1 + stream_results[i] = dataclasses.replace( + sr, + state=RunState.FAILED, + error=f"databricks load: {type(exc).__name__}: {_scrub_dbx(exc)}", + ) + # ``onError: fail`` mode — when ANY bad rows landed in the DLQ, # promote the run to FAILED so the run record reflects the issue. # The DLQ records are still on disk (audit trail intact); the @@ -1907,6 +1963,8 @@ def _execute(ctx: RunContext, runner: DuckdbRunner) -> RunResult: facets["pii_findings"] = pii_findings if bigquery_load: facets["bigquery_load"] = bigquery_load + if databricks_load: + facets["databricks_load"] = databricks_load rows_from = "write" if counted_at_write else "source_count" if any(split.get("late") for split in late_arrival_split_results.values()): # The split moved rows out of the file the COPY wrote. @@ -1951,11 +2009,15 @@ def _resolve_destination_path( ``_object_store_uri``. For local paths, behavior is unchanged: relative paths are rooted under ``ctx.workdir`` and parent directories are created. A BigQuery binding is not a file destination: - the file is staged locally for the load job, see ``_bigquery_target``. + the file is staged locally for the load job, see ``_bigquery_target``. A + Databricks binding is not one either: its file is staged locally, uploaded to + a volume and loaded into the table, see ``_databricks_staging_path``. """ bq = _bigquery_target(ctx) if bq is not None: return str(_bigquery_staging_path(ctx, bq, sink_format)) + if _is_databricks_binding(ctx): + return str(_databricks_staging_path(ctx, stream, sink_format)) expose = _build_expose(ctx) if expose is not None: binding = expose.get("binding", {}) or {} @@ -2166,9 +2228,10 @@ def _binding_destination_uri(ctx: RunContext) -> Optional[str]: no secret, so the COPY reached S3 unauthenticated and came back 403. None for a BigQuery binding too: its file is staged locally and loaded by a - job, so the ``gs://`` path it may carry is never written to. + job, so the ``gs://`` path it may carry is never written to; and for a + Databricks binding, whose file is staged locally and uploaded to a volume. """ - if _bigquery_target(ctx) is not None: + if _bigquery_target(ctx) is not None or _is_databricks_binding(ctx): return None expose = _build_expose(ctx) if expose is None: @@ -2221,6 +2284,33 @@ def _bigquery_staging_path(ctx: RunContext, target: Dict[str, Any], sink_format: return path +def _is_databricks_binding(ctx: RunContext) -> bool: + """True when the expose this build writes is bound to Databricks. + + Its ``location.path`` (and ``out/``) are never written: the rows land in the table. + """ + from ...iac.provider_match import is_cloud + + expose = _build_expose(ctx) + return expose is not None and is_cloud(expose.get("binding") or {}, "databricks") + + +def _databricks_staging_path(ctx: RunContext, stream: str, sink_format: str) -> Path: + """Where a Databricks-bound build lands its file before the upload. + + Beside the BigQuery staging, under ``.fluid/staging``, and not ``out/``, which is the + local target's own output. Named after the table, as the catalogue names it. + """ + ext = _FILE_FORMAT_EXT.get((sink_format or "parquet").lower(), sink_format) + expose = _build_expose(ctx) or {} + loc = (expose.get("binding") or {}).get("location") or {} + build = _path_part(getattr(ctx, "build_id", None) or "build") + name = _path_part(loc.get("table") or stream) + path = Path(ctx.workdir) / ".fluid" / "staging" / build / f"{name}.{ext}" + path.parent.mkdir(parents=True, exist_ok=True) + return path + + def _path_part(value: Any) -> str: """One safe path component: contract-supplied ids must not climb out of staging.""" return re.sub(r"[^A-Za-z0-9_-]", "_", str(value)) or "_" diff --git a/fluid_build/cli/_verify_databricks.py b/fluid_build/cli/_verify_databricks.py index 051c555ff..4808f834c 100644 --- a/fluid_build/cli/_verify_databricks.py +++ b/fluid_build/cli/_verify_databricks.py @@ -48,9 +48,20 @@ or the table's owner, and its body, parameters and return type the expected ones, since a content-hashed name is computable from the contract and could have been taken by someone else's function. ``skip`` when the contract restricts nothing. -5. **The jobs.** Each dbt build that writes the expose has exactly one job of +5. **Retention.** An expose with ``lifecycle.expire`` is accepted because its table is rebuilt in + full and the contract's retention covers its Delta history (``databricks_retention.py``). + The table's live ``delta.logRetentionDuration`` and ``delta.deletedFileRetentionDuration`` + (``properties`` and ``delta_runtime_properties_kvpairs`` of the GetTable already read; + unset means Delta's 30 and 7 days) must not make the history longer than the contract's + retention: CRITICAL if they do, with the ``ALTER TABLE`` that sets them. Predictive + optimization, which runs ``VACUUM`` on managed tables, off for the table + (``effective_predictive_optimization_flag``) is a warning: nothing removes the old files + until a ``VACUUM`` runs. ``skip`` when the contract sets no expire. +6. **The jobs.** Each dbt build that writes the expose has exactly one job of its name. None means ``fluid apply`` did not run (or the job was deleted); - several mean an apply without its state created a duplicate. + several mean an apply without its state created a duplicate. A job that is paused while + its table's masks or row filter are not in place fails too, with the apply that binds them + and starts the job as the fix. The client is ``databricks.sdk.WorkspaceClient`` built from ``sdk_config`` (the credential policy the provider and the doctor share), used only through its @@ -63,6 +74,7 @@ from __future__ import annotations +import copy import os from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence from urllib.parse import quote @@ -384,6 +396,118 @@ def _residency_dimension(client: Any, table: Any) -> Dict[str, Any]: } +def _live_properties(live_table: Mapping[str, Any]) -> Dict[str, Any]: + """The table's properties as GetTable serves them: ``properties``, with the Delta runtime + ones (``delta_runtime_properties_kvpairs``, the table's current state) over them.""" + props: Dict[str, Any] = {} + declared = live_table.get("properties") + if isinstance(declared, Mapping): + props.update(declared) + runtime = live_table.get("delta_runtime_properties_kvpairs") + runtime = runtime.get("delta_runtime_properties") if isinstance(runtime, Mapping) else None + if isinstance(runtime, Mapping): + props.update(runtime) + return props + + +def _predictive_optimization(live_table: Mapping[str, Any]) -> Optional[str]: + """``ENABLE`` or ``DISABLE`` for the table, as Unity Catalog reports it (the effective flag, + else the table's own unless it inherits); ``None`` when it does not say.""" + effective = live_table.get("effective_predictive_optimization_flag") + value = effective.get("value") if isinstance(effective, Mapping) else None + if not value: + value = live_table.get("enable_predictive_optimization") + value = str(value or "").upper() + return value if value in {"ENABLE", "DISABLE"} else None + + +def _retention_dimension(table: Any, live_table: Mapping[str, Any]) -> Dict[str, Any]: + """The table's live Delta history retention against the contract's ``lifecycle.retention``. + + ``skip`` when the contract sets no expire: nothing was promised. ``fail`` when the longer + of ``delta.logRetentionDuration`` and ``delta.deletedFileRetentionDuration`` (each at + Delta's default when unset) exceeds the contract's retention, or one is not an interval. + ``warn`` (not a failure) when predictive optimization is off for the table. + """ + from fluid_build.iac.providers.databricks_retention import ( + DELETED_FILE_RETENTION, + LOG_RETENTION, + days_text, + history_seconds, + ) + + promised = getattr(table, "retention_seconds", None) + if promised is None: + return { + "status": "skip", + "message": "the contract sets no lifecycle.expire to check the Delta history against", + "findings": [], + "warnings": [], + } + period = getattr(table, "retention_period", None) + optimization = _predictive_optimization(live_table) + quoted = ".".join(_quote_name(part) for part in table.full_name.split(".")) + schema = ".".join(_quote_name(part) for part in table.full_name.split(".")[:2]) + warnings: List[str] = [] + if optimization == "DISABLE": + warnings.append( + f"predictive optimization is off for {table.full_name}, so nothing runs VACUUM and " + "the old data files stay past the retention. Enable it " + f"(ALTER SCHEMA {schema} ENABLE PREDICTIVE OPTIMIZATION), or schedule " + f"VACUUM {quoted}" + ) + result: Dict[str, Any] = { + "retention": period, + "predictive_optimization": optimization, + "findings": [], + "warnings": warnings, + } + try: + effective = history_seconds(_live_properties(live_table)) + except ValueError as exc: + return { + **result, + "status": "fail", + "message": f"a Delta retention property of {table.full_name} cannot be read: {exc}", + "findings": [{"message": str(exc)}], + "fix": ( + f"Set {LOG_RETENTION} and {DELETED_FILE_RETENTION} on the table to " + "'interval N days'" + ), + } + result["history_seconds"] = effective + if "properties" not in live_table and "delta_runtime_properties_kvpairs" not in live_table: + warnings.append( + f"Unity Catalog returned no table properties for {table.full_name}, so Delta's " + f"defaults were assumed. Check them with SHOW TBLPROPERTIES {quoted}" + ) + if effective > promised: + days = max(promised // 86400, 1) + fix = ( + f"ALTER TABLE {quoted} SET TBLPROPERTIES ('{LOG_RETENTION}' = 'interval {days} " + f"days', '{DELETED_FILE_RETENTION}' = 'interval {days} days'), or raise " + "lifecycle.retention" + ) + message = ( + f"the Delta history of {table.full_name} is {days_text(effective)}, longer than " + f"the contract's retention {period} ({days_text(promised)})" + ) + return { + **result, + "status": "fail", + "message": message, + "findings": [{"message": message}], + "fix": fix, + } + message = ( + f"the Delta history of {table.full_name} is {days_text(effective)}, within the " + f"contract's retention {period} ({days_text(promised)})" + ) + if warnings: + return {**result, "status": "warn", "message": message, "fix": warnings[0]} + return {**result, "status": "pass", "message": message} + + def _tags_dimension(client: Any, table: Any) -> Dict[str, Any]: """The governance tags the module sets on the table and its schema, against the live ones. @@ -609,8 +733,31 @@ def _function_findings( return out -def _jobs_dimension(client: Any, expose_id: str, contract: Mapping[str, Any], jobs: Any) -> Any: - """The dimension for the dbt jobs that write ``expose_id``; ``None`` when there are none.""" +def _paused(job: Mapping[str, Any]) -> bool: + """Whether the job's schedule or continuous block is ``PAUSED``.""" + settings = job.get("settings") + if not isinstance(settings, Mapping): + return False + return any( + isinstance(settings.get(block), Mapping) and settings[block].get("pause_status") == "PAUSED" + for block in ("schedule", "continuous") + ) + + +def _jobs_dimension( + client: Any, + expose_id: str, + contract: Mapping[str, Any], + jobs: Any, + enforcement: Optional[Mapping[str, Any]] = None, + table_name: str = "", +) -> Any: + """The dimension for the dbt jobs that write ``expose_id``; ``None`` when there are none. + + A job that is paused while the table's masks or row filter are not in place is the one + ``fluid apply`` creates paused for that reason (``DatabricksIacPlugin.emit``): it is a finding + whose fix is the apply that binds them and starts it. + """ from fluid_build.util.contract import builds_writing_expose writers = { @@ -633,6 +780,17 @@ def _jobs_dimension(client: Any, expose_id: str, contract: Mapping[str, Any], jo "run fluid apply", } ) + elif len(hits) == 1 and _paused(hits[0]) and (enforcement or {}).get("status") == "fail": + findings.append( + { + "build": job.build_id, + "message": ( + f"job {job.name!r} is paused because {table_name or 'its table'} is not " + "protected (the row_filter_masks dimension fails): run fluid apply, which " + "binds the masks and row filter, then starts the job" + ), + } + ) elif len(hits) > 1: findings.append( { @@ -664,6 +822,7 @@ def _severity( residency: Optional[Mapping[str, Any]] = None, tags: Optional[Mapping[str, Any]] = None, enforcement: Optional[Mapping[str, Any]] = None, + retention: Optional[Mapping[str, Any]] = None, ) -> Dict[str, Any]: """The shared drift grading, raised to CRITICAL by a missing table, reader, job, a workspace outside the declared region, a governance tag that is not there or a row @@ -699,6 +858,9 @@ def _severity( enforcement["message"] + ": " + "; ".join(f["message"] for f in enforcement["findings"]) ) actions.append(enforcement["fix"]) + if (retention or {}).get("status") == "fail": + reasons.append(retention["message"]) + actions.append(retention["fix"]) if not reasons: return severity already = severity["level"] == "CRITICAL" @@ -742,11 +904,15 @@ def verify_databricks_expose( gives one (an external pipeline owns its creation). ``client_factory()`` defaults to ``WorkspaceClient(**sdk_config(os.environ))``. """ + from fluid_build.cli._common import resolve_env_templates_in_contract from fluid_build.iac.base import UnsupportedBindingError from fluid_build.iac.providers.databricks import managed_objects try: - tables, jobs = managed_objects(contract, os.environ) + # What fluid apply provisions, from the contract as apply reads it: with its + # ``{{ env.X }}`` placeholders substituted. + resolved = resolve_env_templates_in_contract(copy.deepcopy(dict(contract or {}))) + tables, jobs = managed_objects(resolved, os.environ) except UnsupportedBindingError as exc: return _error(f"{exc} ({exc.kind})", remediation=tuple(exc.remediation)) except Exception as exc: # noqa: BLE001 — the emitter's refusal is the result @@ -801,7 +967,8 @@ def verify_databricks_expose( residency = _residency_dimension(client, table) tags = _tags_dimension(client, table) enforcement = _enforcement_dimension(table, live_table, client) - job_dim = _jobs_dimension(client, expose_id, contract, jobs) + retention = _retention_dimension(table, live_table) + job_dim = _jobs_dimension(client, expose_id, contract, jobs, enforcement, target) except Exception as exc: # noqa: BLE001 — a check that could not run is an error return _error( f"Databricks check for {target} could not run: {_failure(exc)}", @@ -813,6 +980,7 @@ def verify_databricks_expose( dimensions["residency"] = residency dimensions["governance_tags"] = tags dimensions["row_filter_masks"] = enforcement + dimensions["retention"] = retention if job_dim is not None: dimensions["jobs"] = job_dim @@ -826,6 +994,7 @@ def verify_databricks_expose( "residency", "governance_tags", "row_filter_masks", + "retention", "jobs", ) if name in dimensions @@ -836,7 +1005,13 @@ def verify_databricks_expose( "table_id": target, "target": target, "severity": _severity( - diff, access, job_dim, residency=residency, tags=tags, enforcement=enforcement + diff, + access, + job_dim, + residency=residency, + tags=tags, + enforcement=enforcement, + retention=retention, ), "dimensions": dimensions, "metadata": { diff --git a/fluid_build/cli/verify.py b/fluid_build/cli/verify.py index 288e344e2..f9808c824 100644 --- a/fluid_build/cli/verify.py +++ b/fluid_build/cli/verify.py @@ -1194,6 +1194,7 @@ def _render_masking_dimension(masking: Dict[str, Any], ok: str, heading: str) -> ("residency", "\n 🔍 Residency (workspace region)"), ("governance_tags", "\n 🔍 Governance tags (Unity Catalog)"), ("row_filter_masks", "\n 🔍 Row filter and column masks (Unity Catalog)"), + ("retention", "\n 🔍 Retention (Delta history)"), ("jobs", "\n 🔍 Jobs"), ) @@ -1211,6 +1212,11 @@ def _render_findings_dimensions(dimensions: Dict[str, Any]) -> None: if dimension.get("status") == "skip": cprint(f" ⚪ SKIPPED - {dimension.get('message', '')}", markup=False) continue + if dimension.get("status") == "warn": + cprint(f" ⚠️ WARN - {dimension.get('message', '')}", markup=False) + for warning in dimension.get("warnings") or []: + cprint(f" Fix: {warning}", markup=False) + continue cprint(f" ❌ FAIL - {dimension.get('message', '')}", markup=False) if dimension.get("fix"): cprint(f" Fix: {dimension['fix']}", markup=False) @@ -1221,6 +1227,8 @@ def _render_findings_dimensions(dimensions: Dict[str, Any]) -> None: f" Fix (run by an admin of the catalog): {finding['fix']}", markup=False, ) + for warning in dimension.get("warnings") or []: + cprint(f" ⚠️ {warning}", markup=False) #: What ``fluid apply`` provisions for a GCP-bound expose, as far as stage 9 diff --git a/fluid_build/iac/access.py b/fluid_build/iac/access.py index 7acc74957..cde8f1e01 100644 --- a/fluid_build/iac/access.py +++ b/fluid_build/iac/access.py @@ -52,8 +52,9 @@ from __future__ import annotations +import re from dataclasses import dataclass -from typing import Any, Iterable, Mapping, Optional, Tuple +from typing import Any, FrozenSet, Iterable, Mapping, Optional, Tuple #: Principal type prefixes the ``accessPolicy`` schema documents. USER = "user" @@ -73,6 +74,54 @@ _SERVICE_ACCOUNT_SUFFIX = ".gserviceaccount.com" +# The JSONPath scopes a grant's ``resources`` may carry that an emitter can evaluate: one +# expose by id (``$.exposes[?(@.exposeId=='orders')]``, either quote, spaces allowed), or +# every expose (``$.exposes[*]``, ``$.exposes``). Anything else is kept as written and +# reported as unevaluable, never read as "every expose". +_ONE_EXPOSE_RE = re.compile( + r"""\$\.exposes\s*\[\s*\?\s*\(\s*@\.exposeId\s*==\s*(['"])([^'"]+)\1\s*\)\s*\]""" +) +_EVERY_EXPOSE_RE = re.compile(r"\$\.exposes(?:\s*\[\s*\*\s*\])?") + + +@dataclass(frozen=True) +class GrantScope: + """Which exposes a grant's ``resources`` name. + + A grant without ``resources`` covers every expose (``all_exposes``). ``unevaluable`` + holds each resource that is not one of the shapes above, as written: a grant with any + is one an emitter cannot place, and :meth:`covers` is then false for every expose. + """ + + all_exposes: bool = True + expose_ids: FrozenSet[str] = frozenset() + unevaluable: Tuple[str, ...] = () + + def covers(self, expose_id: Any) -> bool: + if self.unevaluable: + return False + return self.all_exposes or str(expose_id) in self.expose_ids + + +def parse_grant_scope(resources: Iterable[str]) -> GrantScope: + """The :class:`GrantScope` of a grant's ``resources``.""" + resources = tuple(resources) + if not resources: + return GrantScope() + every = False + ids: set[str] = set() + unevaluable: list[str] = [] + for resource in resources: + text = resource.strip() + if _EVERY_EXPOSE_RE.fullmatch(text): + every = True + elif one := _ONE_EXPOSE_RE.fullmatch(text): + ids.add(one.group(2)) + else: + unevaluable.append(resource) + return GrantScope(all_exposes=every, expose_ids=frozenset(ids), unevaluable=tuple(unevaluable)) + + @dataclass(frozen=True) class AccessGrant: """One normalized access grant, provider-agnostic. @@ -89,6 +138,11 @@ class AccessGrant: permissions: Tuple[str, ...] resources: Tuple[str, ...] = () + @property + def scope(self) -> GrantScope: + """The exposes ``resources`` names (every expose when it is empty).""" + return parse_grant_scope(self.resources) + @property def is_service_account(self) -> bool: return self.principal_type == SERVICE_ACCOUNT @@ -149,7 +203,9 @@ def _from_access_policy(contract: Mapping[str, Any]) -> Tuple[AccessGrant, ...]: if not split: continue principal, principal_type = split - resources = tuple(str(r) for r in (entry.get("resources") or ()) if str(r).strip()) + # A blank entry is kept: dropped, it would leave an empty list, which means every + # expose; kept, ``parse_grant_scope`` reports it as unevaluable. + resources = tuple(str(r) for r in (entry.get("resources") or ())) grants.append( AccessGrant( principal=principal, @@ -257,9 +313,11 @@ def role_grants( "AccessGrant", "DOMAIN", "GROUP", + "GrantScope", "SERVICE_ACCOUNT", "USER", "normalize_access_grants", + "parse_grant_scope", "role_grants", "uses_legacy_policy_surface", ] diff --git a/fluid_build/iac/providers/databricks.py b/fluid_build/iac/providers/databricks.py index 73185f8d5..84626e51f 100644 --- a/fluid_build/iac/providers/databricks.py +++ b/fluid_build/iac/providers/databricks.py @@ -31,7 +31,10 @@ set: Delta drops and renames columns only under column mapping. * ``accessPolicy.grants[]`` and ``policy.authz.readers`` / ``writers`` become one ``databricks_grant`` per (table, principal), plus ``USE_SCHEMA`` on the - schema. ``databricks_grant`` is authoritative for its principal on its + schema. A grant's ``resources`` places it on the tables of the exposes it names + (``$.exposes[?(@.exposeId=='')]``, or ``$.exposes[*]`` / ``$.exposes`` for + all; ``iac/access.py::parse_grant_scope``); any other JSONPath, or an exposeId + the contract does not have, is refused, never widened. ``databricks_grant`` is authoritative for its principal on its securable, so a principal named twice is merged into one resource rather than two that undo each other. It also means the product takes over each named principal's privileges on its schema: on a schema that existed before @@ -72,6 +75,10 @@ databricks``) becomes a serverless ``databricks_job`` with a ``dbt_task``, and a local project its ``databricks_workspace_file`` uploads (``databricks_jobs.py``). +* ``lifecycle.expire`` is accepted on a table that is rebuilt in full on every run (a dbt + ``table`` model, or forge-cli's DuckDB acquisition) when ``lifecycle.retention`` covers the + table's Delta history, so ``VACUUM`` removes older copies; anything else is refused + (``databricks_retention.py``). ``fluid verify`` reads the live properties. * A table a dbt build writes, locally or as a job, is rebuilt by dbt's ``table`` materialization with CREATE OR REPLACE, so the module creates it and then leaves its definition to dbt: ``lifecycle.ignore_changes`` on what that @@ -123,6 +130,7 @@ from __future__ import annotations +import json import logging import os import re @@ -130,7 +138,12 @@ from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple from ...providers._sql_safety import validate_ident, validate_sql_type_param_payload -from ...util.contract import is_databricks_job_build +from ...util.binding_paths import ENV_PLACEHOLDER_RE +from ...util.contract import ( + builds_writing_expose, + is_databricks_job_build, + is_duckdb_acquisition_build, +) from ..access import normalize_access_grants from ..base import UnsupportedBindingError from ..governance_labels import governance_labels @@ -155,6 +168,7 @@ dbt_written_exposes, run_as_allowlist, ) +from .databricks_retention import Retention, check_expire _LOG = logging.getLogger(__name__) @@ -259,6 +273,8 @@ class _Table: region: str = "" #: The column masks and row filter Unity Catalog enforces on it, if the expose has any. enforcement: Optional[TableEnforcement] = None + #: The ``lifecycle.expire`` it promises and this module accepted (``databricks_retention.py``). + retention: Optional[Retention] = None @property def full_name(self) -> str: @@ -434,20 +450,141 @@ def _packaging_refusal(where: str) -> UnsupportedBindingError: ) +def _describe_writer(build: Mapping[str, Any]) -> str: + engine = build.get("engine") + parts = [f"engine {engine!r}" if engine else "no engine"] + if build.get("pattern"): + parts.append(f"pattern {build.get('pattern')!r}") + return f"build {build.get('id')!r} ({', '.join(parts)})" + + +def masking_unapplied_reason(exposure: Mapping[str, Any], builds: Any) -> Optional[str]: + """Why no mask would be applied to this expose's rows, or ``None`` when one is. + + ``policy.privacy.masking`` is applied by the DuckDB acquisition runner at landing, before + the rows leave the machine (``build_runners/_masking.py``), so it holds on a Databricks + table only when every build that writes the table is such a build + (``is_duckdb_acquisition_build``). A dbt model, another engine, or no build at all writes + rows nothing masked. + """ + writers = builds_writing_expose(exposure.get("exposeId"), builds) + if not writers: + return "no build writes it, so nothing would apply the mask." + unmasked = [build for build in writers if not is_duckdb_acquisition_build(build)] + if not unmasked: + return None + return ( + f"{', '.join(_describe_writer(build) for build in unmasked)} " + f"{'writes' if len(unmasked) == 1 else 'write'} it and applies no mask." + ) + + +def _refuse_unapplied_masking( + where: str, + exposure: Mapping[str, Any], + builds: Sequence[Mapping[str, Any]], + refusals: List[UnsupportedBindingError], +) -> None: + """``policy.privacy.masking`` on an expose no DuckDB acquisition build alone writes.""" + policy = exposure.get("policy") + privacy = policy.get("privacy") if isinstance(policy, Mapping) else None + if not isinstance(privacy, Mapping) or not _set(privacy.get("masking")): + return + unapplied = masking_unapplied_reason(exposure, builds) + if unapplied is None: + return + refusals.append( + _refusal( + "governance-unmapped", + f"{where} declares policy.privacy.masking, which only forge-cli's DuckDB " + "acquisition runner applies, at landing, before the rows leave the machine. " + f"{unapplied[:1].upper()}{unapplied[1:]} Applied anyway, the table would hold the " + "values the contract promises to mask.", + "Write this expose with a DuckDB acquisition build (pattern: acquisition, " + "engine: duckdb), which masks the columns before it loads the table. Or remove " + "policy.privacy.masking and enforce access with policy.authz.columnRestrictions, " + "which become Unity Catalog column masks.", + ) + ) + + +def _refuse_unlandable( + where: str, + exposure: Mapping[str, Any], + builds: Sequence[Mapping[str, Any]], + refusals: List[UnsupportedBindingError], + *, + validating: bool = False, +) -> None: + """What the DuckDB acquisition runner would refuse at run time for this expose, refused + here: by then ``fluid apply`` has created the table. Its source mode and sink format + (``build_runners/_databricks_load.unsupported_reason``) and, when it masks at landing, + each masking rule (``build_runners/_masking.masking_rules``). A value that is still an + ``{{ env.X }}`` placeholder is left alone only while validating (``fluid validate`` reads + the contract unsubstituted); the emitter reads it substituted, and one still unresolved + there (its variable unset) is refused.""" + from ...build_runners._databricks_load import unsupported_reason + from ...build_runners._masking import MaskingPolicyError, masking_rules + + writers = [ + b + for b in builds_writing_expose(exposure.get("exposeId"), builds) + if is_duckdb_acquisition_build(b) + ] + for build in writers: + properties = build.get("properties") + properties = properties if isinstance(properties, Mapping) else {} + source = properties.get("source") + sink = properties.get("sink") + mode = source.get("mode") if isinstance(source, Mapping) else None + sink_format = (sink.get("format") if isinstance(sink, Mapping) else None) or "parquet" + placeholder = any( + isinstance(v, str) and ENV_PLACEHOLDER_RE.search(v) for v in (mode, sink_format) + ) + if validating and placeholder: + continue + reason = unsupported_reason(str(mode or ""), str(sink_format).lower(), 1) + if reason: + refusals.append( + _refusal( + "landing-unsupported", + f"{where}: build {build.get('id')!r} lands this expose with forge-cli's " + f"DuckDB acquisition runner, and {reason}. The build would be refused at " + "run time, after fluid apply had created the table.", + "Set properties.source.mode: full_refresh and properties.sink.format: " + "parquet on the build (one stream), or write the expose another way.", + ) + ) + if writers and not masking_unapplied_reason(exposure, builds): + try: + masking_rules(exposure) + except MaskingPolicyError as exc: + refusals.append( + _refusal( + "masking-invalid", + f"{where} policy.privacy.masking: {exc} The DuckDB acquisition runner " + "would refuse to land anything.", + "Correct the rule as the message says, or remove it from " + "policy.privacy.masking.", + ) + ) + + def _refuse_governance( where: str, binding: Mapping[str, Any], exposure: Mapping[str, Any], refusals: List[UnsupportedBindingError], + builds: Sequence[Mapping[str, Any]] = (), ) -> None: """Governance the module would apply with exit 0 but never enforce.""" policy = exposure.get("policy") or {} authz = (policy.get("authz") or {}) if isinstance(policy, Mapping) else {} privacy = (policy.get("privacy") or {}) if isinstance(policy, Mapping) else {} + _refuse_unapplied_masking(where, exposure, builds, refusals) unmapped = [ name for name, value in ( - ("policy.privacy.masking", privacy.get("masking")), ("policy.privacy.rowLevelPolicy", privacy.get("rowLevelPolicy")), ("binding.governance", binding.get("governance")), ) @@ -460,8 +597,8 @@ def _refuse_governance( f"{where} declares {', '.join(unmapped)}, which the Databricks emitter does not " "map yet. Applied anyway, the table would be readable without the mask, filter " "or restriction the contract promises.", - "Remove them from this expose until they are mapped (landing-time masking and " - "ABAC policies are not), and enforce access with grants. policy.authz." + "Remove them from this expose until they are mapped (ABAC policies are not), " + "and enforce access with grants. policy.authz." "columnRestrictions and policy.authz.rowFilters are mapped: they become Unity " "Catalog column masks and a row filter.", ) @@ -475,17 +612,6 @@ def _refuse_governance( "Remove binding.encryption; configure customer-managed keys on the workspace.", ) ) - lifecycle = exposure.get("lifecycle") or {} - if isinstance(lifecycle, Mapping) and lifecycle.get("expire"): - refusals.append( - _refusal( - "retention-unmapped", - f"{where} sets lifecycle.expire, and nothing on Databricks would expire the " - "data.", - "Remove lifecycle.expire from the databricks binding, or schedule the delete " - "in a build.", - ) - ) def _refuse_shape( @@ -671,7 +797,9 @@ def _resolve_table( exposure: Mapping[str, Any], refusals: List[UnsupportedBindingError], *, + contract: Mapping[str, Any], validating: bool, + builds: Sequence[Mapping[str, Any]] = (), ) -> Optional[_Table]: """One Databricks expose → :class:`_Table`, or ``None`` with its refusals.""" binding = exposure.get("binding") or {} @@ -685,7 +813,12 @@ def _resolve_table( columns = _columns(where, exposure, refusals, validating=validating) partitions = _partitions(where, loc, columns, refusals, validating=validating) region = _region(where, loc, refusals) - _refuse_governance(where, binding, exposure, refusals) + _refuse_governance(where, binding, exposure, refusals, builds) + _refuse_unlandable(where, exposure, builds, refusals, validating=validating) + retention, retention_refusals = check_expire( + where, exposure, names.get("table") or "", contract, validating=validating + ) + refusals.extend(retention_refusals) enforcement: Optional[TableEnforcement] = None if len(refusals) == before: try: @@ -715,6 +848,7 @@ def _resolve_table( exposure=exposure, region=region, enforcement=enforcement, + retention=retention, ) @@ -860,17 +994,38 @@ def _principal_identities( def _refuse_grant_shapes( contract: Mapping[str, Any], grants: Sequence[Any], refusals: List[UnsupportedBindingError] ) -> None: - """Grants a Unity Catalog grant cannot express: scoped by resources, or conditional.""" + """Grants a Unity Catalog grant cannot express: scoped by resources it cannot evaluate or + by an expose the contract does not have, or conditional.""" + expose_ids = { + str(e.get("exposeId")) for e in contract.get("exposes") or () if isinstance(e, Mapping) + } + fix = ( + "Scope the grant with $.exposes[?(@.exposeId=='')] for one expose (repeat it for " + "several) or $.exposes[*] for every expose, or drop resources (every table of the " + "product)." + ) for grant in grants: - if grant.resources: + scope = grant.scope + if scope.unevaluable: refusals.append( _refusal( "grant-scope-unmapped", - f"accessPolicy grant to {grant_key(grant)!r} is scoped by resources, which " - "the Databricks emitter does not evaluate: the grant would reach no " - "Databricks table.", - "Drop resources (the grant then covers every table of the product), or " - "use the expose's policy.authz.readers / writers.", + f"accessPolicy grant to {grant_key(grant)!r} is scoped by " + f"{', '.join(repr(r) for r in scope.unevaluable)}, which the Databricks " + "emitter cannot evaluate, so it cannot tell which tables the grant is for. " + "It is refused rather than placed on more tables than it names.", + fix, + ) + ) + unknown = sorted(scope.expose_ids - expose_ids) + if unknown: + refusals.append( + _refusal( + "grant-scope-unmapped", + f"accessPolicy grant to {grant_key(grant)!r} is scoped to exposeId " + f"{', '.join(repr(i) for i in unknown)}, which no expose of this contract " + "has.", + "Correct the exposeId in resources, or remove that resource. " + fix, ) ) for entry in (contract.get("accessPolicy") or {}).get("grants") or (): @@ -891,7 +1046,7 @@ def _target_privileges( """``{principal: privileges}`` on one expose's table, from accessPolicy and its authz.""" named: List[Tuple[Tuple[str, ...], set]] = [] for grant in grants: - if grant.resources: + if not grant.scope.covers(target.exposure.get("exposeId")): continue identities = _principal_identities( grant_key(grant), target, refusals, fallback=grant.principal @@ -1032,7 +1187,8 @@ def _compile( if _set(contract.get("packaging")): refusals.append(_packaging_refusal("the contract")) - targets = _resolve_tables(compiled, exposures, validating=validating) + builds = [b for b in contract.get("builds") or [] if isinstance(b, Mapping)] + targets = _resolve_tables(compiled, exposures, contract, validating=validating, builds=builds) warehouse = _warehouse(env, refusals) _refuse_duplicate_tables(compiled.tables, refusals) _refuse_region_conflict(compiled.tables, refusals) @@ -1071,12 +1227,24 @@ def _compile( def _resolve_tables( - compiled: _Compiled, exposures: Sequence[Tuple[int, Mapping[str, Any]]], *, validating: bool + compiled: _Compiled, + exposures: Sequence[Tuple[int, Mapping[str, Any]]], + contract: Mapping[str, Any], + *, + validating: bool, + builds: Sequence[Mapping[str, Any]] = (), ) -> List[_GrantTarget]: """Resolve each Databricks expose into ``compiled.tables``; a grant target for every one.""" targets: List[_GrantTarget] = [] for index, exposure in exposures: - table = _resolve_table(index, exposure, compiled.refusals, validating=validating) + table = _resolve_table( + index, + exposure, + compiled.refusals, + contract=contract, + validating=validating, + builds=builds, + ) if table is not None: compiled.tables.append(table) targets.append( @@ -1182,6 +1350,40 @@ def _emit_table( compiled.imports.append(ImportBlock(f"databricks_sql_table.{table_key}", table.full_name)) +#: The residency precondition's own message (``_region_precondition``), and the diagnostic +#: ``values`` entry that carries the live region (``statement: is "us-west-2"``). In ``tofu +#: -json`` output the text is one JSON string; in human output it is wrapped, hence ``\s+``. +_RESIDENCY_RE = re.compile( + r"The\s+contract\s+declares\s+binding\.location\.region\s+([a-z0-9-]+),\s+but\s+the\s+" + r"workspace\s+DATABRICKS_HOST\s+names\s+is\s+in\s+(\S+?)\.?\s+Residency" +) +_RESIDENCY_START_RE = re.compile( + r"The\s+contract\s+declares\s+binding\.location\.region\s+([a-z0-9-]+)," +) +_LIVE_VALUE_RE = re.compile(r'is\s+\\?"([a-z0-9-]+)\\?"') + + +def _residency_failure(output: str) -> Optional[UnsupportedBindingError]: + """One clean line for a plan that fails the residency precondition: the declared region, + the live one and the fix. ``None`` when the output is not that failure.""" + start = _RESIDENCY_START_RE.search(output) + if start is None: + return None + declared = start.group(1) + full = _RESIDENCY_RE.search(output) + live = full.group(2) if full else "" + if not re.fullmatch(r"[a-z0-9-]+", live): # "(unknown)", a value not yet known, or cut off + value = _LIVE_VALUE_RE.search(output) + live = value.group(1) if value else "" + seen = f"in {live}" if live else "in a region the plan output does not show" + return _refusal( + "residency-region", + f"the contract declares binding.location.region {declared}, but the workspace " + f"DATABRICKS_HOST names is {seen}. Point DATABRICKS_HOST at a workspace in {declared}, " + "or correct binding.location.region.", + ) + + def _region_precondition(compiled: _Compiled, cid: str, region: str) -> Dict[str, Any]: """The plan-time check that the workspace is in the declared ``region``. @@ -1562,7 +1764,11 @@ def emit( if compiled.refusals: raise compiled.refusals[0] _enforce_sovereignty(contract, compiled.tables) - for job, _tables in _jobs_to_pause(compiled, contract): + # Decided once, here: ``plan_notices`` of the same run says what this paused, so a + # transient read error cannot pause a job without the notice (or the reverse). + to_pause = _jobs_to_pause(compiled, contract) + self._pause_decision = (_contract_key(contract), [(j.name, t) for j, t in to_pause]) + for job, _tables in to_pause: _pause(compiled, job) return compiled.resources @@ -1723,9 +1929,15 @@ def plan_notices(self, contract: Mapping[str, Any]) -> List[str]: it creates paused until the tables they write are protected.""" compiled = _compile(contract, os.environ) notices = _use_catalog_notices(compiled) - for job, tables in _jobs_to_pause(compiled, contract): + decided = getattr(self, "_pause_decision", None) + self._pause_decision = None # used once: the next run decides again + if decided is not None and decided[0] is not None and decided[0] == _contract_key(contract): + paused = list(decided[1]) + else: + paused = [(j.name, t) for j, t in _jobs_to_pause(compiled, contract)] + for name, tables in paused: notices.append( - f"job {job.name!r} is created paused (pause_status PAUSED): it writes " + f"job {name!r} is created paused (pause_status PAUSED): it writes " f"{', '.join(sorted(tables))}, whose masks and row filter are not bound yet, and " "a run before that would fill an unprotected table. The next fluid apply, once " "they are bound, starts its schedule; an explicit run-now works meanwhile." @@ -1752,6 +1964,9 @@ def explain_plan_failure( "DATABRICKS_HOST with DATABRICKS_CLIENT_ID and DATABRICKS_CLIENT_SECRET (a " "service principal's OAuth secret), then re-run 'fluid apply'.", ) + residency = _residency_failure(output or "") + if residency is not None: + return residency if _UNREACHABLE_RE.search(output or ""): return _refusal( "workspace-unreachable", @@ -1842,6 +2057,16 @@ def _column_details( return out +def _contract_key(contract: Mapping[str, Any]) -> Optional[str]: + """A stable text of the contract, to tie a decision to the contract it was made for; + ``None`` for one JSON cannot write (mixed-type or date keys, a cycle), and then no + decision is ever reused for it: :meth:`plan_notices` works it out again.""" + try: + return json.dumps(contract, sort_keys=True, default=str) + except (TypeError, ValueError): + return None + + def _jobs_to_pause( compiled: _Compiled, contract: Mapping[str, Any] ) -> List[Tuple[DbtJob, Dict[str, str]]]: @@ -2031,6 +2256,10 @@ class ManagedTable: #: The whole expectation, with each function's definition: what ``fluid verify`` checks #: a bound function's owner, body, parameters and return type against. enforcement: Optional[TableEnforcement] = None + #: The accepted ``lifecycle.expire``: its retention as written and in seconds, which the + #: table's Delta history must not exceed. ``None`` when the expose sets no expire. + retention_period: Optional[str] = None + retention_seconds: Optional[int] = None def managed_objects( @@ -2076,6 +2305,8 @@ def managed_objects( for mask in (table.enforcement.masks if table.enforcement else ()) ), enforcement=table.enforcement, + retention_period=table.retention.period if table.retention else None, + retention_seconds=table.retention.seconds if table.retention else None, ) for table in compiled.tables ] diff --git a/fluid_build/iac/providers/databricks_enforce.py b/fluid_build/iac/providers/databricks_enforce.py index 83fbc8f23..23dbc72d4 100644 --- a/fluid_build/iac/providers/databricks_enforce.py +++ b/fluid_build/iac/providers/databricks_enforce.py @@ -26,7 +26,9 @@ * :func:`post_apply` binds whatever is still missing (the tables the apply just created, empty at that moment), and drops what the contract no longer asks for: a mask or filter whose function carries this module's name prefix. A filter an admin set by hand is left alone. - Dropping widens access, so it waits for the apply to succeed and is printed as such. + Dropping widens access, so it waits for the apply to succeed and is printed as such. After an + apply that succeeded it also drops this module's superseded functions that no table in the + schema uses and this run's identity owns (:func:`drop_orphan_functions`). Both read the live bindings first (``GET /api/2.1/unity-catalog/tables/{name}``: ``row_filter.function_name`` / ``input_column_names`` and ``columns[].mask.function_name``, @@ -47,6 +49,7 @@ from __future__ import annotations import os +import re import time from dataclasses import dataclass, field from typing import Any, Callable, Collection, Dict, List, Mapping, Optional, Sequence, Tuple @@ -144,7 +147,7 @@ def default_client() -> Any: return WorkspaceClient(**config) -def _scrub(text: Any) -> str: +def scrub(text: Any) -> str: """One short line with every credential value in the environment removed.""" from ...observability.secret_redactor import redact_secret_text from .databricks_auth import credential_values @@ -155,7 +158,7 @@ def _scrub(text: Any) -> str: return line if len(line) <= _MAX_MESSAGE else line[: _MAX_MESSAGE - 3] + "..." -def _is_not_found(exc: BaseException) -> bool: +def is_not_found(exc: BaseException) -> bool: for attr in ("status_code", "status", "http_status"): if getattr(exc, attr, None) == 404: return True @@ -177,7 +180,7 @@ def read_table(client: Any, full_name: str) -> Optional[Dict[str, Any]]: "GET", f"/api/2.1/unity-catalog/tables/{quote(full_name, safe='.')}" ) except Exception as exc: # noqa: BLE001 — a 404 is an answer, anything else is raised on - if _is_not_found(exc): + if is_not_found(exc): return None raise return out if isinstance(out, dict) else {} @@ -190,7 +193,7 @@ def read_function(client: Any, full_name: str) -> Optional[Dict[str, Any]]: "GET", f"/api/2.1/unity-catalog/functions/{quote(full_name, safe='.')}" ) except Exception as exc: # noqa: BLE001 — a 404 is an answer, anything else is raised on - if _is_not_found(exc): + if is_not_found(exc): return None raise return out if isinstance(out, dict) else {} @@ -218,6 +221,12 @@ def _type(text: Any) -> str: return "".join(str(text or "").split()).lower() +def owner_matches(owner: Any, owners: Collection[str]) -> bool: + """Whether a function's ``owner`` is one of ``owners`` (case-insensitive): the one comparison + the read-back of a function and the orphan drop share.""" + return str(owner or "").lower() in {o.lower() for o in owners} + + def function_problem( info: Optional[Mapping[str, Any]], expected: Function, owners: Collection[str] ) -> Optional[str]: @@ -233,7 +242,7 @@ def function_problem( return f"{expected.full_name} does not exist" problems: List[str] = [] owner = str(info.get("owner") or "") - if owner.lower() not in {o.lower() for o in owners}: + if not owner_matches(owner, owners): problems.append(f"its owner is {owner or 'unknown'}, not {' or '.join(sorted(owners))}") body_language = str(info.get("routine_body") or "SQL").upper() if body_language != "SQL": @@ -281,7 +290,12 @@ def __init__( def run(self, statement: str) -> None: """Run one statement to ``SUCCEEDED``; :class:`StatementFailed` otherwise.""" + self.execute(statement) + + def execute(self, statement: str) -> Mapping[str, Any]: + """:meth:`run`, and the finished statement's document (its ``result`` and ``manifest``).""" do = self._client.api_client.do + doc: Any = None try: doc = do( "POST", @@ -303,13 +317,20 @@ def run(self, statement: str) -> None: except StatementFailed: raise except Exception as exc: # noqa: BLE001 — every transport failure is the statement's - raise StatementFailed(f"{type(exc).__name__}: {_scrub(exc)}") from exc + raise StatementFailed(f"{type(exc).__name__}: {scrub(exc)}") from exc + except BaseException: + # Ctrl-C or SystemExit while the statement runs: it would go on running on the + # warehouse after this process has gone. + if isinstance(doc, Mapping) and doc.get("statement_id"): + self._cancel(doc) + raise state = self._state(doc) if state != "SUCCEEDED": error = (doc.get("status") or {}).get("error") or {} raise StatementFailed( - f"{state}: {_scrub(error.get('message') or 'no message from the server')}" + f"{state}: {scrub(error.get('message') or 'no message from the server')}" ) + return doc @staticmethod def _state(doc: Any) -> str: @@ -504,7 +525,7 @@ def run(self, step: Step) -> None: raise except Exception as exc: # noqa: BLE001 raise FunctionMismatch( - f"could not read {step.function.full_name}: {type(exc).__name__}: {_scrub(exc)}" + f"could not read {step.function.full_name}: {type(exc).__name__}: {scrub(exc)}" ) from exc if problem is not None: raise FunctionMismatch( @@ -559,7 +580,7 @@ def _client_for( if required: raise EnforcementError( f"could not create the Databricks client for the masks and row filters: " - f"{type(exc).__name__}: {_scrub(exc)}" + f"{type(exc).__name__}: {scrub(exc)}" ) from exc say( " note: masks and row filters an earlier apply bound were not checked for removal " @@ -574,7 +595,7 @@ def _read(client: Any, target: Target) -> Optional[Dict[str, Any]]: except Exception as exc: # noqa: BLE001 raise EnforcementError( f"could not read {target.full_name} from Unity Catalog to see which masks and row " - f"filters it has: {type(exc).__name__}: {_scrub(exc)}", + f"filters it has: {type(exc).__name__}: {scrub(exc)}", ("Check DATABRICKS_HOST and the credential, then re-run fluid apply.",), ) from exc @@ -741,6 +762,136 @@ def post_apply( "first (replacing a deny-all filter) and grants again afterwards.", ), ) + if not after_failed_apply: + drop_orphan_functions(runner, targets, say) + + +def _list_named(client: Any, path: str, query: Mapping[str, str], key: str) -> List[Dict[str, Any]]: + """Every entry of a paged Unity Catalog list (``key`` holds the page's rows).""" + out: List[Dict[str, Any]] = [] + token: Optional[str] = None + for _ in range(100): + page = client.api_client.do( + "GET", path, query={**query, **({"page_token": token} if token else {})} + ) + out += [r for r in (page.get(key) or []) if isinstance(r, Mapping)] + token = page.get("next_page_token") or None + if not token: + return out + raise StatementFailed("the list never ended") + + +def _bare(name: Any) -> str: + return str(name or "").replace("`", "").lower() + + +def _orphans( + names: Sequence[str], table: str, columns: Collection[str], keep: Collection[str] +) -> List[str]: + """The functions among ``names`` (bare, lower case) that carry this table's fluid_ name + pattern and are not in ``keep``: ``fluid_rf_
_<8 hex>``, the deny-all + ``fluid_rf_
_deny_all_<8 hex>`` and ``fluid_mask_
__<8 hex>`` for a column + the table has.""" + t = re.escape(table.lower()) + shapes = [ + f"{ROW_FILTER_PREFIX}{t}_[0-9a-f]{{8}}", + f"{ROW_FILTER_PREFIX}{t}_deny_all_[0-9a-f]{{8}}", + ] + shapes += [f"{MASK_PREFIX}{t}_{re.escape(c.lower())}_[0-9a-f]{{8}}" for c in columns] + rx = re.compile("|".join(f"(?:{x})" for x in shapes)) + return [n for n in names if n.startswith("fluid_") and rx.fullmatch(n) and n not in keep] + + +def drop_orphan_functions(session: Session, targets: Sequence[Target], say: Say) -> None: + """Drop this module's functions that a changed contract left behind, after a successful apply. + + A function is dropped only when its name is this table's fluid_ pattern + (:func:`_orphans`), it is not one the contract expects now, and no table in its schema has it + as a row filter or column mask (every table of the schema is read first). If the schema + cannot be read completely, nothing is dropped in it. Each drop is printed. + """ + schemas: Dict[Tuple[str, str], List[Target]] = {} + for target in targets: + parts = target.full_name.split(".") + if len(parts) == 3: + schemas.setdefault((parts[0], parts[1]), []).append(target) + expected = set() + for target in targets: + e = target.enforcement + if e is None: + continue + expected |= {_bare(m.function.full_name) for m in e.masks} + if e.row_filter is not None: + expected.add(_bare(e.row_filter.function.full_name)) + if e.deny_all is not None: + expected.add(_bare(e.deny_all.function.full_name)) + client = session.client + for (catalog, schema), members in sorted(schemas.items()): + where = f"{catalog}.{schema}" + try: + query = {"catalog_name": catalog, "schema_name": schema} + bound: set = set() + columns: Dict[str, List[str]] = {} + # include_browse: a table the run may only browse is listed too, and reading it + # fails, so nothing is dropped rather than a binding being missed + tables_query = {**query, "include_browse": "true"} + for row in _list_named(client, "/api/2.1/unity-catalog/tables", tables_query, "tables"): + full = str(row.get("full_name") or f"{where}.{row.get('name')}") + live = read_table(client, full) + if live is None: + continue + bound.add(_bare((live.get("row_filter") or {}).get("function_name"))) + live_cols = _live_columns(live) + bound |= { + _bare((c.get("mask") or {}).get("function_name")) for c in live_cols.values() + } + columns[full.rsplit(".", 1)[-1].lower()] = [ + str(c["name"]) for c in live_cols.values() + ] + listed = _list_named(client, "/api/2.1/unity-catalog/functions", query, "functions") + except Exception as exc: # noqa: BLE001 — unknown binding: drop nothing + say( + f" note: orphan functions in {where} not dropped: its tables or functions could " + f"not be read ({type(exc).__name__}: {scrub(exc)})." + ) + continue + names = [ + str(r.get("name") or str(r.get("full_name") or "").rsplit(".", 1)[-1]).lower() + for r in listed + ] + mine = f"{where}.".lower() + keep = {n[len(mine) :] for n in expected | bound if n.startswith(mine)} + settled: List[str] = [] + for target in members: + table = target.full_name.split(".")[2] + for name in _orphans(names, table, columns.get(table.lower(), ()), keep): + if name not in settled and _drop_if_owned(session, f"{where}.{name}", say): + settled.append(name) + + +def _drop_if_owned(session: Session, full: str, say: Say) -> bool: + """Drop one orphan function when the deploy identity owns it, printing what happened. + + ``True`` when it is settled: dropped, or left (and named) because another identity owns it + or the owner cannot be read, so it is not tried again for another table. ``False`` when the + ``DROP`` itself failed. + """ + try: + owner = str((read_function(session.client, full) or {}).get("owner") or "") + me = session.identity + except Exception: # noqa: BLE001 — unknown owner: leave it + owner, me = "", "" + if not me or not owner_matches(owner, {me}): + why = f"owned by {owner}, not by the deploy identity" if owner else "owner unknown" + say(f" left function {full}: {why}") + return True + try: + session.runner.run(f"DROP FUNCTION IF EXISTS {quote_table(full)}") + except StatementFailed as exc: + say(f" note: superseded function {full} was not dropped: {exc}") + return False + say(f" dropped superseded function {full}") + return True @dataclass @@ -761,6 +912,13 @@ class FailSafe: checked: bool = False #: Whether the table's grants could be listed (else only the restricted identities were revoked). listed: bool = True + #: Whether any effective-permission entry said where a privilege comes from + #: (``inherited_from_*``); if none did, "nothing inherited is open" cannot be told from + #: "the listing does not say". + inheritance_visible: bool = False + #: The table's owner could not be read, so the lock could not skip it (only this run's own + #: identity was skipped). + owner_unknown: bool = False _READ = frozenset({"SELECT", "ALL_PRIVILEGES"}) @@ -913,8 +1071,16 @@ def _lock( result: FailSafe, say: Say, ) -> None: - """Revoke SELECT (and ALL PRIVILEGES) on the table from every grantee but its owner.""" + """Revoke SELECT (and ALL PRIVILEGES) on the table from every grantee but its owner and the + identity this run is: revoking from the deploy identity would lock the run out of the table + it has to repair.""" owner = str((live or {}).get("owner") or "") + result.owner_unknown = not owner + try: + me = session.identity + except Exception: # noqa: BLE001 — unreadable: nothing to skip but the owner + me = "" + skip = {p.lower() for p in (owner, me) if p} grantees: Dict[str, set] = {} try: for entry in _list_pages( @@ -931,7 +1097,7 @@ def _lock( ) grantees = {p: {"SELECT"} for p in enforcement.restricted} for principal, held in sorted(grantees.items()): - if owner and principal.lower() == owner.lower(): + if principal.lower() in skip: continue statements = [] if "SELECT" in held: @@ -947,6 +1113,16 @@ def _lock( else: result.revoked.append(statement) say(f" revoked: {statement}") + restricted = {p.lower() for p in enforcement.restricted} + for held_by in dict.fromkeys(h for h in (owner, me) if h): # once when the run owns it + if held_by.lower() in restricted: + what = "owns" if held_by == owner else "is the identity of this run, and has" + result.still_open.append( + f"{held_by} {what} {table} and the contract restricts it; an owner reads its " + "table whatever is revoked, so this lock cannot close that path. Run: ALTER TABLE " + f"{'.'.join(quote_ident(p) for p in table.split('.'))} OWNER TO " + "``" + ) _open_paths(session, table, owner, result) @@ -961,6 +1137,12 @@ def _open_paths(session: Session, table: str, owner: str, result: FailSafe) -> N result.checked = False return result.checked = True + result.inheritance_visible = any( + str(k).startswith("inherited_from") + for entry in entries + for detail in _privileges(entry).values() + for k in detail + ) for entry in entries: principal = str(entry.get("principal") or "") if not principal or (owner and principal.lower() == owner.lower()): @@ -992,10 +1174,15 @@ def _describe(target: Target, fs: FailSafe) -> str: done = f"The deny-all row filter could not be bound ({fs.deny_all_problem}). " + ( "SELECT (and ALL PRIVILEGES where held) was revoked on the table from " + ", ".join(sorted({s.rsplit(" FROM ", 1)[1] for s in fs.revoked})) - + " (every grantee but its owner). " + + " (every grantee but its owner and this run's own identity). " if fs.revoked else "Nothing was revoked. " ) + if fs.owner_unknown: + done += ( + "The table's owner could not be read, so it could not be skipped: if the owner held " + "SELECT through a grant, it was revoked too. " + ) if not fs.listed: done += "Its grants could not be listed, so only the restricted identities were revoked. " if fs.unrevoked: @@ -1004,8 +1191,13 @@ def _describe(target: Target, fs: FailSafe) -> str: ) if fs.still_open: done += "Still open: " + " | ".join(fs.still_open) + ". " - elif fs.checked: + elif fs.checked and fs.inheritance_visible: done += "No grant inherited from the schema or the catalog was found open. " + elif fs.checked: + done += ( + "The permission listing did not say where any privilege comes from, so it could not " + "determine whether a schema or catalog grant is open: check them. " + ) else: done += "Grants inherited from the schema or the catalog could not be checked: check them. " return done.rstrip() diff --git a/fluid_build/iac/providers/databricks_retention.py b/fluid_build/iac/providers/databricks_retention.py new file mode 100644 index 000000000..86205c726 --- /dev/null +++ b/fluid_build/iac/providers/databricks_retention.py @@ -0,0 +1,887 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``lifecycle.expire`` on a Databricks table: limit the Delta history. + +Nothing on Databricks deletes rows by age. What can make a retention promise +true is the table's shape: a table rebuilt **in full** on every run holds no row +copy older than the last run, except in its Delta history (the old data files +Delta keeps for time travel). ``VACUUM`` removes those files once they are older +than the table's retention, and predictive optimization runs ``VACUUM`` on Unity +Catalog managed tables. So ``expire: true`` is accepted when + +1. every build that writes the expose rewrites the whole table: + + * a dbt build whose model for the table is materialized as ``table`` + (``CREATE OR REPLACE`` each run). ``incremental`` and ``snapshot`` keep old + rows, a ``view`` stores none, and anything dbt cannot be shown to have + resolved to ``table`` is refused, not guessed; + * forge-cli's own DuckDB acquisition. The landing it does into Databricks is + an ``INSERT OVERWRITE`` of the whole table, and this module relies on that: + an append there would keep every run's rows and make this acceptance false; + +2. the contract's ``lifecycle.retention`` is at least the table's history + retention, the larger of ``delta.logRetentionDuration`` (default 30 days) and + ``delta.deletedFileRetentionDuration`` (default 7 days) + (https://docs.databricks.com/aws/en/delta/history, + https://docs.delta.io/latest/delta-batch.html#data-retention). A retention + under the 30-day default is refused for now: this module does not set the + table's properties, since dbt owns a dbt-written table's definition. + +``fluid verify`` is the gate on the live table (``cli/_verify_databricks.py``). + +dbt's configuration is resolved statically, as dbt resolves it +(https://docs.getdbt.com/reference/model-configs, +https://docs.getdbt.com/reference/dbt-jinja-functions/config): the ``config()`` +call in the model's SQL beats the model's ``config:`` in a properties YAML, which +beats ``dbt_project.yml`` (``models:``, the most specific path winning); the +default materialization is ``view``. The model is the one whose relation name +(its ``alias``, else its file name) is the expose's table. A value that is not a +plain literal (``var()``, a conditional ``config()``) is *undeterminable* and +refused, with the fix. +""" + +from __future__ import annotations + +import ast +import fnmatch +import os +import re +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath +from typing import Any, Dict, Iterator, List, Mapping, Optional, Sequence, Tuple + +from ...util.binding_paths import ENV_PLACEHOLDER_RE +from ...util.contract import builds_writing_expose, is_duckdb_acquisition_build +from ..base import UnsupportedBindingError +from ..source import current_contract_source + +DAY = 86400 + +#: Delta's table properties for how long history is kept, and their defaults +#: (Databricks "Work with Delta Lake table history"). +LOG_RETENTION = "delta.logRetentionDuration" +DELETED_FILE_RETENTION = "delta.deletedFileRetentionDuration" +DEFAULT_LOG_SECONDS = 30 * DAY +DEFAULT_DELETED_FILE_SECONDS = 7 * DAY + +#: The most a project read here may hold: the same bounds the job upload keeps. +MAX_FILE_BYTES = 2 * 1024 * 1024 +MAX_FILES = 1000 + + +def _refusal(kind: str, message: str, *remediation: str) -> UnsupportedBindingError: + return UnsupportedBindingError(f"databricks-retention-{kind}", message, tuple(remediation)) + + +# ── Durations ─────────────────────────────────────────────────────────────── + +#: The schema's ``isoDuration`` pattern (``fluid-schema-0.7.7.json``), as groups. The ``T`` +#: needs a time part after it (``PT``, ``P1DT`` are not durations), digits are bounded, +#: and a fraction, a sign or a lower-case letter is not accepted. +_ISO_RE = re.compile( + r"^P(?!$)(?:(\d{1,12})Y)?(?:(\d{1,12})M)?(?:(\d{1,12})W)?(?:(\d{1,12})D)?" + r"(?:T(?=\d)(?:(\d{1,12})H)?(?:(\d{1,12})M)?(?:(\d{1,12})S)?)?$" +) + +#: A year and a month are not fixed lengths. The *shortest* is taken, so a contract's +#: retention is never read as longer than it is, and the check that it covers the +#: history never passes one that does not: P1Y = 365 days (not 366), P1M = 28 days (not 30). +_YEAR_DAYS = 365 +_MONTH_DAYS = 28 + + +def iso_duration_seconds(text: Any) -> int: + """An ISO 8601 duration in seconds, years and months at their shortest. + + Raises ``ValueError`` for anything the schema's ``isoDuration`` does not allow. + """ + if not isinstance(text, str) or not (match := _ISO_RE.match(text.strip())): + raise ValueError(f"{text!r} is not an ISO 8601 duration") + years, months, weeks, days, hours, minutes, seconds = (int(g or 0) for g in match.groups()) + return ( + (years * _YEAR_DAYS + months * _MONTH_DAYS + weeks * 7 + days) * DAY + + hours * 3600 + + minutes * 60 + + seconds + ) + + +_INTERVAL_PART = re.compile( + r"(\d{1,12})\s*(week|day|hour|minute|second|millisecond|microsecond)s?\b", re.IGNORECASE +) +_UNIT_SECONDS = { + "week": 7 * DAY, + "day": DAY, + "hour": 3600, + "minute": 60, + "second": 1, + "millisecond": 0.001, + "microsecond": 0.000001, +} + + +def interval_seconds(text: Any) -> int: + """A Delta interval property (``interval 30 days``, ``interval 1 week 12 hours``) in + seconds, rounded up. Delta takes no month or year here; ``ValueError`` for anything + else, so an unreadable property is never taken as a short one.""" + if not isinstance(text, str): + raise ValueError(f"{text!r} is not an interval") + body = re.sub(r"^\s*interval\b", "", text.strip(), flags=re.IGNORECASE).strip() + parts = _INTERVAL_PART.findall(body) + if not parts or _INTERVAL_PART.sub("", body).strip(): + raise ValueError(f"{text!r} is not a Delta interval such as 'interval 30 days'") + total = sum(int(n) * _UNIT_SECONDS[unit.lower()] for n, unit in parts) + return int(-(-total // 1)) + + +def history_seconds(properties: Mapping[str, Any]) -> int: + """The Delta history retention of a table with these properties: the larger of the log + retention and the deleted-file retention, each at its default when unset. + + Property names are matched case-insensitively, as Delta does. ``ValueError`` when one + is set to something that is not an interval. + """ + lowered = {str(k).lower(): v for k, v in properties.items()} + log = lowered.get(LOG_RETENTION.lower()) + deleted = lowered.get(DELETED_FILE_RETENTION.lower()) + return max( + DEFAULT_LOG_SECONDS if log is None else interval_seconds(log), + DEFAULT_DELETED_FILE_SECONDS if deleted is None else interval_seconds(deleted), + ) + + +def days_text(seconds: int) -> str: + """``30 days``, ``36 hours``: how a message names a span.""" + if seconds % DAY == 0: + days = seconds // DAY + return f"{days} day{'s' if days != 1 else ''}" + if seconds % 3600 == 0: + return f"{seconds // 3600} hours" + return f"{seconds} seconds" + + +# ── dbt: which materialization does the model get? ────────────────────────── + + +class DbtUnresolved(Exception): + """The model's materialization cannot be worked out; the message says why.""" + + +class _Unknown: + """A config value that is not a plain literal (``var()``, a name, a ``**``).""" + + def __repr__(self) -> str: # pragma: no cover - debugging only + return "" + + +UNKNOWN = _Unknown() + + +@dataclass(frozen=True) +class DbtModel: + """What dbt would do for the model that writes a table.""" + + name: str + materialized: str + #: The Delta ``tblproperties`` it sets, as written (strings). + tblproperties: Mapping[str, Any] = field(default_factory=dict) + #: dbt's fully qualified name (project, folders below the model path, name), and the + #: file's path below the project, which dbt's selectors match. + fqn: Tuple[str, ...] = () + path: str = "" + + +def _jinja(value: Any) -> bool: + return isinstance(value, str) and ("{{" in value or "{%" in value) + + +def _yaml(path: Path) -> Any: + import yaml + + try: + if path.stat().st_size > MAX_FILE_BYTES: + raise DbtUnresolved(f"{path.name} is larger than {MAX_FILE_BYTES} bytes") + return yaml.safe_load(path.read_text(encoding="utf-8")) + except DbtUnresolved: + raise + except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: + raise DbtUnresolved(f"{path.name} could not be read ({type(exc).__name__})") from None + + +def _literal(node: ast.AST) -> Any: + """A Python or Jinja literal (``true`` and ``none`` too); ``UNKNOWN`` otherwise.""" + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Name) and node.id.lower() in ("true", "false", "none"): + return {"true": True, "false": False, "none": None}[node.id.lower()] + if isinstance(node, (ast.List, ast.Tuple)): + items = [_literal(item) for item in node.elts] + return UNKNOWN if any(item is UNKNOWN for item in items) else items + if isinstance(node, ast.Dict) and all(key is not None for key in node.keys): + out = {} + for key, value in zip(node.keys, node.values, strict=True): + k, v = _literal(key), _literal(value) # type: ignore[arg-type] + if k is UNKNOWN or not isinstance(k, str): + return UNKNOWN + out[k] = v + return out + return UNKNOWN + + +_RAW_OPEN_RE = re.compile(r"\{%-?\s*raw\s*-?%\}") +_RAW_CLOSE_RE = re.compile(r"\{%-?\s*endraw\s*-?%\}") + + +# The scans below match what the lazy regexes ``\{#.*?#\}``, ``{% raw %}.*?{% endraw %}`` and +# ``\{\{(.*?)\}\}|\{%(.*?)%\}`` would, in linear time: an opener with no closer after it means no +# later opener of that kind has one either, so the scan stops looking for that kind instead of +# re-reading the rest of the file from every opener (quadratic on a file of unclosed ``{{``). +def _without_comments_and_raw(text: str) -> str: + out: List[str] = [] + pos = 0 + while (start := text.find("{#", pos)) >= 0 and (end := text.find("#}", start + 2)) >= 0: + out.append(text[pos:start]) + pos = end + 2 + text = "".join(out) + text[pos:] + out, pos = [], 0 + while (opened := _RAW_OPEN_RE.search(text, pos)) and ( + closed := _RAW_CLOSE_RE.search(text, opened.end()) + ): + out.append(text[pos : opened.start()]) + pos = closed.end() + return "".join(out) + text[pos:] + + +def _tags(text: str) -> Iterator[Tuple[Optional[str], Optional[str]]]: + """``(expression, None)`` for each ``{{ ... }}`` and ``(None, statement)`` for each + ``{% ... %}``, in order.""" + closer = {"{{": "}}", "{%": "%}"} + nxt: Dict[str, int] = {} + pos = 0 + while True: + for opener in list(closer): + if nxt.get(opener, -1) < pos: + nxt[opener] = text.find(opener, pos) + if nxt[opener] < 0: + del closer[opener] + if not closer: + return + start, opener = min((nxt[o], o) for o in closer) + end = text.find(closer[opener], start + 2) + if end < 0: + del closer[opener] # no closer after this opener, so none after a later one + continue + inner = text[start + 2 : end] + yield (inner, None) if opener == "{{" else (None, inner) + pos = end + 2 + + +_BLOCKS = frozenset({"if", "for", "macro", "call", "filter", "with", "block"}) +_ENDS = frozenset({"endif", "endfor", "endmacro", "endcall", "endfilter", "endwith", "endblock"}) + + +def sql_config(text: str) -> Dict[str, Any]: + """The configs a model's SQL sets with ``{{ config(...) }}``, later calls over earlier. + + Only a call outside every ``{% if %}`` / ``{% for %}`` / ``{% macro %}`` block, whose + keyword values are literals, is read. Raises :class:`DbtUnresolved` for a ``config()`` + inside a block, or one with a ``**`` expansion or positional argument. A keyword whose + value is not a literal is :data:`UNKNOWN`. + """ + text = _without_comments_and_raw(text) + found: Dict[str, Any] = {} + depth = 0 + for expression, statement in _tags(text): + if statement is not None: + words = statement.strip(" \t\r\n-+").split(None, 1) + word = words[0] if words else "" + if word in _BLOCKS or (word == "set" and "=" not in statement): + depth += 1 + elif word in _ENDS or word == "endset": + depth = max(0, depth - 1) + continue + inner = expression.strip(" \t\r\n-+") + if not re.match(r"config\s*\(", inner): + continue + if depth: + raise DbtUnresolved("its config() is inside a Jinja block, so it may not apply") + try: + call = ast.parse(inner, mode="eval").body + except SyntaxError: + raise DbtUnresolved("its config() call could not be parsed") from None + if ( + not isinstance(call, ast.Call) + or not isinstance(call.func, ast.Name) + or call.func.id != "config" + or call.args + or any(keyword.arg is None for keyword in call.keywords) + ): + raise DbtUnresolved("its config() call is not plain keyword arguments") + for keyword in call.keywords: + found[str(keyword.arg)] = _literal(keyword.value) + return found + + +def _tree_configs(node: Any) -> Dict[str, Any]: + """The configs set directly on one level of ``dbt_project.yml``'s ``models:`` tree: + a ``+key``, or a key whose value is not a mapping (a folder holds a mapping).""" + out: Dict[str, Any] = {} + if not isinstance(node, Mapping): + return out + for key, value in node.items(): + key = str(key) + if key.startswith("+"): + out[key[1:].replace("-", "_")] = UNKNOWN if _jinja(value) else value + elif not isinstance(value, Mapping): + out[key.replace("-", "_")] = UNKNOWN if _jinja(value) else value + return out + + +def _project_configs(models: Any, project: str, fqn: Sequence[str]) -> Dict[str, Any]: + """``dbt_project.yml``'s configs for the node at ``fqn`` (folders, then the model), + the most specific last: root ``+`` configs, the project's, then each folder's.""" + merged = dict(_tree_configs(models)) + node = models.get(project) if isinstance(models, Mapping) else None + for level in [None, *fqn]: + if level is not None: + node = node.get(level) if isinstance(node, Mapping) else None + merged.update(_tree_configs(node)) + if not isinstance(node, Mapping): + break + return merged + + +def _model_paths(config: Mapping[str, Any]) -> List[PurePosixPath]: + raw = config.get("model-paths", config.get("source-paths", ["models"])) + out = [] + for entry in [raw] if isinstance(raw, str) else raw if isinstance(raw, list) else []: + rel = PurePosixPath(str(entry)) + if rel.is_absolute() or ".." in rel.parts or not rel.parts: + raise DbtUnresolved(f"model-paths names {str(entry)!r}, outside the project") + out.append(rel) + return out + + +def _walk_models( + project: Path, roots: Sequence[PurePosixPath] +) -> Iterator[Tuple[Path, Tuple[str, ...]]]: + """``(file, folders below its model path)`` for every file under the model paths, + never through a link.""" + count = 0 + for rel in roots: + base = project / rel + if base.is_symlink() or not base.is_dir(): + continue + if not base.resolve().is_relative_to(project.resolve()): + continue # a folder on the way is a link out of the project + for here, dirs, names in os.walk(base, followlinks=False): + dirs[:] = sorted( + d for d in dirs if not d.startswith(".") and not (Path(here) / d).is_symlink() + ) + for name in sorted(names): + path = Path(here) / name + if name.startswith(".") or path.is_symlink() or not path.is_file(): + continue + count += 1 + if count > MAX_FILES: + raise DbtUnresolved(f"the model paths hold more than {MAX_FILES} files") + yield path, Path(here).relative_to(base).parts + + +def resolve_model(project: Path, relation: str) -> DbtModel: + """The dbt model of ``project`` that creates the relation ``relation``, with the + materialization and Delta properties dbt resolves for it. + + Raises :class:`DbtUnresolved` when there is no such model, more than one, or a value + that decides the outcome is not a plain literal. + """ + config = _yaml(project / "dbt_project.yml") + if not isinstance(config, Mapping) or not isinstance(config.get("name"), str): + raise DbtUnresolved("dbt_project.yml has no project name") + name = config["name"] + models = config.get("models") or {} + sql: Dict[str, Tuple[Path, Tuple[str, ...]]] = {} + properties: Dict[str, Mapping[str, Any]] = {} + for path, folders in _walk_models(project, _model_paths(config)): + if path.suffix == ".sql": + if path.stem in sql: + raise DbtUnresolved(f"two models are named {path.stem!r}") + sql[path.stem] = (path, folders) + elif path.suffix == ".py": + sql.setdefault(path.stem, (path, folders)) + elif path.suffix in (".yml", ".yaml"): + document = _yaml(path) + for entry in (document.get("models") if isinstance(document, Mapping) else None) or []: + if isinstance(entry, Mapping) and isinstance(entry.get("name"), str): + if entry.get("versions"): + raise DbtUnresolved(f"model {entry['name']!r} is versioned") + block = entry.get("config") + properties[entry["name"]] = block if isinstance(block, Mapping) else {} + + wanted = relation.lower() + candidates: Dict[str, Dict[str, Any]] = {} + for model, (path, folders) in sql.items(): + merged = _project_configs(models, name, (*folders, model)) + merged.update( + { + str(k).replace("-", "_"): (UNKNOWN if _jinja(v) else v) + for k, v in properties.get(model, {}).items() + } + ) + text = "" + if path.suffix == ".sql": + try: + if path.stat().st_size > MAX_FILE_BYTES: + raise DbtUnresolved(f"{path.name} is larger than {MAX_FILE_BYTES} bytes") + text = path.read_text(encoding="utf-8") + merged.update(sql_config(text)) + except (OSError, UnicodeDecodeError): + raise DbtUnresolved(f"{path.name} could not be read") from None + except DbtUnresolved as exc: + if "alias" in text: + raise DbtUnresolved( + f"model {model!r}: {exc}, so it cannot be ruled out as the writer of " + f"{relation!r}" + ) from None + merged["_sql_error"] = str(exc) + alias = merged.get("alias") + if alias is UNKNOWN: + raise DbtUnresolved( + f"the alias of model {model!r} is not a plain value, so it cannot be ruled " + f"out as the writer of {relation!r}" + ) + if str(alias if alias else model).lower() == wanted: + candidates[model] = merged + if not candidates: + raise DbtUnresolved(f"no model in the project creates a relation named {relation!r}") + if len(candidates) > 1: + raise DbtUnresolved(f"models {', '.join(sorted(candidates))} all create {relation!r}") + model, merged = next(iter(candidates.items())) + if sql[model][0].suffix != ".sql": + raise DbtUnresolved(f"model {model!r} is a Python model") + if merged.get("_sql_error"): + raise DbtUnresolved(f"model {model!r}: {merged['_sql_error']}") + if merged.get("enabled") is UNKNOWN or merged.get("enabled") in (False, "false", "False"): + raise DbtUnresolved(f"model {model!r} is disabled or its enabled is not a plain value") + for key in ("schema", "database", "catalog"): + if merged.get(key) not in (None, ""): + raise DbtUnresolved( + f"model {model!r} sets {key}, so its relation may not be in the expose's " + f"{'schema' if key == 'schema' else 'catalog'}" + ) + materialized = merged.get("materialized", "view") + if not isinstance(materialized, str): + raise DbtUnresolved(f"model {model!r}: materialized is not a plain value") + props = merged.get("tblproperties") or {} + if not isinstance(props, Mapping): + raise DbtUnresolved(f"model {model!r}: tblproperties is not a plain mapping") + path, folders = sql[model] + return DbtModel( + model, + materialized, + {str(k): v for k, v in props.items()}, + fqn=(name, *folders, model), + path=path.relative_to(project).as_posix(), + ) + + +# ── dbt: does the build select the model? ─────────────────────────────────── + +#: Selection methods this module does not evaluate (a tag, a config, the state of a +#: previous run): a selector using one cannot be said to include the model. +_METHOD_RE = re.compile(r"^([A-Za-z_.]+):") +_GRAPH_RE = re.compile(r"^(@)?(\d*\+)?(.*?)(\+\d*)?$") + + +def _fqn_selects(fqn: List[str], selector: str) -> bool: + """dbt's fqn method for one node, as dbt-core 1.11 has it (``dbt/graph/selector_methods.py`` + ``is_selected_node``, called with the fqn and with the fqn less its project by + ``QualifiedNameSelectorMethod.node_is_match``; Apache-2.0): the model's own name exactly, + else the selector's dot-separated parts as an exact prefix of the fqn up to the first part + with a wildcard, and from there one ``fnmatch`` of the rest. Case-sensitive, the stricter + reading. A versioned model is refused before this is asked.""" + if fqn[-1] == selector: + return True + flat = [item for segment in fqn for item in segment.split(".")] + parts = selector.split(".") + if len(flat) < len(parts): + return False + for i, part in enumerate(parts): + if any(w in part for w in ("*", "?", "[", "]")): + return fnmatch.fnmatchcase(".".join(flat[i:]), ".".join(parts[i:])) + if flat[i] != part: + return False + return True + + +def _part_selects(part: str, model: DbtModel) -> Optional[bool]: + """Whether one selector (no space, no comma) includes ``model``: ``True``, ``False``, or + ``None`` when it cannot be told without dbt's graph or run state (a graph operator on + another node, a tag or config method). + + Only what is decidable from the project's files is: a node's own name or fully qualified + name (prefixes and ``*`` included), and ``path:``. A graph operator (``+``, ``N+``, ``@``) + includes the named node, and nodes around it that only dbt's graph knows. + """ + if part == "*": + return True + method = _METHOD_RE.match(part) + value = part + if method: + name = method.group(1).lower() + value = part[method.end() :] + if name not in ("fqn", "path", "model"): + return None + else: + name = "path" if "/" in part or part.endswith(".sql") else "fqn" + graph = _GRAPH_RE.match(value) + operators = bool(graph and (graph.group(1) or graph.group(2) or graph.group(4))) + value = graph.group(3) if graph else value + if name == "path": + wanted = PurePosixPath(value.rstrip("/")) + have = PurePosixPath(model.path) + hit = len(wanted.parts) <= len(have.parts) and all( + fnmatch.fnmatchcase(h, w) for h, w in zip(have.parts, wanted.parts, strict=False) + ) + else: + hit = any(_fqn_selects(list(fqn), value) for fqn in (model.fqn, model.fqn[1:]) if fqn) + if hit: + return True + return None if operators else False + + +def selects_model(selectors: Sequence[str], model: DbtModel) -> Optional[bool]: + """Whether ``dbt build --select `` builds ``model``: ``True``, ``False``, or + ``None`` when it cannot be told. No selector builds every model. Space-separated + selectors are a union, a comma joins an intersection (dbt's selection syntax, + https://docs.getdbt.com/reference/node-selection/syntax).""" + tokens = [t for item in selectors for t in str(item).split()] + if not tokens: + return True + unknown = False + for token in tokens: + results = [_part_selects(part, model) for part in token.split(",") if part] + if not results or any(r is False for r in results): + continue + if all(r is True for r in results): + return True + unknown = True + return None if unknown else False + + +# ── the decision ──────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class Retention: + """An accepted ``lifecycle.expire``: what ``fluid verify`` holds the live table to.""" + + #: ``lifecycle.retention`` as written, and in seconds (years and months at their shortest). + period: str + seconds: int + + +def _is_dbt(build: Mapping[str, Any]) -> bool: + engine = str(build.get("engine") or "").strip().lower() + return engine == "dbt" or engine.startswith("dbt-") + + +def _fix_materialized() -> str: + return "Declare materialized='table' on the model ({{ config(materialized='table') }})." + + +def _templated(value: Any) -> bool: + """Whether ``value`` still holds an ``{{ env.X }}`` placeholder: ``fluid validate`` reads + the contract before the environment is substituted, the emitter after.""" + return isinstance(value, str) and ENV_PLACEHOLDER_RE.search(value) is not None + + +def _dbt_model( + build: Mapping[str, Any], table: str, where: str, validating: bool +) -> Tuple[Optional[DbtModel], Optional[UnsupportedBindingError]]: + """The dbt model of ``build`` for ``table``; ``(None, None)`` when ``validating`` with + no contract file to find the project from, as the job emitter does, or with a table + name that is still an environment placeholder (the emitter resolves it).""" + if validating and _templated(table): + return None, None + label = f"{where}: build {build.get('id')!r}" + repository = str(build.get("repository") or "").strip() + if "://" in repository or repository.startswith("git@"): + return None, _refusal( + "dbt-unresolved", + f"{label} runs a dbt project from a Git repository ({repository.split('?')[0]}), " + "which forge-cli cannot read here, so it cannot tell whether the model for " + f"{table!r} is rebuilt in full.", + "Point repository at a local dbt project directory (forge-cli reads it, and " + "uploads it to the workspace), and " + _fix_materialized(), + ) + source = current_contract_source() + if source is None: + if validating: + return None, None + return None, _refusal( + "dbt-unresolved", + f"{label}: the contract's location is not known here, so its dbt project cannot " + "be read to see how the model is materialized.", + "Run it through fluid apply / fluid generate iac with the contract's path.", + ) + from ...build_runners.dbt.runner import resolve_dbt_project_path + + project = resolve_dbt_project_path(Path(source), dict(build)) + if project is None: + return None, _refusal( + "dbt-unresolved", + f"{label}: repository {repository!r} holds no dbt_project.yml, so the model's " + "materialization cannot be read.", + "Point repository at the dbt project directory, relative to the contract.", + ) + try: + return resolve_model(project, table), None + except DbtUnresolved as exc: + return None, _refusal( + "dbt-unresolved", + f"{label}: cannot tell whether the model for {table!r} is rebuilt in full: {exc}.", + _fix_materialized(), + ) + + +def _selection_refusal( + build: Mapping[str, Any], model: DbtModel, table: str, label: str +) -> Optional[UnsupportedBindingError]: + """A refusal unless the build's dbt selection includes the resolved model: a model the + build does not run is not rebuilt, and its table keeps whatever it last held.""" + from ...build_runners.dbt.runner import build_selectors + + selectors = build_selectors(dict(build)) + chosen = selects_model(selectors, model) + if chosen: + return None + shown = " ".join(selectors) + how = ( + "does not select it" + if chosen is False + else "selects it only through graph operators, tags, configs or state that forge-cli " + "cannot evaluate here" + ) + return _refusal( + "not-selected", + f"{label}: the table {table!r} is written by model {model.name!r}, but the build's " + f"selection ({shown!r}, from properties.model or properties.select) {how}, so a run " + "need not rebuild the table and rows older than the retention could stay.", + f"Select the model by name: properties.select: [{model.name!r}] (or " + f"properties.model: {model.name}).", + ) + + +def _acquisition_mode_refusal( + build: Mapping[str, Any], label: str, validating: bool = False +) -> Optional[UnsupportedBindingError]: + """A refusal unless the DuckDB acquisition's ``properties.source.mode`` is one the + Databricks landing runs as a full overwrite of the table (an environment placeholder + is left to the emitter, which reads it substituted).""" + from ...build_runners._databricks_load import LOAD_MODES + + properties = build.get("properties") + source = properties.get("source") if isinstance(properties, Mapping) else None + mode = source.get("mode") if isinstance(source, Mapping) else None + mode = getattr(mode, "value", mode) + if mode in LOAD_MODES or (validating and _templated(mode)): + return None + fix = ( + f"Set properties.source.mode to {LOAD_MODES[0]!r} (the landing overwrites the whole " + "table), or remove lifecycle.expire." + ) + if not mode: + return _refusal( + "unmapped", + f"{label} sets no properties.source.mode, so nothing says it rewrites the whole " + "table, and rows older than the retention could stay.", + fix, + ) + return _refusal( + "incremental", + f"{label}: the acquisition's source mode is {mode!r}, and only " + f"{', '.join(repr(m) for m in LOAD_MODES)} rewrites the whole table on Databricks, so " + "rows older than the retention could stay.", + fix, + ) + + +def check_expire( + where: str, + exposure: Mapping[str, Any], + table: str, + contract: Mapping[str, Any], + *, + validating: bool = False, +) -> Tuple[Optional[Retention], List[UnsupportedBindingError]]: + """The retention ``exposure`` promises and every reason this module refuses it. + + ``(None, [])`` when it sets no ``lifecycle.expire``. ``table`` is the expose's table + name, the relation dbt must create. + """ + lifecycle = exposure.get("lifecycle") + if not isinstance(lifecycle, Mapping) or not lifecycle.get("expire"): + return None, [] + refusals: List[UnsupportedBindingError] = [] + period = lifecycle.get("retention") + seconds: Optional[int] = None + if validating and _templated(period): + pass # the emitter checks it once the environment is substituted + elif not period: + refusals.append( + _refusal( + "period", + f"{where} sets lifecycle.expire but no lifecycle.retention, so there is no " + "period to keep the data for.", + "Set lifecycle.retention (an ISO 8601 duration such as P365D), or drop expire.", + ) + ) + else: + try: + seconds = iso_duration_seconds(period) + except ValueError: + refusals.append( + _refusal( + "period", + f"{where} lifecycle.retention is {period!r}, which is not an ISO 8601 " + "duration.", + "Write it as an ISO 8601 duration, such as P30D, P12W or P6Y.", + ) + ) + tblproperties: Mapping[str, Any] = {} + builds = [b for b in contract.get("builds") or [] if isinstance(b, Mapping)] + writers = builds_writing_expose(exposure.get("exposeId"), builds) + if not writers: + refusals.append( + _refusal( + "unmapped", + f"{where} sets lifecycle.expire, and no build writes it, so nothing says the " + "table is rebuilt in full: rows it holds could be any age, and nothing on " + "Databricks would expire them.", + "Name the build that rewrites the whole table in builds[].outputs, or remove " + "lifecycle.expire.", + ) + ) + for build in writers: + label = f"{where}: build {build.get('id')!r}" + if is_duckdb_acquisition_build(build): + # forge-cli's acquisition lands into Databricks as an INSERT OVERWRITE of the + # whole table, and only in the modes the landing accepts (it refuses any other + # before the build runs); acceptance rests on both. If the landing ever appends, + # this must refuse. + refusal = _acquisition_mode_refusal(build, label, validating) + if refusal is not None: + refusals.append(refusal) + continue + if not _is_dbt(build): + refusals.append( + _refusal( + "unmapped", + f"{label} (engine {build.get('engine')!r}) writes it, and forge-cli cannot " + "tell that it rewrites the whole table, so rows older than the retention " + "could stay.", + "Use a dbt build with materialized='table', or a DuckDB acquisition, or " + "remove lifecycle.expire.", + ) + ) + continue + model, refusal = _dbt_model(build, table, where, validating) + if refusal is not None: + refusals.append(refusal) + elif model is not None and (selection := _selection_refusal(build, model, table, label)): + refusals.append(selection) + elif model is not None: + kind = model.materialized + if kind in ("incremental", "snapshot"): + refusals.append( + _refusal( + "incremental", + f"{label}: model {model.name!r} is materialized as {kind!r}, which " + "keeps rows from earlier runs, so no run leaves the table without a " + "row older than the retention. Delta history limits cannot expire " + "rows.", + "Rebuild it in full (materialized='table'), or drop lifecycle.expire " + "and delete old rows in the model.", + ) + ) + elif kind != "table": + refusals.append( + _refusal( + "not-table", + f"{label}: model {model.name!r} is materialized as {kind!r} (dbt's " + "default is 'view'), not 'table', so a run does not replace the " + "stored table.", + _fix_materialized(), + ) + ) + else: + tblproperties = {**tblproperties, **model.tblproperties} + effective = DEFAULT_LOG_SECONDS + try: + effective = history_seconds(tblproperties) + except ValueError as exc: + refusals.append( + _refusal( + "dbt-unresolved", + f"{where}: the model's Delta retention properties cannot be read ({exc}).", + f"Write {LOG_RETENTION} and {DELETED_FILE_RETENTION} as 'interval N days'.", + ) + ) + if seconds is not None and seconds < effective: + + def _shown(key: str, default: int) -> str: + value = tblproperties.get(key) + return f"{value!s} (declared)" if value else f"{days_text(default)} (Delta's default)" + + refusals.append( + _refusal( + "too-short", + f"{where} lifecycle.retention is {period} ({days_text(seconds)}), shorter than " + f"the table's Delta history of {days_text(effective)} ({LOG_RETENTION} " + f"{_shown(LOG_RETENTION, DEFAULT_LOG_SECONDS)}, {DELETED_FILE_RETENTION} " + f"{_shown(DELETED_FILE_RETENTION, DEFAULT_DELETED_FILE_SECONDS)}). VACUUM keeps " + "old data files that long, so a copy older than the retention would remain. " + "forge-cli does not shorten the history yet.", + f"Set lifecycle.retention to at least P{-(-effective // DAY)}D, or remove " + "lifecycle.expire.", + ) + ) + if refusals: + return None, refusals + if seconds is None: + # validating with a placeholder retention: nothing to promise yet; the emitter + # reads it substituted + return None, [] + return Retention(period=str(period), seconds=seconds), [] + + +__all__ = [ + "DEFAULT_DELETED_FILE_SECONDS", + "DEFAULT_LOG_SECONDS", + "DELETED_FILE_RETENTION", + "LOG_RETENTION", + "DbtModel", + "DbtUnresolved", + "Retention", + "check_expire", + "days_text", + "history_seconds", + "interval_seconds", + "iso_duration_seconds", + "selects_model", + "resolve_model", + "sql_config", +] diff --git a/fluid_build/util/contract.py b/fluid_build/util/contract.py index 31bceda67..02891e6e8 100644 --- a/fluid_build/util/contract.py +++ b/fluid_build/util/contract.py @@ -202,6 +202,21 @@ def is_databricks_job_build(build: Mapping[str, Any]) -> bool: return isinstance(platform, str) and platform.strip().lower() == "databricks" +def is_duckdb_acquisition_build(build: Mapping[str, Any]) -> bool: + """True when forge-cli's DuckDB acquisition runner runs the build: ``pattern: + acquisition`` and ``engine: duckdb``. That runner applies ``policy.privacy.masking`` to + the rows it lands, before they leave the machine (``build_runners/_masking.py``) and + lands a Databricks expose in its table (``build_runners/_databricks_load.py``).""" + pattern = build.get("pattern") + engine = get_build_engine(build) + return ( + isinstance(pattern, str) + and pattern.strip().lower() == "acquisition" + and isinstance(engine, str) + and engine.strip().lower() == "duckdb" + ) + + def is_unscheduled_build(build: Mapping[str, Any]) -> bool: """True when no scheduler starts the build's runs: its job runs outside forge-cli (:func:`is_externally_run_build`), continuously diff --git a/tests/build_runners/test_duckdb_databricks_load.py b/tests/build_runners/test_duckdb_databricks_load.py new file mode 100644 index 000000000..daed68e6b --- /dev/null +++ b/tests/build_runners/test_duckdb_databricks_load.py @@ -0,0 +1,717 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Databricks binding is built locally, uploaded to a volume and loaded into the managed table. + +What these pin, each of which was wrong before: for ``platform: databricks`` the acquisition +runner wrote a LOCAL file (``location.path``, else ``out/``) and reported success with nothing +in Databricks. Now the run's Parquet stages under ``.fluid/staging``, goes to +``$DATABRICKS_STAGING_VOLUME`` through the Files API, one ``INSERT OVERWRITE ... read_files`` +on ``$DATABRICKS_WAREHOUSE_ID`` replaces the table's rows, and the staged files are deleted +whether the load worked or not. + +The workspace is ``tests/iac/_fake_databricks.py``, an in-process stand-in: no Databricks +account, credential or network is involved. These tests need no ``databricks-sdk``: the client is +the stand-in's own ``api_client.do`` over HTTP. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import time +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional + +import pytest + +from fluid_build.build_runners import _databricks_load as load + +# The iac-tests job collects every file with no duckdb installed. +duckdb = pytest.importorskip("duckdb") +from fluid_build.build_runners import _masking as masking # noqa: E402 +from fluid_build.build_runners.duckdb.runner import execute_duckdb_build # noqa: E402 + +from ..iac._fake_databricks import FakeDatabricks # noqa: E402 + +pytestmark = pytest.mark.unit + +VOLUME = "/Volumes/main/staging/landing" +TABLE = "main.bronze.subs" +SALT = "landing-test-salt-0123456789" +COLUMNS = [("id", "string", True), ("msisdn", "string", False), ("zip", "string", False)] +ROWS = [("1", "+46701234567", "11122"), ("2", "+46709876543", "22233"), ("3", "", "33344")] + + +@pytest.fixture +def fake() -> Iterator[FakeDatabricks]: + server = FakeDatabricks().start() + server.add_volume("main.staging.landing") + server.create_table(TABLE, COLUMNS) + try: + yield server + finally: + server.stop() + + +@pytest.fixture(autouse=True) +def workspace(monkeypatch, fake: FakeDatabricks) -> None: + """The stand-in as the only workspace the loader can reach, and its secrets as env.""" + for name in ("DATABRICKS_HOST", "DATABRICKS_TOKEN", "DATABRICKS_CONFIG_PROFILE"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("DATABRICKS_WAREHOUSE_ID", fake.warehouse_id) + monkeypatch.setenv(load.STAGING_VOLUME_ENV, VOLUME) + monkeypatch.setenv(masking.DEFAULT_SALT_ENV, SALT) + monkeypatch.setattr(load, "default_client", fake.client) + + +# ── the staging volume: one environment variable, three identifiers ────── + + +@pytest.mark.parametrize("value", [VOLUME, VOLUME + "/", "/Volumes/Main_1/_s/v2"]) +def test_a_staging_volume_is_three_identifiers_under_volumes(value): + volume = load.staging_volume({load.STAGING_VOLUME_ENV: value}) + assert (volume.catalog, volume.schema, volume.volume) == tuple(value.strip("/").split("/")[1:]) + assert volume.path == value.rstrip("/") + + +@pytest.mark.parametrize( + "value", + [ + "", + " ", + "/Volumes/main/staging", + "/Volumes/main/staging/landing/extra", + "main.staging.landing", + "/volumes/main/staging/landing", + "/Volumes/main/../landing", + "/Volumes/ma-in/staging/landing", + "/Volumes/main/stag ing/landing", + "/Volumes/main/staging/`landing`", + "/Volumes/main/staging/landing;DROP", + "/Volumes//staging/landing", + "dbfs:/Volumes/main/staging/landing", + ], +) +def test_a_staging_volume_that_is_not_one_is_refused_by_name(value): + with pytest.raises(load.DatabricksLoadError) as exc: + load.staging_volume({load.STAGING_VOLUME_ENV: value}) + assert load.STAGING_VOLUME_ENV in str(exc.value) + assert "/Volumes///" in str(exc.value) + + +def test_an_unset_staging_volume_is_refused_by_name(): + with pytest.raises(load.DatabricksLoadError) as exc: + load.staging_volume({}) + assert load.STAGING_VOLUME_ENV in str(exc.value) and "not set" in str(exc.value) + + +def test_a_run_stages_under_its_product_build_and_run(): + volume = load.staging_volume({load.STAGING_VOLUME_ENV: VOLUME}) + prefix = load.run_prefix(volume, "bronze.subs", "ingest", "run-20261103-abc") + assert prefix == f"{VOLUME}/bronze_subs/ingest/run-20261103-abc" + + +@pytest.mark.parametrize("hostile", ["../../x", "..", "a/b", "a\\b", "a'b", "a b", "", "é"]) +def test_a_contract_supplied_id_cannot_climb_out_of_the_volume(hostile): + volume = load.staging_volume({load.STAGING_VOLUME_ENV: VOLUME}) + for prefix in ( + load.run_prefix(volume, hostile, "b", "r"), + load.run_prefix(volume, "p", hostile, "r"), + load.run_prefix(volume, "p", "b", hostile), + ): + assert prefix.startswith(VOLUME + "/") + tail = prefix[len(VOLUME) + 1 :].split("/") + assert len(tail) == 3 and all(tail) and ".." not in tail + assert all( + set(part) <= set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-") + for part in tail + ) + + +# ── the target and the statement ───────────────────────────────────────── + + +def _expose(**location: Any) -> Dict[str, Any]: + loc = {"database": "Main", "schema": "bronze", "table": "subs", **location} + return { + "exposeId": "subs", + "kind": "table", + "binding": {"platform": "databricks", "format": "delta_table", "location": loc}, + "contract": { + # varchar, not string: the run-time schema gate compares the declared type with + # DuckDB's own (VARCHAR) and reads "string" as a type change; the emitter maps both + # to string + "schema": [ + {"name": "id", "type": "varchar"}, + {"name": "msisdn", "type": "varchar"}, + {"name": "zip", "type": "varchar"}, + ], + }, + } + + +def test_the_target_is_the_table_apply_made_with_its_columns_in_order(): + expose = _expose() + target = load.databricks_load_target(expose["binding"], expose) + assert target.full_name == "main.bronze.subs" # lower-cased, as the emitter creates it + assert target.columns == ("id", "msisdn", "zip") + + +@pytest.mark.parametrize("platform", ["local", "gcp", "aws", "snowflake", ""]) +def test_another_platform_has_no_databricks_target(platform): + expose = _expose() + expose["binding"]["platform"] = platform + assert load.databricks_load_target(expose["binding"], expose) is None + + +def test_the_insert_overwrite_statement_is_exact(): + expose = _expose() + target = load.databricks_load_target(expose["binding"], expose) + assert load.insert_overwrite_statement(target, f"{VOLUME}/bronze_subs/ingest/run-1") == ( + "INSERT OVERWRITE `main`.`bronze`.`subs` (`id`, `msisdn`, `zip`) SELECT `id`, `msisdn`, `zip` " + f"FROM read_files('{VOLUME}/bronze_subs/ingest/run-1', format => 'parquet')" + ) + + +def test_the_path_is_a_spark_string_literal_with_backslash_escapes(): + expose = _expose() + target = load.databricks_load_target(expose["binding"], expose) + sql = load.insert_overwrite_statement(target, "/Volumes/a/b/c/it's\\x") + assert sql.endswith("read_files('/Volumes/a/b/c/it\\'s\\\\x', format => 'parquet')") + + +@pytest.mark.parametrize( + "mutate, fragment", + [ + (lambda e: e["binding"]["location"].update(table="subs`; DROP TABLE x"), "table"), + (lambda e: e["binding"]["location"].update(database="a.b"), "database"), + (lambda e: e["binding"]["location"].pop("schema"), "schema"), + (lambda e: e["contract"].update(schema=[]), "contract.schema"), + (lambda e: e["contract"]["schema"].append({"name": "ID", "type": "string"}), "twice"), + ( + lambda e: e["contract"]["schema"].append({"name": "x`) FROM t --", "type": "string"}), + "name", + ), + ], +) +def test_an_unsafe_or_incomplete_binding_is_refused_before_any_statement(mutate, fragment): + expose = _expose() + mutate(expose) + with pytest.raises(load.DatabricksLoadError) as exc: + load.databricks_load_target(expose["binding"], expose) + assert fragment in str(exc.value) + + +@pytest.mark.parametrize( + "mode, sink, streams, fragment", + [ + ("full_refresh", "parquet", 2, "exactly one stream"), + ("full_refresh", "csv", 1, "sink format"), + ("incremental_append", "parquet", 1, "full_refresh"), + ("cdc", "parquet", 1, "full_refresh"), + ], +) +def test_unsupported_shapes_are_named(mode, sink, streams, fragment): + assert fragment in load.unsupported_reason(mode, sink, streams) + + +def test_a_full_refresh_of_one_parquet_stream_is_supported(): + assert load.unsupported_reason("full_refresh", "parquet", 1) is None + + +# ── upload, load, delete: the landing itself ───────────────────────────── + + +def _parquet(tmp_path: Path, rows=ROWS, name: str = "subs.parquet") -> Path: + path = tmp_path / name + con = duckdb.connect() + try: + values = ", ".join("(" + ", ".join(repr(v) for v in row) + ")" for row in rows) + con.execute( + f"COPY (SELECT * FROM (VALUES {values}) AS t(id, msisdn, zip)) TO '{path}' (FORMAT parquet)" + ) + finally: + con.close() + return path + + +def _landing(fake: FakeDatabricks) -> "load.Landing": + expose = _expose() + target = load.databricks_load_target(expose["binding"], expose) + return load.open_landing(target) + + +def _load(landing, path: Path, *, expected: int = 3, run_id: str = "run-1") -> Dict[str, Any]: + return landing.load( + str(path), + product_id="bronze.subs", + build_id="ingest", + run_id=run_id, + expected_rows=expected, + logger=logging.getLogger("test"), + ) + + +def test_the_file_is_uploaded_loaded_and_the_staged_copy_deleted(fake, tmp_path): + facts = _load(_landing(fake), _parquet(tmp_path)) + + assert [r["id"] for r in fake.rows(TABLE)] == ["1", "2", "3"] + assert facts["table"] == TABLE and facts["rows"] == 3 + assert facts["staged_files_deleted"] is True + run_dir = f"{VOLUME}/bronze_subs/ingest/run-1" + assert facts["staged_to"] == run_dir + assert fake.volume_files() == [], "never leave landed data in the volume" + puts = [p for m, p in fake.file_ops if m == "PUT"] + assert puts == [f"{run_dir}/subs.parquet"] + # in order: earlier runs looked for, the directory, the file, then (after the load) the file + # and the directory go + assert [m for m, _ in fake.file_ops] == ["LIST", "PUT-DIR", "PUT", "DELETE", "DELETE-DIR"] + assert fake.sql_log[-1].startswith("INSERT OVERWRITE `main`.`bronze`.`subs`") + assert fake.unknown == [] + + +def test_the_uploaded_bytes_are_the_files_bytes(fake, tmp_path): + path = _parquet(tmp_path) + fake.fail_on("INSERT OVERWRITE") # the load fails, so only the upload is looked at + landing = _landing(fake) + seen: List[bytes] = [] + real = fake._fs_put + + def spy(m, q, b): + seen.append(b.get("_raw")) + return real(m, q, b) + + fake.routes = [(mth, rx, spy if fn == real else fn) for mth, rx, fn in fake.routes] + with pytest.raises(load.DatabricksLoadError): + _load(landing, path) + assert seen == [path.read_bytes()] + + +def test_a_second_load_replaces_the_rows_it_does_not_append(fake, tmp_path): + landing = _landing(fake) + _load(landing, _parquet(tmp_path), run_id="run-1") + newer = _parquet(tmp_path, rows=[("9", "+1", "99999")], name="later.parquet") + _load(landing, newer, expected=1, run_id="run-2") + assert [r["id"] for r in fake.rows(TABLE)] == ["9"] + assert fake.volume_files() == [] + + +def test_the_load_keeps_the_table_s_definition_row_filter_and_masks(fake, tmp_path): + fake.tables[TABLE]["row_filter"] = { + "function_name": "main.bronze.fluid_rf_x", + "input_column_names": ["id"], + } + fake.tables[TABLE]["columns"][1]["mask"] = {"function_name": "main.bronze.fluid_mask_y"} + before = json.dumps(fake.tables[TABLE], sort_keys=True) + _load(_landing(fake), _parquet(tmp_path)) + assert json.dumps(fake.tables[TABLE], sort_keys=True) == before + assert not [ + s for s in fake.sql_log if s.lstrip().upper().startswith(("CREATE", "ALTER", "DROP")) + ] + assert not [s for s in fake.sql_log if "REPLACE TABLE" in s.upper()], "INSERT OVERWRITE only" + + +def test_a_missing_table_names_fluid_apply_and_uploads_nothing(fake, tmp_path): + del fake.tables[TABLE] + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + assert "fluid apply" in str(exc.value) and TABLE in str(exc.value) + assert fake.file_ops == [], "nothing was uploaded for a table that is not there" + assert not [s for s in fake.sql_log if "INSERT" in s] + assert TABLE not in fake.tables, "the loader never creates the table" + + +def test_a_failed_load_still_deletes_the_staged_files(fake, tmp_path): + fake.fail_on("INSERT OVERWRITE", "[INSUFFICIENT_PERMISSIONS] no MODIFY on the table") + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + assert "INSUFFICIENT_PERMISSIONS" in str(exc.value) + assert fake.volume_files() == [] + assert "deleted" in str(exc.value) + assert fake.rows(TABLE) == [] + + +def test_a_short_load_fails_the_build_and_deletes_the_staged_files(fake, tmp_path): + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path), expected=4) + assert f"loaded 3 rows into {TABLE}, but the landed file holds 4" in str(exc.value) + assert fake.volume_files() == [] + + +def test_a_load_that_reports_no_count_is_counted_in_the_table(fake, tmp_path): + fake.insert_reports_count = False + facts = _load(_landing(fake), _parquet(tmp_path)) + assert facts["rows"] == 3 and facts["rows_from"] == "count_after_load" + assert any("COUNT" in s.upper() for s in fake.sql_log) + + +def test_a_count_that_fails_after_the_load_says_the_load_ran(fake, tmp_path): + fake.insert_reports_count = False + fake.fail_on("COUNT(", "[INTERNAL_ERROR] no count") + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + text = str(exc.value) + assert f"loaded {TABLE}, but counting its rows afterwards failed" in text + assert "loading" not in text.split("but")[0] + assert fake.volume_files() == [] + + +def test_a_staged_file_that_cannot_be_deleted_fails_the_run_naming_it(fake, tmp_path): + fake.fail_route("DELETE", r"/fs/files/", 403, "no permission to delete") + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + text = str(exc.value) + assert "could not delete" in text and "subs.parquet" in text + assert [r["id"] for r in fake.rows(TABLE)] == ["1", "2", "3"], "the load itself worked" + assert fake.volume_files() != [], "and the file is still there: the error says so" + + +def test_a_failed_upload_is_cleaned_up_and_nothing_loads(fake, tmp_path): + fake.fail_route("PUT", r"/fs/files/", 403, "PERMISSION_DENIED: no WRITE VOLUME") + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + assert "PERMISSION_DENIED" in str(exc.value) + assert not [s for s in fake.sql_log if "INSERT" in s] + assert fake.volume_files() == [] + + +def test_the_load_puts_each_column_in_the_column_of_its_name(fake, tmp_path): + # fluid apply appended `id` to a live table that keeps the other two first: by position, + # the msisdn values would land in `id` + cols = fake.tables[TABLE]["columns"] + fake.tables[TABLE]["columns"] = cols[1:] + cols[:1] + _load(_landing(fake), _parquet(tmp_path)) + assert [r["id"] for r in fake.rows(TABLE)] == ["1", "2", "3"] + assert fake.rows(TABLE)[0]["msisdn"] == ROWS[0][1] + + +def _volume_file( + fake: FakeDatabricks, volume_path: str, data: bytes, *, age_seconds: float = 0 +) -> None: + real = fake._fs_path(volume_path) + assert real is not None, volume_path + Path(real).parent.mkdir(parents=True, exist_ok=True) + Path(real).write_bytes(data) + if age_seconds: + when = time.time() - age_seconds + os.utime(real, (when, when)) + + +_OLD = load.STALE_STAGING_SECONDS + 60 +#: A run id as the acquisition runner makes one (``generate_run_id``). +EARLIER_RUN = "01" + "0" * 15 + "Z" + + +def test_the_load_reads_the_uploaded_file_not_the_run_directory(fake, tmp_path): + run_dir = f"{VOLUME}/bronze_subs/ingest/run-1" + # someone with WRITE VOLUME drops a file into the run's directory before the INSERT + planted = _parquet(tmp_path, rows=[("666", "+0", "00000")], name="planted.parquet") + _volume_file(fake, f"{run_dir}/planted.parquet", planted.read_bytes()) + + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + + insert = [s for s in fake.sql_log if s.startswith("INSERT OVERWRITE")] + assert insert and insert[0].endswith( + f"read_files('{run_dir}/subs.parquet', format => 'parquet')" + ) + assert [r["id"] for r in fake.rows(TABLE)] == ["1", "2", "3"], "the planted rows are not loaded" + # the directory cannot be removed while the planted file is in it: the run says so + assert "could not delete" in str(exc.value) and run_dir in str(exc.value) + + +def test_an_interrupt_mid_load_still_deletes_the_staged_files(fake, tmp_path, monkeypatch): + landing = _landing(fake) + + def interrupted(statement): + raise KeyboardInterrupt + + monkeypatch.setattr(landing._sql, "execute", interrupted) + with pytest.raises(KeyboardInterrupt): + _load(landing, _parquet(tmp_path)) + assert fake.volume_files() == [], "Ctrl-C must not leave landed data in the volume" + assert [m for m, _ in fake.file_ops][-2:] == ["DELETE", "DELETE-DIR"] + + +def test_an_interrupted_statement_is_cancelled_on_the_warehouse(): + from fluid_build.iac.providers.databricks_enforce import SqlRunner + + calls: List[tuple] = [] + + class _Api: + def do(self, method, path, **kwargs): + calls.append((method, path)) + if method == "POST" and path.endswith("/statements/"): + return {"statement_id": "s-1", "status": {"state": "RUNNING"}} + if method == "GET": + raise KeyboardInterrupt + return {} + + class _Client: + api_client = _Api() + + runner = SqlRunner(_Client(), "wh", sleep=lambda _s: None) + with pytest.raises(KeyboardInterrupt): + runner.execute("INSERT OVERWRITE `a`.`b`.`c` SELECT 1") + assert ("POST", "/api/2.0/sql/statements/s-1/cancel") in calls + + +def test_staged_files_an_earlier_killed_run_left_are_deleted_first(fake, tmp_path): + stale = f"{VOLUME}/bronze_subs/ingest/{EARLIER_RUN}" + other_build = f"{VOLUME}/bronze_subs/other/{EARLIER_RUN}" + _volume_file(fake, f"{stale}/subs.parquet", b"left by a run that was killed", age_seconds=_OLD) + _volume_file(fake, f"{other_build}/subs.parquet", b"another build's run", age_seconds=_OLD) + + facts = _load(_landing(fake), _parquet(tmp_path), run_id="run-1") + + assert facts["earlier_runs_staging_deleted"] == [stale] + assert fake.volume_files() == [f"{other_build}/subs.parquet"], "only this build's directory" + assert [r["id"] for r in fake.rows(TABLE)] == ["1", "2", "3"] + + +def test_a_recent_run_of_the_same_build_keeps_its_staged_files(fake, tmp_path): + # another run of this build, uploaded a minute ago, may still be loading + running = f"{VOLUME}/bronze_subs/ingest/{EARLIER_RUN}" + _volume_file(fake, f"{running}/subs.parquet", b"in flight", age_seconds=60) + + facts = _load(_landing(fake), _parquet(tmp_path), run_id="run-1") + + assert facts["earlier_runs_staging_deleted"] == [] + assert fake.volume_files() == [f"{running}/subs.parquet"] + + +@pytest.mark.parametrize( + "path", + [ + "{volume}/bronze_subs/ingest/2024/part-0.parquet", # not a run directory's name + "{volume}/bronze_subs/ingest/{run}/notes.txt", # not what a load stages + "{volume}/bronze_subs/ingest/{run}/nested/part-0.parquet", # a subdirectory + ], +) +def test_the_sweep_leaves_what_a_load_did_not_stage(fake, tmp_path, path): + planted = path.format(volume=VOLUME, run=EARLIER_RUN) + _volume_file(fake, planted, b"someone else's data", age_seconds=_OLD) + + facts = _load(_landing(fake), _parquet(tmp_path), run_id="run-1") + + assert facts["earlier_runs_staging_deleted"] == [] + assert fake.volume_files() == [planted] + + +def test_a_stale_run_that_cannot_be_deleted_does_not_stop_the_load(fake, tmp_path): + stale = f"{VOLUME}/bronze_subs/ingest/{EARLIER_RUN}" + _volume_file(fake, f"{stale}/subs.parquet", b"left behind", age_seconds=_OLD) + fake.fail_route( + "DELETE", rf"/fs/files/Volumes/.*/ingest/{EARLIER_RUN}/", 403, "PERMISSION_DENIED" + ) + facts = _load(_landing(fake), _parquet(tmp_path), run_id="run-1") + assert facts["earlier_runs_staging_deleted"] == [] + assert fake.volume_files() == [f"{stale}/subs.parquet"] + assert [r["id"] for r in fake.rows(TABLE)] == ["1", "2", "3"] + + +def test_a_volume_that_does_not_exist_names_it(fake, tmp_path, monkeypatch): + monkeypatch.setenv(load.STAGING_VOLUME_ENV, "/Volumes/main/staging/nowhere") + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + assert "/Volumes/main/staging/nowhere" in str(exc.value) + + +def test_no_credential_reaches_an_error_message(fake, tmp_path, monkeypatch): + secret = "dapi-0123456789abcdef-not-real" # pragma: allowlist secret + monkeypatch.setenv("DATABRICKS_TOKEN", secret) + fake.fail_on("INSERT OVERWRITE", f"[FAILED] rejected the token {secret}") + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + assert secret not in str(exc.value) + assert "rejected the token" in str(exc.value) + + +def test_nothing_calls_a_second_client_or_endpoint(fake, tmp_path): + _load(_landing(fake), _parquet(tmp_path)) + assert fake.unknown == [] + allowed = ("/api/2.0/fs/", "/api/2.0/sql/statements", "/api/2.1/unity-catalog/tables/") + assert all(path.startswith(allowed) for _, path in fake.requests), fake.requests + + +# ── the runner: a databricks binding lands in the table, never in a local file ──── + + +def _source(tmp_path: Path, rows=ROWS) -> Path: + src = tmp_path / "subs.csv" + lines = ["id,msisdn,zip"] + [",".join(row) for row in rows] + src.write_text("\n".join(lines) + "\n", encoding="utf-8") + return src + + +def _contract( + tmp_path: Path, + *, + mode: str = "full_refresh", + rows=ROWS, + masking_rules: Optional[List[Dict[str, Any]]] = None, + location: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + expose = _expose(**(location or {})) + expose["binding"]["location"]["database"] = "main" + if masking_rules is not None: + expose["policy"] = {"privacy": {"masking": masking_rules}} + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "bronze.subs", + "name": "Subs", + "builds": [ + { + "id": "ingest", + "pattern": "acquisition", + "engine": "duckdb", + "properties": { + "source": { + "kind": "filesystem", + "mode": mode, + "connection": {"uri": str(_source(tmp_path, rows))}, + # all_varchar: the CSV reader would otherwise type msisdn and zip as numbers + "reader": {"format": "csv", "options": {"all_varchar": True}}, + "streams": ["subs"], + }, + "sink": {"format": "parquet"}, + }, + "outputs": ["subs"], + } + ], + "exposes": [expose], + } + + +def _run(contract: Dict[str, Any], tmp_path: Path) -> int: + return execute_duckdb_build(contract["builds"][0], contract, tmp_path, dry_run=False) + + +def _sha(value: str) -> str: + return hashlib.sha256((SALT + value).encode("utf-8")).hexdigest() + + +HASH = [{"column": "msisdn", "strategy": "hash"}] + + +def test_the_runner_lands_the_masked_rows_in_the_table(fake, tmp_path): + assert _run(_contract(tmp_path, masking_rules=HASH), tmp_path) == 0 + + rows = sorted(fake.rows(TABLE), key=lambda r: r["id"]) + assert [r["id"] for r in rows] == ["1", "2", "3"] + assert [r["msisdn"] for r in rows] == [_sha("+46701234567"), _sha("+46709876543"), None] + assert [r["zip"] for r in rows] == ["11122", "22233", "33344"], "the rest is unchanged" + assert "+4670" not in json.dumps(rows), "no cleartext left the machine" + assert fake.volume_files() == [], "and nothing is left in the volume" + assert fake.unknown == [] + + +def test_the_run_record_says_what_was_loaded(fake, tmp_path): + assert _run(_contract(tmp_path, masking_rules=HASH), tmp_path) == 0 + (record,) = (tmp_path / ".fluid" / "runs" / "bronze.subs" / "ingest" / "runs").glob("*.json") + facets = json.loads(record.read_text(encoding="utf-8"))["facets"] + facts = facets["databricks_load"] + assert facts["table"] == TABLE and facts["rows"] == 3 + assert facts["staged_files_deleted"] is True + assert facts["staged_to"].startswith(f"{VOLUME}/bronze_subs/ingest/") + assert facets["masking"]["columns"]["msisdn"]["strategy"] == "hash" + assert VOLUME not in json.dumps(facets["landed"]), "the landed facet names no volume path" + + +@pytest.mark.parametrize( + "location", + [None, {"path": "./bronze/subs.parquet"}, {"path": "dbfs:/mnt/bronze/subs/"}], +) +def test_a_databricks_binding_never_writes_the_local_fallback(fake, tmp_path, location): + """It used to write ``location.path``, or ``out/.parquet``, and report success.""" + assert _run(_contract(tmp_path, location=location), tmp_path) == 0 + assert [r["id"] for r in sorted(fake.rows(TABLE), key=lambda r: r["id"])] == ["1", "2", "3"] + assert list((tmp_path / "out").glob("*")) == [] + assert not (tmp_path / "bronze").exists() + assert not (tmp_path / "dbfs:").exists() + written = {p.name for p in tmp_path.rglob("*.parquet")} + assert written <= {"subs.parquet"} + assert all(".fluid/staging" in str(p) for p in tmp_path.rglob("*.parquet")) + + +def test_a_second_run_overwrites_the_table(fake, tmp_path): + assert _run(_contract(tmp_path), tmp_path) == 0 + again = _contract(tmp_path, rows=[("7", "+1", "00007")]) + assert _run(again, tmp_path) == 0 + assert [r["id"] for r in fake.rows(TABLE)] == ["7"] + + +def test_an_unset_staging_volume_is_refused_before_any_work(fake, tmp_path, monkeypatch, caplog): + monkeypatch.delenv(load.STAGING_VOLUME_ENV) + assert _run(_contract(tmp_path), tmp_path) != 0 + assert load.STAGING_VOLUME_ENV in caplog.text + assert not (tmp_path / ".fluid" / "staging").exists(), "refused before anything landed" + assert fake.requests == [], "and before the workspace was touched" + + +def test_an_unset_warehouse_is_refused_before_any_work(fake, tmp_path, monkeypatch, caplog): + monkeypatch.delenv("DATABRICKS_WAREHOUSE_ID") + assert _run(_contract(tmp_path), tmp_path) != 0 + assert "DATABRICKS_WAREHOUSE_ID" in caplog.text + assert fake.requests == [] + + +def test_an_unsupported_mode_is_refused_before_any_work(fake, tmp_path, caplog): + assert _run(_contract(tmp_path, mode="incremental_append"), tmp_path) != 0 + assert "full_refresh" in caplog.text + assert not (tmp_path / ".fluid" / "staging").exists() + assert fake.requests == [] + + +def test_a_missing_table_fails_the_build_naming_fluid_apply(fake, tmp_path, caplog): + del fake.tables[TABLE] + assert _run(_contract(tmp_path), tmp_path) != 0 + assert "fluid apply" in caplog.text + assert not [p for m, p in fake.file_ops if m == "PUT"] + + +def test_a_failed_load_fails_the_build_and_leaves_the_volume_empty(fake, tmp_path, caplog): + fake.fail_on("INSERT OVERWRITE", "[PERMISSION_DENIED] no MODIFY") + assert _run(_contract(tmp_path, masking_rules=HASH), tmp_path) != 0 + assert "databricks load" in caplog.text and "PERMISSION_DENIED" in caplog.text + assert fake.volume_files() == [] + + +def test_a_local_binding_never_touches_databricks(fake, tmp_path): + contract = _contract(tmp_path) + out = tmp_path / "out" / "subs.parquet" + contract["exposes"][0]["binding"] = { + "platform": "local", + "format": "parquet", + "location": {"path": str(out)}, + } + assert _run(contract, tmp_path) == 0 + assert out.exists() + assert fake.requests == [] + + +def test_each_build_loads_its_own_table(fake, tmp_path): + """The target is the expose the build's ``outputs`` names, not ``exposes[0]``.""" + fake.create_table("main.bronze.other", COLUMNS) + contract = _contract(tmp_path) + decoy = _expose(table="other") + decoy["exposeId"] = "other" + contract["exposes"].insert(0, decoy) + assert _run(contract, tmp_path) == 0 + assert fake.rows("main.bronze.other") == [] + assert len(fake.rows(TABLE)) == 3 diff --git a/tests/cli/test_verify_databricks.py b/tests/cli/test_verify_databricks.py index efd876f4e..af46522aa 100644 --- a/tests/cli/test_verify_databricks.py +++ b/tests/cli/test_verify_databricks.py @@ -137,10 +137,10 @@ def __init__(self) -> None: self.api_client = StubApiClient(self) self.calls: List[Any] = [] - def add_jobs(self, name: str, count: int) -> None: + def add_jobs(self, name: str, count: int, **settings: Any) -> None: for _ in range(count): job_id = len(self.jobs) + 1 - self.jobs.append({"job_id": job_id, "settings": {"name": name}}) + self.jobs.append({"job_id": job_id, "settings": {"name": name, **settings}}) def answer(self, method: str, path: str, query: Dict[str, Any]) -> Dict[str, Any]: assert method == "GET" @@ -1058,3 +1058,50 @@ def test_the_real_sdk_reads_the_stand_in_workspace(tmp_path, monkeypatch): assert result["status"] == "mismatch", result assert [f["field"] for f in result["dimensions"]["structure"]["missing_fields"]] == ["region"] assert [f["principal"] for f in result["dimensions"]["catalog_access"]["findings"]] == ["bi"] + + +# ── A job this module paused because its table is not protected ───────── + + +def restricted_contract_with_job() -> Dict[str, Any]: + document = restricted_contract() + document["builds"] = contract_with_job()["builds"] + return document + + +def test_a_job_paused_while_its_table_is_not_protected_is_a_finding(workspace): + contract = restricted_contract_with_job() + workspace.add_jobs( + job_name(), 1, schedule={"quartz_cron_expression": "0 0 2 * * ?", "pause_status": "PAUSED"} + ) + + result = verify(workspace, contract) + + dimension = result["dimensions"]["jobs"] + assert dimension["status"] == "fail" + [finding] = dimension["findings"] + assert finding["build"] == "dbt_daily" + assert "is paused" in finding["message"] and "not protected" in finding["message"] + assert "run fluid apply" in finding["message"] + assert result["severity"]["level"] == "CRITICAL" + + +def test_a_paused_job_on_a_protected_table_is_not_a_finding(workspace): + contract = restricted_contract_with_job() + bind_all(workspace, contract) + workspace.add_jobs( + job_name(), 1, schedule={"quartz_cron_expression": "0 0 2 * * ?", "pause_status": "PAUSED"} + ) + + assert verify(workspace, contract)["dimensions"]["jobs"]["status"] == "pass" + + +def test_an_unpaused_job_on_an_unprotected_table_has_no_paused_finding(workspace): + contract = restricted_contract_with_job() + workspace.add_jobs( + job_name(), + 1, + schedule={"quartz_cron_expression": "0 0 2 * * ?", "pause_status": "UNPAUSED"}, + ) + + assert verify(workspace, contract)["dimensions"]["jobs"]["status"] == "pass" diff --git a/tests/cli/test_verify_databricks_retention.py b/tests/cli/test_verify_databricks_retention.py new file mode 100644 index 000000000..dae62f2a5 --- /dev/null +++ b/tests/cli/test_verify_databricks_retention.py @@ -0,0 +1,263 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fluid verify``'s retention dimension for a Databricks expose with ``lifecycle.expire``. + +The contract's retention must cover the table's live Delta history (the larger of +``delta.logRetentionDuration`` and ``delta.deletedFileRetentionDuration``, 30 and 7 days when +unset); predictive optimization off is a warning. Driven through ``verify_databricks_expose`` +with the stub of ``test_verify_databricks.py``, and once over HTTP against the stand-in workspace. +""" + +from __future__ import annotations + +import copy +from typing import Any, Dict, Optional + +import pytest + +from fluid_build.cli._verify_databricks import verify_databricks_expose +from tests.cli.test_verify_databricks import ( + CONTRACT, + LIVE_COLUMNS, + TABLE, + StubWorkspace, +) + +pytestmark = pytest.mark.unit + +DAY = 86400 + + +class RetentionWorkspace(StubWorkspace): + """The stub, with Delta properties and a predictive optimization flag on GetTable.""" + + def __init__(self) -> None: + super().__init__() + self.extras: Dict[str, Any] = {} + #: GetTable serves a managed Delta table's properties (``delta.*``) with it. + self.serves_properties = True + + def answer(self, method: str, path: str, query: Dict[str, Any]) -> Dict[str, Any]: + out = super().answer(method, path, query) + if path == f"/api/2.1/unity-catalog/tables/{TABLE}": + served = {"properties": {"delta.minReaderVersion": "3"}} + out = {**(served if self.serves_properties else {}), **out} + out = {**out, **copy.deepcopy(self.extras)} + return out + + +def _contract(retention: Optional[str] = "P365D", expire: bool = True) -> Dict[str, Any]: + contract = copy.deepcopy(CONTRACT) + if retention is not None: + contract["exposes"][0]["lifecycle"] = {"retention": retention, "expire": expire} + contract["builds"] = [ + { + "id": "dbt_daily", + "engine": "dbt", + "repository": "./dbt", + "properties": {"model": "orders"}, + "outputs": ["orders"], + "execution": {"runtime": {"platform": "databricks"}}, + } + ] + return contract + + +def _verify(workspace: Any, contract: Dict[str, Any]) -> Dict[str, Any]: + return verify_databricks_expose( + "orders", + contract["exposes"][0], + contract=contract, + client_factory=lambda: workspace, + ) + + +@pytest.fixture +def workspace() -> RetentionWorkspace: + ws = RetentionWorkspace() + ws.add_jobs("fluid dbx.fake.orders dbt_daily", 1) + return ws + + +def _dimension(result: Dict[str, Any]) -> Dict[str, Any]: + assert "retention" in result["dimensions"], result + return result["dimensions"]["retention"] + + +def test_no_expire_skips(workspace): + result = _verify(workspace, _contract(None)) + assert _dimension(result)["status"] == "skip" + assert result["status"] == "match" + + +def test_expire_false_skips(workspace): + assert _dimension(_verify(workspace, _contract("P365D", expire=False)))["status"] == "skip" + + +def test_delta_defaults_inside_the_retention_pass(workspace): + workspace.extras = {"effective_predictive_optimization_flag": {"value": "ENABLE"}} + result = _verify(workspace, _contract()) + dim = _dimension(result) + assert dim["status"] == "pass", dim + assert dim["history_seconds"] == 30 * DAY and dim["retention"] == "P365D" + assert dim["predictive_optimization"] == "ENABLE" and dim["warnings"] == [] + assert result["status"] == "match" and result["severity"]["level"] == "SUCCESS" + + +def test_a_history_exactly_the_retention_passes(workspace): + assert _dimension(_verify(workspace, _contract("P30D")))["status"] == "pass" + + +def test_a_history_longer_than_the_retention_is_critical_with_the_fix(workspace): + workspace.extras = {"properties": {"delta.logRetentionDuration": "interval 400 days"}} + result = _verify(workspace, _contract("P365D")) + dim = _dimension(result) + assert dim["status"] == "fail" and "400 days" in dim["message"] + assert "ALTER TABLE `main`.`sales`.`orders` SET TBLPROPERTIES" in dim["fix"] + assert "'delta.logRetentionDuration' = 'interval 365 days'" in dim["fix"] + assert result["status"] == "mismatch" and result["severity"]["level"] == "CRITICAL" + assert any("ALTER TABLE" in action for action in result["severity"]["actions"]) + + +def test_the_deleted_file_retention_counts(workspace): + workspace.extras = {"properties": {"delta.deletedFileRetentionDuration": "interval 500 days"}} + assert _dimension(_verify(workspace, _contract("P365D")))["status"] == "fail" + + +def test_a_history_property_of_the_runtime_wins(workspace): + workspace.extras = { + "properties": {"delta.logRetentionDuration": "interval 10 days"}, + "delta_runtime_properties_kvpairs": { + "delta_runtime_properties": {"delta.logRetentionDuration": "interval 400 days"} + }, + } + assert _dimension(_verify(workspace, _contract("P365D")))["status"] == "fail" + workspace.extras = { + "delta_runtime_properties_kvpairs": { + "delta_runtime_properties": {"delta.logRetentionDuration": "interval 60 days"} + }, + } + assert _dimension(_verify(workspace, _contract("P365D")))["status"] == "pass" + + +def test_an_unreadable_property_fails(workspace): + workspace.extras = {"properties": {"delta.logRetentionDuration": "forever"}} + dim = _dimension(_verify(workspace, _contract())) + assert dim["status"] == "fail" and "forever" in dim["message"] + + +def test_predictive_optimization_off_is_a_warning_not_a_failure(workspace): + workspace.extras = {"effective_predictive_optimization_flag": {"value": "DISABLE"}} + result = _verify(workspace, _contract()) + dim = _dimension(result) + assert dim["status"] == "warn" and dim["predictive_optimization"] == "DISABLE" + (warning,) = dim["warnings"] + assert "ALTER SCHEMA `main`.`sales` ENABLE PREDICTIVE OPTIMIZATION" in warning + assert "VACUUM `main`.`sales`.`orders`" in warning + assert result["status"] == "match" and result["severity"]["level"] == "SUCCESS" + + +def test_the_tables_own_flag_is_read_when_no_effective_one_is_served(workspace): + workspace.extras = {"enable_predictive_optimization": "DISABLE"} + assert _dimension(_verify(workspace, _contract()))["status"] == "warn" + + +@pytest.mark.parametrize( + "extras", + [ + {}, + {"enable_predictive_optimization": "INHERIT"}, + {"effective_predictive_optimization_flag": {}}, + ], +) +def test_a_flag_that_is_not_reported_is_not_a_warning(workspace, extras): + workspace.extras = extras + dim = _dimension(_verify(workspace, _contract())) + assert dim["status"] == "pass" and dim["predictive_optimization"] is None + + +def test_a_failure_keeps_the_warning(workspace): + workspace.extras = { + "properties": {"delta.logRetentionDuration": "interval 400 days"}, + "effective_predictive_optimization_flag": {"value": "DISABLE"}, + } + dim = _dimension(_verify(workspace, _contract("P60D"))) + assert dim["status"] == "fail" and len(dim["warnings"]) == 1 + + +def test_the_other_dimensions_are_untouched(workspace): + workspace.tables[TABLE] = copy.deepcopy(LIVE_COLUMNS) + result = _verify(workspace, _contract()) + assert result["dimensions"]["structure"]["status"] == "pass" + assert result["dimensions"]["jobs"]["status"] == "pass" + + +def test_a_table_the_module_refuses_to_expire_is_not_verified(workspace): + contract = _contract() + contract["builds"][0]["repository"] = "https://github.com/o/r?tag=v1" + result = _verify(workspace, contract) + assert result["status"] == "unsupported" + + +# ── over HTTP, against the stand-in workspace ─────────────────────────────── +def test_the_stand_in_workspace_serves_the_properties_verify_reads(tmp_path): + from tests.iac._fake_databricks import FakeDatabricks + + class Workspace(FakeDatabricks): + def _table_doc(self, full: str) -> Dict[str, Any]: + return { + **super()._table_doc(full), + "effective_predictive_optimization_flag": {"value": "ENABLE"}, + } + + fake = Workspace().start() + try: + fake.create_table( + TABLE, + [("id", "bigint", False), ("amount", "double", True), ("region", "string", True)], + properties={"delta.logRetentionDuration": "interval 400 days"}, + ) + fake.grant_catalog("main", "analysts", "USE_CATALOG") + fake.grant_catalog("main", "bi", "USE_CATALOG") + failing = _verify(fake.client(), _contract("P365D")) + fake.tables[TABLE]["properties"] = {"delta.logRetentionDuration": "interval 90 days"} + passing = _verify(fake.client(), _contract("P365D")) + finally: + fake.stop() + assert _dimension(failing)["status"] == "fail" + assert _dimension(passing)["status"] == "pass" + assert _dimension(passing)["predictive_optimization"] == "ENABLE" + + +def test_a_table_served_without_properties_is_a_warning_that_says_the_defaults_were_assumed( + workspace, +): + workspace.serves_properties = False + workspace.extras = {"effective_predictive_optimization_flag": {"value": "ENABLE"}} + dim = _dimension(_verify(workspace, _contract())) + assert dim["status"] == "warn", dim + assert "no table properties" in dim["warnings"][0] + assert "SHOW TBLPROPERTIES" in dim["fix"] + + +def test_verify_reads_the_contract_as_apply_does_with_its_placeholders_substituted( + workspace, monkeypatch +): + monkeypatch.setenv("FLUID_TEST_RETENTION", "P365D") + contract = _contract("{{ env.FLUID_TEST_RETENTION }}") + workspace.extras = {"effective_predictive_optimization_flag": {"value": "ENABLE"}} + dim = _dimension(_verify(workspace, contract)) + assert dim["status"] == "pass", dim + assert dim["retention"] == "P365D" diff --git a/tests/iac/_fake_databricks.py b/tests/iac/_fake_databricks.py index 661c36440..3de8eb4a5 100644 --- a/tests/iac/_fake_databricks.py +++ b/tests/iac/_fake_databricks.py @@ -49,8 +49,17 @@ so a test asserts on exactly what the provider submitted; ``add_job`` / ``add_workspace_file`` seed brownfield objects. +It also serves the Files API on Unity Catalog volumes (``add_volume``): PUT / DELETE of a file, +PUT / GET / DELETE of a directory, backed by a temporary directory so the bytes a client uploads +are really on disk (``volume_files`` lists them, ``file_ops`` orders the calls). Two data +statements run over those files: ``INSERT OVERWRITE
SELECT FROM +read_files('', format => 'parquet')`` reads the Parquet with DuckDB and REPLACES the table's +rows (``rows``; the table's definition, row filter and masks are left as they are, as the real +statement leaves them), reporting ``num_affected_rows`` / ``num_inserted_rows`` as Databricks does +unless ``insert_reports_count`` is False, and ``SELECT COUNT(*) FROM
``. + Not proven here: what real Databricks accepts (SQL dialect, warehouse state, the privilege -model, Unity Catalog storage), whether a job's settings are valid (the fake stores any), job +model, Unity Catalog storage, ``read_files`` itself), whether a job's settings are valid (the fake stores any), job runs, and any workspace-level setting. The provider is pointed at it with ``host`` and a dummy ``token``. Standalone: @@ -62,10 +71,14 @@ import base64 import copy import json +import os import re +import shutil +import tempfile import threading import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple from urllib.parse import parse_qs, unquote, urlparse @@ -300,6 +313,7 @@ def _execute(fake: "FakeDatabricks", sql: str) -> Tuple[bool, str]: if col["name"] in kept: col["mask"] = kept[col["name"]] fake.tables[full] = new + fake.table_rows.pop(full, None) # a new table holds no rows return True, "" m = re.match( @@ -353,6 +367,17 @@ def _execute(fake: "FakeDatabricks", sql: str) -> Tuple[bool, str]: held[who] = [p for p in held.get(who, []) if p != gone] return True, "" + m = re.match(rf"DROP\s+FUNCTION\s+(IF\s+EXISTS\s+)?{_FULL}\s*$", s, re.I) + if m: + ie, cat, sch, name = m.groups() + full = f"{cat}.{sch}.{name}" + if full not in fake.functions: + return (True, "") if ie else (False, f"[ROUTINE_NOT_FOUND] {full}") + if fake.function_in_use(full): + return False, f"[FUNCTION_IN_USE] {full} is used by a row filter or a column mask" + del fake.functions[full] + return True, "" + m = re.match(rf"DROP\s+TABLE\s+(IF\s+EXISTS\s+)?{_FULL}\s*$", s, re.I) if m: ie, cat, sch, name = m.groups() @@ -360,6 +385,7 @@ def _execute(fake: "FakeDatabricks", sql: str) -> Tuple[bool, str]: if full not in fake.tables: return (True, "") if ie else (False, f"[TABLE_OR_VIEW_NOT_FOUND] {full}") del fake.tables[full] + fake.table_rows.pop(full, None) fake.grants.pop(("table", full), None) return True, "" @@ -543,6 +569,127 @@ def find(col: str) -> int: return False, f"fake_databricks: unsupported statement: {s[:200]}" +def _count_doc(n: int) -> Dict[str, Any]: + """The result of a one-column query: a row of strings, as the Statement Execution API sends.""" + return { + "manifest": { + "format": "JSON_ARRAY", + "schema": { + "column_count": 1, + "columns": [{"name": "count(1)", "type_name": "LONG", "position": 0}], + }, + "total_row_count": 1, + }, + "result": {"row_count": 1, "data_array": [[str(n)]]}, + } + + +def _insert_doc(n: int, report: bool) -> Dict[str, Any]: + """What ``INSERT`` answers: ``num_affected_rows`` and ``num_inserted_rows`` (Databricks SQL).""" + if not report: + return {} + names = ("num_affected_rows", "num_inserted_rows") + return { + "manifest": { + "format": "JSON_ARRAY", + "schema": { + "column_count": 2, + "columns": [ + {"name": name, "type_name": "LONG", "position": i} + for i, name in enumerate(names) + ], + }, + "total_row_count": 1, + }, + "result": {"row_count": 1, "data_array": [[str(n), str(n)]]}, + } + + +def _data_statement(fake: "FakeDatabricks", sql: str) -> Optional[Tuple[bool, str, Dict[str, Any]]]: + """``(ok, error, result)`` for a statement that reads or writes rows, ``None`` for any other. + + ``INSERT OVERWRITE [()] SELECT FROM read_files('', format => + 'parquet')`` replaces the rows of ```` with the Parquet file at ````, or every one + under it when it is a directory (a path on a volume the Files API holds). With a column list + each selected column goes to the target column it names (a table column it leaves out is + NULL); without one, by position in table order, as Databricks does. + """ + s = sql.strip().rstrip(";").strip() + m = re.match( + rf"INSERT\s+OVERWRITE\s+(?:TABLE\s+)?{_FULL}\s+(?:\(([^)]*)\)\s+)?SELECT\s+(.+?)\s+" + rf"FROM\s+read_files\s*\(\s*{_QUOTED}\s*,\s*format\s*=>\s*'parquet'\s*\)\s*$", + s, + re.S | re.I, + ) + if m: + cat, sch, name, targets, select, literal = m.groups() + full = f"{cat}.{sch}.{name}" + table = fake.tables.get(full) + if table is None: + return ( + False, + f"[TABLE_OR_VIEW_NOT_FOUND] The table or view `{cat}`.`{sch}`.`{name}` cannot " + "be found.", + {}, + ) + wanted = _names(select) + names = [c["name"] for c in table["columns"]] + into = _names(targets) if targets is not None else names + unknown = [c for c in into if c.lower() not in {n.lower() for n in names}] + if unknown: + return ( + False, + f"[UNRESOLVED_COLUMN] Cannot write to {full}: no column {unknown[0]!r}.", + {}, + ) + if len(wanted) != len(into): + return ( + False, + f"[INSERT_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS] Cannot write to {full}: " + f"the insert names {len(into)} columns, the query {len(wanted)}.", + {}, + ) + source = fake._fs_path(re.sub(r"\\(.)", r"\1", literal)) + if source is not None and os.path.isfile(source): + files = [source] + elif source is not None and os.path.isdir(source): + files = sorted(str(p) for p in Path(source).rglob("*.parquet")) + else: + files = [] + if not files: + return ( + False, + f"[UNABLE_TO_INFER_SCHEMA] read_files found no Parquet file at {literal!r}.", + {}, + ) + import duckdb + + con = duckdb.connect(":memory:") + try: + quoted = ", ".join('"' + c.replace('"', '""') + '"' for c in wanted) + fetched = con.execute(f"SELECT {quoted} FROM read_parquet(?)", [files]).fetchall() + except Exception as exc: # noqa: BLE001 — what the file lacks is the statement's error + return False, f"[UNRESOLVED_COLUMN] {type(exc).__name__}: {exc}", {} + finally: + con.close() + by_lower = {n.lower(): n for n in names} + fake.table_rows[full] = [ + { + **{n: None for n in names}, + **{by_lower[t.lower()]: v for t, v in zip(into, row, strict=True)}, + } + for row in fetched + ] + return True, "", _insert_doc(len(fetched), fake.insert_reports_count) + m = re.match(rf"SELECT\s+COUNT\(\s*(?:\*|1)\s*\)\s+FROM\s+{_FULL}\s*$", s, re.I) + if m: + full = ".".join(m.groups()) + if full not in fake.tables: + return False, f"[TABLE_OR_VIEW_NOT_FOUND] The table or view {full} cannot be found.", {} + return True, "", _count_doc(len(fake.table_rows.get(full, []))) + return None + + # ---------------------------------------------------------------------- a client class _HttpError(Exception): """What the SDK raises for an error status; ``status_code`` is what forge-cli reads.""" @@ -559,15 +706,29 @@ class _ApiClient: def __init__(self, endpoint: str): self._endpoint = endpoint - def do(self, method: str, path: str, query: Any = None, body: Any = None, **_: Any) -> Any: + def do( + self, + method: str, + path: str, + query: Any = None, + body: Any = None, + headers: Any = None, + data: Any = None, + **_: Any, + ) -> Any: import urllib.error import urllib.parse import urllib.request url = self._endpoint + path + ("?" + urllib.parse.urlencode(query) if query else "") - data = json.dumps(body).encode() if body is not None else None - request = urllib.request.Request(url, data=data, method=method) + if data is not None: # a stream or bytes, sent as the SDK sends a file upload + payload = data if isinstance(data, (bytes, bytearray)) else data.read() + else: + payload = json.dumps(body).encode() if body is not None else None + request = urllib.request.Request(url, data=payload, method=method) request.add_header("Content-Type", "application/json") + for name, value in (headers or {}).items(): + request.add_header(name, value) try: with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 raw = response.read() @@ -625,6 +786,16 @@ def __init__( # (method, path substring, status, message): matching requests are answered with an error self.failing_routes: List[Tuple[str, str, int, str]] = [] self.replaced: List[str] = [] # tables a CREATE OR REPLACE hit while they existed + # Unity Catalog volumes the Files API serves ("cat.schema.volume"), their bytes in a + # temporary directory, and every call against them in order: (method, path) + self.volumes: set = set() + self.files_root = tempfile.mkdtemp(prefix="fake-databricks-files-") + self.file_ops: List[Tuple[str, str]] = [] + # full table name -> rows (dicts by column name, table order) the last INSERT OVERWRITE + # wrote; a table nothing was loaded into has none + self.table_rows: Dict[str, List[Dict[str, Any]]] = {} + # False: an INSERT answers with no row count, as a statement that returns none would + self.insert_reports_count = True self._marker = 0 self._job_seq = 0 self._object_ids: Dict[str, int] = {} # workspace path -> object_id @@ -648,14 +819,16 @@ def _handle(self, method: str) -> None: n = int(self.headers.get("Content-Length") or 0) raw = self.rfile.read(n) if n else b"" ctype = self.headers.get("Content-Type") or "" + path = unquote(parsed.path) try: - if ctype.startswith("multipart/form-data"): + if path.startswith("/api/2.0/fs/files"): + body = {"_raw": raw} # an octet stream, not JSON + elif ctype.startswith("multipart/form-data"): body = _multipart(ctype, raw) else: body = json.loads(raw) if raw else {} except ValueError: body = {} - path = unquote(parsed.path) with fake._lock: fake.requests.append((method, path)) fake.queries.append((method, path, parsed.query)) @@ -694,6 +867,7 @@ def stop(self) -> None: if self._server is not None: self._server.shutdown() self._server.server_close() + shutil.rmtree(self.files_root, ignore_errors=True) @property def endpoint(self) -> str: @@ -849,6 +1023,8 @@ def _register(self) -> None: a("PATCH", r"/api/2\.1/unity-catalog/schemas/([^/]+)", self._patch_schema) a("DELETE", r"/api/2\.1/unity-catalog/schemas/([^/]+)", self._delete_schema) a("GET", r"/api/2\.1/unity-catalog/functions/([^/]+)", self._get_function) + a("GET", r"/api/2\.1/unity-catalog/functions", self._list_functions) + a("GET", r"/api/2\.1/unity-catalog/tables", self._list_tables) a("GET", r"/api/2\.1/unity-catalog/tables/([^/]+)", self._get_table) a("DELETE", r"/api/2\.1/unity-catalog/tables/([^/]+)", self._delete_table) a("GET", r"/api/2\.1/unity-catalog/permissions/([a-z_]+)/([^/]+)", self._get_perms) @@ -886,6 +1062,12 @@ def _register(self) -> None: a("POST", r"/api/2\.0/workspace/import", self._ws_import) a("GET", r"/api/2\.0/workspace/get-status", self._ws_status) a("POST", r"/api/2\.0/workspace/delete", self._ws_delete) + # ---- files (Unity Catalog volumes) + a("PUT", r"/api/2\.0/fs/directories(/.*)", self._fs_mkdir) + a("GET", r"/api/2\.0/fs/directories(/.*)", self._fs_list) + a("DELETE", r"/api/2\.0/fs/directories(/.*)", self._fs_rmdir) + a("PUT", r"/api/2\.0/fs/files(/.*)", self._fs_put) + a("DELETE", r"/api/2\.0/fs/files(/.*)", self._fs_delete) # ---- sql a("POST", r"/api/2\.0/sql/statements/?", self._exec_sql) a("GET", r"/api/2\.0/sql/statements/([^/]+)", self._get_stmt) @@ -1041,6 +1223,41 @@ def _get_function(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: }, } + def _list_functions(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + """The functions of one schema, a page at a time (``list_page_size``).""" + cat = (q.get("catalog_name") or [""])[0] + sch = (q.get("schema_name") or [""])[0] + names = sorted(n for n in self.functions if n.rsplit(".", 1)[0] == f"{cat}.{sch}") + return 200, self._page(names, q, "functions", self._function_summary) + + def _function_summary(self, full: str) -> Dict[str, Any]: + cat, sch, name = full.split(".") + return { + "name": name, + "catalog_name": cat, + "schema_name": sch, + "full_name": full, + "owner": self.functions[full].get("owner"), + } + + def _list_tables(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + cat = (q.get("catalog_name") or [""])[0] + sch = (q.get("schema_name") or [""])[0] + names = sorted(n for n in self.tables if n.rsplit(".", 1)[0] == f"{cat}.{sch}") + return 200, self._page( + names, q, "tables", lambda n: {"name": n.rsplit(".", 1)[1], "full_name": n} + ) + + def _page( + self, names: List[str], q: Any, key: str, row: Callable[[str], Any] + ) -> Dict[str, Any]: + size = int(getattr(self, "list_page_size", 100)) + start = int((q.get("page_token") or ["0"])[0]) + out: Dict[str, Any] = {key: [row(n) for n in names[start : start + size]]} + if start + size < len(names): + out["next_page_token"] = str(start + size) + return out + # -- tables def _get_table(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: full = m.group(1) @@ -1087,7 +1304,7 @@ def _table_doc(self, full: str) -> Dict[str, Any]: "data_source_format": "DELTA", "table_id": f"tid-{full}", "metastore_id": "ms-fake", - "owner": _USER, + "owner": t.get("owner", _USER), "columns": cols, "properties": t["properties"], "comment": t.get("comment"), @@ -1100,6 +1317,7 @@ def _delete_table(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: if full not in self.tables: return _err(404, "TABLE_DOES_NOT_EXIST", f"Table '{full}' does not exist.") del self.tables[full] + self.table_rows.pop(full, None) self.grants.pop(("table", full), None) self._drop_tags("tables", full) return 200, {} @@ -1341,6 +1559,115 @@ def _ws_delete(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: self.workspace_dirs.discard(path) return 200, {} + # -- files on volumes (the Files API) + def add_volume(self, full_name: str) -> None: + """Make ``/Volumes///`` exist; creates the catalog and schema.""" + cat, sch, _volume = full_name.split(".") + with self._lock: + self._ensure_schema(cat, sch) + self.volumes.add(full_name) + os.makedirs(self._fs_path("/Volumes/" + full_name.replace(".", "/")), exist_ok=True) + + def _fs_path(self, volume_path: str) -> Optional[str]: + """The temporary-directory path of a ``/Volumes/...`` path; ``None`` outside a volume + the fake holds, or on a path that climbs out of it.""" + parts = volume_path.split("/") + if parts[:2] != ["", "Volumes"] or len(parts) < 5: + return None + if ".." in parts or "." in parts[2:]: + return None + if ".".join(parts[2:5]) not in self.volumes: + return None + return os.path.join(self.files_root, *[p for p in parts[1:] if p]) + + def volume_files(self, prefix: str = "/Volumes") -> List[str]: + """Every file the Files API holds under ``prefix``, as ``/Volumes/...`` paths.""" + found: List[str] = [] + for root, _dirs, names in os.walk(self.files_root): + for name in names: + path = "/" + os.path.relpath(os.path.join(root, name), self.files_root) + if path.startswith(prefix.rstrip("/") + "/") or path == prefix: + found.append(path.replace(os.sep, "/")) + return sorted(found) + + def file_bytes(self, volume_path: str) -> bytes: + path = self._fs_path(volume_path) + assert path is not None, volume_path + with open(path, "rb") as fh: + return fh.read() + + def rows(self, full_name: str) -> List[Dict[str, Any]]: + """The rows the last INSERT OVERWRITE left in the table.""" + return list(self.table_rows.get(full_name, [])) + + def _fs_target(self, method: str, m: Any) -> Tuple[Optional[str], Optional[Tuple[int, Any]]]: + path = m.group(1) + self.file_ops.append((method, path)) + real = self._fs_path(path) + if real is None: + return None, _err(404, NOT_FOUND, f"The path {path} is not in a volume that exists.") + return real, None + + def _fs_mkdir(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + real, failed = self._fs_target("PUT-DIR", m) + if failed: + return failed + os.makedirs(real, exist_ok=True) + return 204, None + + def _fs_put(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + real, failed = self._fs_target("PUT", m) + if failed: + return failed + overwrite = (q.get("overwrite") or ["true"])[0].lower() != "false" + if os.path.exists(real) and not overwrite: + return _err(409, "ALREADY_EXISTS", f"The file {m.group(1)} already exists.") + os.makedirs(os.path.dirname(real), exist_ok=True) + with open(real, "wb") as fh: + fh.write(b.get("_raw") or b"") + return 204, None + + def _fs_delete(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + real, failed = self._fs_target("DELETE", m) + if failed: + return failed + if not os.path.isfile(real): + return _err(404, NOT_FOUND, f"The file {m.group(1)} does not exist.") + os.remove(real) + return 204, None + + def _fs_rmdir(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + real, failed = self._fs_target("DELETE-DIR", m) + if failed: + return failed + if not os.path.isdir(real): + return _err(404, NOT_FOUND, f"The directory {m.group(1)} does not exist.") + if os.listdir(real): + return _err(400, "FAILED_PRECONDITION", f"The directory {m.group(1)} is not empty.") + os.rmdir(real) + return 204, None + + def _fs_list(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + real, failed = self._fs_target("LIST", m) + if failed: + return failed + if not os.path.isdir(real): + return _err(404, NOT_FOUND, f"The directory {m.group(1)} does not exist.") + base = m.group(1).rstrip("/") + contents = [] + for name in sorted(os.listdir(real)): + full = os.path.join(real, name) + entry: Dict[str, Any] = { + "name": name, + "path": f"{base}/{name}", + "is_directory": os.path.isdir(full), + } + if os.path.isfile(full): + entry["file_size"] = os.path.getsize(full) + entry["last_modified"] = int(os.path.getmtime(full) * 1000) + contents.append(entry) + return 200, {"contents": contents} + # -- warehouse / sql def _get_wh(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: wid = m.group(1) @@ -1361,7 +1688,14 @@ def _exec_sql(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: self.sql_log.append(sql) self.timeline.append(("sql", sql)) injected = next((msg for sub, msg in self.failing if sub in sql), None) - ok, err = (False, injected) if injected is not None else _execute(self, sql) + data = None if injected is not None else _data_statement(self, sql) + if injected is not None: + ok, err, answer = False, injected, {} + elif data is not None: + ok, err, answer = data + else: + ok, err = _execute(self, sql) + answer = {} sid = f"stmt-{len(self.statements) + 1}" if ok: doc: Dict[str, Any] = { @@ -1373,6 +1707,7 @@ def _exec_sql(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: "total_row_count": 0, }, "result": {"row_count": 0}, + **answer, } else: doc = { diff --git a/tests/iac/test_iac_databricks.py b/tests/iac/test_iac_databricks.py index 94d462496..72145dd0d 100644 --- a/tests/iac/test_iac_databricks.py +++ b/tests/iac/test_iac_databricks.py @@ -694,7 +694,7 @@ def test_unset_warehouse_is_refused(monkeypatch): {"principal": "group:x", "permissions": ["read"], "conditions": {"ip": "1"}}, "conditions", ), - ({"principal": "group:x", "permissions": ["read"], "resources": ["$.exposes"]}, "scope"), + ({"principal": "group:x", "permissions": ["read"], "resources": ["$.exposes[0]"]}, "scope"), ({"principal": "user:bob@corp.example", "permissions": ["read"]}, "principal-placeholder"), ({"principal": "serviceAccount:sa@p.iam", "permissions": ["read"]}, "principal-unmapped"), ], @@ -1048,6 +1048,58 @@ def test_an_unreachable_workspace(self): refusal = self.plugin.explain_plan_failure(out, _contract(), []) assert refusal.kind == "databricks-workspace-unreachable" + RESIDENCY = ( + "The contract declares binding.location.region eu-central-1, but the workspace " + "DATABRICKS_HOST names is in us-west-2. Residency is checked against the workspace's " + "metastore. Point DATABRICKS_HOST at a workspace in eu-central-1, or correct " + "binding.location.region." + ) + + def _residency_json(self, padding=2000): + diagnostic = { + "severity": "error", + "summary": "Resource precondition failed", + "detail": self.RESIDENCY, + "range": {"filename": "main.tf.json"}, + "snippet": { + "code": "x" * padding, + "values": [ + { + "traversal": "data.databricks_current_metastore.m.metastore_info[0].region", + "statement": 'is "us-west-2"', + } + ], + }, + } + return json.dumps({"@level": "error", "diagnostic": diagnostic}) + + def test_a_residency_failure_is_one_clean_line_with_both_regions_and_the_fix(self): + out = self._residency_json() + assert len(out) > 1600, "the engine's tail would cut the sentence" + refusal = self.plugin.explain_plan_failure(out, _contract(), []) + text = str(refusal) + assert refusal.kind == "databricks-residency-region" + assert "\n" not in text and not refusal.remediation + assert "eu-central-1" in text and "us-west-2" in text + assert "DATABRICKS_HOST" in text and "binding.location.region" in text + + def test_a_residency_failure_in_the_wrapped_human_output(self): + out = "Error: Resource precondition failed\n\n on main.tf.json line 3:\n\n" + "\n".join( + " " + line for line in __import__("textwrap").wrap(self.RESIDENCY, 60) + ) + text = str(self.plugin.explain_plan_failure(out, _contract(), [])) + assert "eu-central-1" in text and "us-west-2" in text and "\n" not in text + + def test_the_live_region_comes_from_the_diagnostic_values_when_the_text_lacks_it(self): + diagnostic = { + "detail": "The contract declares binding.location.region eu-central-1, but the " + "workspace DATABRICKS_HOST names is in (unknown).", + "snippet": {"values": [{"statement": 'is "us-west-2"'}]}, + } + out = json.dumps({"diagnostic": diagnostic}) + text = str(self.plugin.explain_plan_failure(out, _contract(), [])) + assert "us-west-2" in text and "eu-central-1" in text + def test_anything_else_is_left_alone(self): assert self.plugin.explain_plan_failure("Error: 503", _contract(), []) is None @@ -1273,3 +1325,13 @@ def test_the_native_simple_apply_never_takes_the_platform(): _simple_apply_provider(contract, _LOG) assert exc.value.event == "apply_platform_unsupported" assert "exposes[0].binding.platform: databricks" in exc.value.context["error"] + + +@pytest.mark.parametrize( + "value", [{1: "a", "b": "c"}, {__import__("datetime").date(2024, 1, 1): "x"}] +) +def test_a_contract_json_cannot_key_reuses_no_pause_decision(value): + contract = _contract() + contract["metadata"] = {"free_form": value} + assert dbx._contract_key(contract) is None + IAC_PLUGINS["databricks"].emit(contract) # no TypeError from the key diff --git a/tests/iac/test_iac_databricks_enforce.py b/tests/iac/test_iac_databricks_enforce.py index 280e79e4e..9e6a805e7 100644 --- a/tests/iac/test_iac_databricks_enforce.py +++ b/tests/iac/test_iac_databricks_enforce.py @@ -175,7 +175,7 @@ def test_a_second_run_executes_nothing(fake): assert again.said == [] -def test_a_changed_predicate_binds_a_new_function_and_keeps_the_old_one(fake): +def test_a_changed_predicate_binds_a_new_function_and_drops_the_old_one_after_the_apply(fake): seed(fake) Run(fake).pre(contract()) old = live(fake)["row_filter"]["function_name"] @@ -184,14 +184,17 @@ def test_a_changed_predicate_binds_a_new_function_and_keeps_the_old_one(fake): changed = contract(filters=[{**FILTER, "where": "consent_data_analytics = false"}]) run = Run(fake) run.pre(changed) + assert old in fake.functions, "pre_apply drops nothing" run.post(changed) new = live(fake)["row_filter"] assert new["function_name"] != old assert new["input_column_names"] == ["consent_data_analytics"] - assert old in fake.functions, "a superseded function is not dropped" + assert old not in fake.functions, "the superseded function is dropped once the apply is done" assert live_masks(fake) == masks - assert not any(s.startswith("DROP") for s in run.statements()) + assert [s for s in run.statements() if s.startswith("DROP")] == [ + f"DROP FUNCTION IF EXISTS `main`.`cp360`.`{old.rsplit('.', 1)[1]}`" + ] assert len([s for s in run.statements() if s.startswith("CREATE FUNCTION")]) == 1 @@ -1048,6 +1051,158 @@ def test_a_read_failure_before_the_apply_still_stops_the_run(fake): Run(fake).pre(contract()) +# ── orphan functions: what a changed contract leaves behind ───────────────── +OTHER_FILTER = {"principal": ANALYSTS, "where": "is_suppressed = false"} + + +def _fluid_functions(fake: FakeDatabricks) -> set: + return {n for n in fake.functions if n.rsplit(".", 1)[1].startswith("fluid_")} + + +def _changed_contract_applied(fake: FakeDatabricks): + """Apply a contract, then one whose row filter differs: the first filter is superseded.""" + seed(fake) + Run(fake).pre(contract()) + first = live(fake)["row_filter"]["function_name"] + changed = contract(filters=[OTHER_FILTER]) + return first, changed + + +def test_a_superseded_function_is_dropped_after_a_successful_apply_and_said(fake): + first, changed = _changed_contract_applied(fake) + run = Run(fake) + run.pre(changed) + assert first in fake.functions, "pre_apply keeps it until the apply has succeeded" + run.post(changed) + assert first not in fake.functions + now = live(fake)["row_filter"]["function_name"] + assert now in fake.functions and now != first + assert f"DROP FUNCTION IF EXISTS `main`.`cp360`.`{first.rsplit('.', 1)[1]}`" in run.statements() + assert f"dropped superseded function {first}" in run.text + assert all(f in fake.functions for f in live_masks(fake).values()) + + +def test_a_function_bound_to_any_table_in_the_schema_is_never_dropped(fake): + first, changed = _changed_contract_applied(fake) + fake.create_table("main.cp360.other", [("a", "string", True)]) + fake.tables["main.cp360.other"]["row_filter"] = { + "function_name": first, + "input_column_names": [], + } + run = Run(fake) + run.pre(changed) + run.post(changed) + assert first in fake.functions + assert "DROP FUNCTION" not in "\n".join(run.statements()) + + +def test_only_this_tables_fluid_functions_are_dropped(fake): + first, changed = _changed_contract_applied(fake) + keep = [ + "main.cp360.my_helper", # no fluid_ prefix + "main.cp360.fluid_rf_profile_x_deadbeef", # another table's name + "main.cp360.fluid_rf_profile_deadbeef_extra", # not this module's naming + "main.cp360.fluid_mask_profile_nosuchcolumn_deadbeef", # no such column on the table + ] + for name in keep: + fake.add_function(name, [("val", "STRING")], "STRING", "`val`") + run = Run(fake) + run.pre(changed) + run.post(changed) + assert first not in fake.functions + assert all(name in fake.functions for name in keep) + + +def test_an_orphan_not_owned_by_the_deploy_identity_is_left_and_named(fake): + first, changed = _changed_contract_applied(fake) + fake.functions[first]["owner"] = "someone-else@example.test" + run = Run(fake) + run.pre(changed) + run.post(changed) + assert first in fake.functions + assert "DROP FUNCTION" not in "\n".join(run.statements()) + assert f"left function {first}: owned by someone-else@example.test, not by the deploy " in ( + run.text + ) + + +def test_an_orphan_whose_owner_cannot_be_read_is_left_and_named(fake): + first, changed = _changed_contract_applied(fake) + fake.fail_route("GET", rf"/functions/{first}$", 503, "unavailable") + run = Run(fake) + run.pre(changed) + run.post(changed) + assert first in fake.functions + assert f"left function {first}: owner unknown" in run.text + + +def test_nothing_is_dropped_after_a_failed_apply(fake): + first, changed = _changed_contract_applied(fake) + run = Run(fake) + PLUGIN.post_apply( + changed, say=run.said.append, client_factory=fake.client, after_failed_apply=True + ) + assert first in fake.functions + assert "DROP FUNCTION" not in "\n".join(run.statements()) + + +def test_nothing_is_dropped_when_a_bind_fails(fake): + first, changed = _changed_contract_applied(fake) + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") + run = Run(fake) + with pytest.raises(UnsupportedBindingError): + run.post(changed) + assert first in fake.functions + assert "DROP FUNCTION" not in "\n".join(run.statements()) + + +def test_a_schema_whose_tables_cannot_all_be_read_drops_nothing_and_says_so(fake): + first, changed = _changed_contract_applied(fake) + fake.create_table("main.cp360.other", [("a", "string", True)]) + Run(fake).pre(changed) + fake.fail_route("GET", r"/tables/main\.cp360\.other$", 503, "unavailable") + run = Run(fake) + run.post(changed) + assert first in fake.functions + assert "orphan" in run.text and "not dropped" in run.text + + +def test_the_orphan_drop_follows_pages_of_functions(fake): + first, changed = _changed_contract_applied(fake) + fake.list_page_size = 1 + run = Run(fake) + run.pre(changed) + run.post(changed) + assert first not in fake.functions + + +def test_an_orphan_whose_drop_fails_is_kept_named_and_the_apply_still_succeeds(fake): + first, changed = _changed_contract_applied(fake) + fake.fail_on("DROP FUNCTION", "[PERMISSION_DENIED] cannot drop") + run = Run(fake) + run.pre(changed) + run.post(changed) + assert first in fake.functions + assert f"note: superseded function {first} was not dropped" in run.text + assert "PERMISSION_DENIED" in run.text + + +def test_the_orphan_drop_lists_the_tables_it_may_only_browse_too(fake): + first, changed = _changed_contract_applied(fake) + seen: List[Any] = [] + real = fake._list_tables + + def spy(m, q, b): + seen.append(q) + return real(m, q, b) + + fake.routes = [(mth, rx, spy if fn == real else fn) for mth, rx, fn in fake.routes] + run = Run(fake) + run.pre(changed) + run.post(changed) + assert any((q.get("include_browse") or [""])[0] == "true" for q in seen), seen + + # ── the fail-safe: a deny-all filter, then a lock ─────────────────────────── def _holders(fake: FakeDatabricks) -> Dict[str, List[str]]: return {p: v for p, v in fake.grants[("table", TABLE)].items() if v} @@ -1113,6 +1268,7 @@ def test_when_the_deny_all_filter_fails_select_is_revoked_from_every_grantee_but fake.grant("table", TABLE, "power-group", "ALL_PRIVILEGES") fake.grant("table", TABLE, "fake-user@example.test", "SELECT") # the table's owner fake.grant("table", TABLE, "modifier", "MODIFY") + fake.grant("schema", "main.cp360", "browsers", "USE_SCHEMA") # inherited, but not a read fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") run = Run(fake) with pytest.raises(UnsupportedBindingError) as exc: @@ -1129,6 +1285,57 @@ def test_when_the_deny_all_filter_fails_select_is_revoked_from_every_grantee_but assert "No grant inherited from the schema or the catalog was found open" in message +def test_the_lock_never_revokes_from_the_deploy_identity_when_the_owner_is_unreadable(fake): + seed(fake) + fake.grant("table", TABLE, DEPLOY_USER, "SELECT") # this run's own identity + fake.grant("table", TABLE, "other-group", "SELECT") + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") + fake.fail_route("GET", r"/tables/main\.cp360\.profile$", 503, "unavailable") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.post(contract()) + revokes = [s for s in fake.sql_log if s.startswith("REVOKE")] + assert not any(DEPLOY_USER in s for s in revokes), "the run must not lock itself out" + assert any("other-group" in s for s in revokes) + assert fake.grants[("table", TABLE)][DEPLOY_USER] == ["SELECT"] + assert "owner could not be read, so it could not be skipped" in str(exc.value) + + +def test_the_lock_skips_the_deploy_identity_when_it_is_not_the_owner(fake): + seed(fake) + fake.tables[TABLE]["owner"] = "someone-else@example.test" + fake.grant("table", TABLE, DEPLOY_USER, "SELECT") + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") + with pytest.raises(UnsupportedBindingError) as exc: + Run(fake).post(contract()) + assert fake.grants[("table", TABLE)][DEPLOY_USER] == ["SELECT"] + assert "could not be skipped" not in str(exc.value) + + +def test_an_owner_the_contract_restricts_is_named_as_a_path_the_lock_cannot_close(fake): + seed(fake) + fake.tables[TABLE]["owner"] = ANALYSTS_GROUP # the owner is a restricted group + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") + with pytest.raises(UnsupportedBindingError) as exc: + Run(fake).post(contract()) + message = str(exc.value) + assert "Still open:" in message + assert f"{ANALYSTS_GROUP} owns {TABLE}" in message + assert "ALTER TABLE `main`.`cp360`.`profile` OWNER TO" in message + assert "No grant inherited" not in message + + +def test_effective_permissions_with_no_inherited_field_do_not_claim_nothing_is_open(fake): + seed(fake) + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.post(contract()) + message = str(exc.value) + assert "could not determine whether a schema or catalog grant is open" in message + assert "No grant inherited" not in message + + def test_a_grant_inherited_from_the_schema_or_catalog_is_named_with_what_to_run(fake): seed(fake) fake.grant("schema", "main.cp360", "schema-readers", "SELECT") @@ -1239,6 +1446,26 @@ def test_a_job_is_unpaused_once_the_live_bindings_match(workspace_client): assert not any("is created paused" in n for n in PLUGIN.plan_notices(_job_contract())) +def test_the_pause_decision_is_made_once_and_the_notice_reuses_it(workspace_client): + fake = workspace_client + seed(fake) + Run(fake).pre(contract()) # bound: a fresh read would say "protected" + fake.fail_route("GET", r"/tables/main\.cp360\.profile$", 503, "unavailable") + assert _emitted_job(_job_contract())["schedule"]["pause_status"] == "PAUSED" + fake.failing_routes.clear() # the read error was transient + notices = "\n".join(PLUGIN.plan_notices(_job_contract())) + assert "is created paused" in notices and "main.cp360.profile" in notices + + +def test_the_notice_is_not_reused_by_a_later_run(workspace_client): + fake = workspace_client + seed(fake) + assert _emitted_job(_job_contract())["schedule"]["pause_status"] == "PAUSED" + assert "is created paused" in "\n".join(PLUGIN.plan_notices(_job_contract())) + Run(fake).pre(contract()) # a later run, with no emit before its notices + assert not any("is created paused" in n for n in PLUGIN.plan_notices(_job_contract())) + + def test_a_job_stays_paused_while_a_binding_is_missing_or_differs(workspace_client): fake = workspace_client seed(fake) diff --git a/tests/iac/test_iac_databricks_fake_e2e.py b/tests/iac/test_iac_databricks_fake_e2e.py index 35bef8651..cbee5c8ff 100644 --- a/tests/iac/test_iac_databricks_fake_e2e.py +++ b/tests/iac/test_iac_databricks_fake_e2e.py @@ -653,7 +653,7 @@ def test_a_dry_run_prints_the_statements_and_executes_none(workspace, fake, capf @requires_sdk -def test_a_changed_predicate_binds_a_new_function_and_keeps_the_old_one(workspace, fake, capfd): +def test_a_changed_predicate_binds_a_new_function_and_drops_the_old_one(workspace, fake, capfd): assert apply_contract(capfd, write_contract(workspace, _restricted()))[0] == 0 old = _live(fake)["row_filter"]["function_name"] masks = _masks(fake) @@ -661,9 +661,12 @@ def test_a_changed_predicate_binds_a_new_function_and_keeps_the_old_one(workspac rc, out = apply_contract(capfd, write_contract(workspace, _restricted(where="region = 'uk'"))) assert rc == 0, out new = _live(fake)["row_filter"]["function_name"] - assert new != old and old in fake.functions and new in fake.functions + assert new != old and old not in fake.functions and new in fake.functions assert _masks(fake) == masks - assert not any(s.startswith("DROP") for s in fake.sql_log) + assert f"dropped superseded function {old}" in out + assert [s for s in fake.sql_log if s.startswith("DROP")] == [ + f"DROP FUNCTION IF EXISTS `main`.`sales`.`{old.rsplit('.', 1)[1]}`" + ] assert not any("REPLACE FUNCTION" in s for s in fake.sql_log) diff --git a/tests/iac/test_iac_databricks_grant_scope.py b/tests/iac/test_iac_databricks_grant_scope.py new file mode 100644 index 000000000..124906110 --- /dev/null +++ b/tests/iac/test_iac_databricks_grant_scope.py @@ -0,0 +1,175 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``accessPolicy.grants[].resources`` scopes a grant to exposes on Databricks. + +The emitter evaluates the expose filters a data product writes, +``$.exposes[?(@.exposeId=='')]``, and ``$.exposes[*]`` / ``$.exposes`` for every expose: +the grant lands on exactly the tables they name. Anything it cannot evaluate, or an expose id +the contract does not have, is refused (``databricks-grant-scope-unmapped``), never widened. +""" + +from __future__ import annotations + +import copy +import json +from typing import Any, Dict, List + +import pytest + +from fluid_build.iac import IAC_PLUGINS, build_module +from fluid_build.iac.access import normalize_access_grants +from fluid_build.iac.base import UnsupportedBindingError +from fluid_build.iac.providers import databricks as dbx + +from .test_iac_databricks import _contract + +PRINCIPAL = "group:auditors" + + +@pytest.fixture(autouse=True) +def _warehouse(monkeypatch): + monkeypatch.setenv(dbx.WAREHOUSE_ENV, "wh-0123456789abcdef") + + +def _two_exposes() -> Dict[str, Any]: + contract = _contract() + refunds = copy.deepcopy(contract["exposes"][0]) + refunds["exposeId"] = "refunds" + refunds["binding"]["location"]["table"] = "refunds" + contract["exposes"].append(refunds) + return contract + + +def _scoped(contract: Dict[str, Any], *resources: str) -> Dict[str, Any]: + contract["accessPolicy"]["grants"].append( + {"principal": PRINCIPAL, "permissions": ["read"], "resources": list(resources)} + ) + return contract + + +def _tables_granted(contract: Dict[str, Any], principal: str = "auditors") -> List[str]: + """The tables (``main.sales.``) a principal is granted on.""" + module = json.loads(build_module(IAC_PLUGINS["databricks"], contract)) + tables = module["resource"]["databricks_sql_table"] + by_ref = {f"${{databricks_sql_table.{key}.id}}": t for key, t in tables.items()} + return sorted( + "main.sales." + by_ref[g["table"]]["name"] + for g in module["resource"]["databricks_grant"].values() + if "table" in g and g["principal"] == principal + ) + + +def _refusal(contract: Dict[str, Any]) -> UnsupportedBindingError: + with pytest.raises(UnsupportedBindingError) as exc: + IAC_PLUGINS["databricks"].emit(contract) + errors, _ = dbx.validate_databricks_binding(contract) + assert any(exc.value.args[0][:60] in e for e in errors), "validate refuses it too" + return exc.value + + +@pytest.mark.parametrize( + "resource", + [ + "$.exposes[?(@.exposeId=='orders')]", + '$.exposes[?(@.exposeId=="orders")]', + "$.exposes[?( @.exposeId == 'orders' )]", + " $.exposes[ ?(@.exposeId=='orders') ] ", + ], +) +def test_a_grant_scoped_to_the_one_expose_lands_on_its_table(resource): + contract = _scoped(_contract(), resource) + assert _tables_granted(contract) == ["main.sales.orders"] + assert dbx.validate_databricks_binding(contract)[0] == [] + + +def test_a_grant_scoped_to_one_of_two_exposes_lands_on_that_table_only(): + contract = _scoped(_two_exposes(), "$.exposes[?(@.exposeId=='refunds')]") + assert _tables_granted(contract) == ["main.sales.refunds"] + tables, _ = dbx.managed_objects(contract) + readers = {t.expose_id: t.readers for t in tables} + assert "auditors" in readers["refunds"] and "auditors" not in readers["orders"] + + +def test_a_grant_scoped_to_both_exposes_lands_on_both(): + contract = _scoped( + _two_exposes(), + "$.exposes[?(@.exposeId=='orders')]", + "$.exposes[?(@.exposeId=='refunds')]", + ) + assert _tables_granted(contract) == ["main.sales.orders", "main.sales.refunds"] + + +@pytest.mark.parametrize("resource", ["$.exposes[*]", "$.exposes", " $.exposes[ * ] "]) +def test_every_expose_selectors_land_on_every_table(resource): + contract = _scoped(_two_exposes(), resource) + assert _tables_granted(contract) == ["main.sales.orders", "main.sales.refunds"] + + +def test_an_unscoped_grant_still_lands_on_every_table(): + contract = _two_exposes() + contract["accessPolicy"]["grants"].append({"principal": PRINCIPAL, "permissions": ["read"]}) + assert _tables_granted(contract) == ["main.sales.orders", "main.sales.refunds"] + + +def test_an_expose_id_the_contract_does_not_have_is_refused(): + contract = _scoped(_contract(), "$.exposes[?(@.exposeId=='ordres')]") + refusal = _refusal(contract) + assert refusal.kind == "databricks-grant-scope-unmapped" + text = str(refusal) + assert "'ordres'" in text and "no expose" in text + assert "reach no" not in text + + +@pytest.mark.parametrize( + "resource", + [ + "$.exposes[0]", + "$.exposes[?(@.kind=='table')]", + "$..orders", + "$.exposes[?(@.exposeId=='orders' || @.exposeId=='refunds')]", + "$.exposes[?(@.exposeId=='orders')].contract", + "orders", + "", + " ", + ], +) +def test_a_jsonpath_the_emitter_cannot_evaluate_is_refused(resource): + refusal = _refusal(_scoped(_two_exposes(), resource)) + assert refusal.kind == "databricks-grant-scope-unmapped" + assert repr(resource) in str(refusal) + assert "reach no" not in str(refusal) + + +def test_one_unevaluable_resource_beside_a_good_one_refuses_the_grant(): + contract = _scoped(_two_exposes(), "$.exposes[?(@.exposeId=='orders')]", "$.exposes[0]") + assert _refusal(contract).kind == "databricks-grant-scope-unmapped" + + +def test_a_scope_refusal_never_emits_a_wider_grant(): + # the grant is refused, and the table privileges for it are computed for no table + contract = _scoped(_two_exposes(), "$.exposes[0]") + refusals: List[UnsupportedBindingError] = [] + compiled = dbx._compile(contract, {dbx.WAREHOUSE_ENV: "wh"}, validating=True) + refusals = compiled.refusals + assert refusals + assert not [g for g in compiled.grants if g[2] == "auditors"] + + +def test_the_grant_model_parses_the_scope_once(): + contract = _scoped(_two_exposes(), "$.exposes[?(@.exposeId=='refunds')]") + grant = next(g for g in normalize_access_grants(contract) if g.principal == "auditors") + assert grant.scope.expose_ids == frozenset({"refunds"}) + assert not grant.scope.all_exposes and grant.scope.unevaluable == () + assert grant.scope.covers("refunds") and not grant.scope.covers("orders") diff --git a/tests/iac/test_iac_databricks_landing_fake_e2e.py b/tests/iac/test_iac_databricks_landing_fake_e2e.py new file mode 100644 index 000000000..c818f5055 --- /dev/null +++ b/tests/iac/test_iac_databricks_landing_fake_e2e.py @@ -0,0 +1,207 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fluid apply`` creates the table, the acquisition build lands bronze into it, against a stand-in. + +The real ``fluid`` CLI, ``tofu`` and the real ``databricks-sdk`` run in-process; only the workspace +is ``_fake_databricks.FakeDatabricks``. A contract exposes a managed Delta table that a DuckDB +acquisition build writes from a local CSV, with ``policy.privacy.masking`` on one column, a column +restriction and a row filter: + +* ``fluid apply`` accepts the masking (the runner applies it) and creates the table, its grants, its + mask and its row filter; +* ``fluid apply --mode amend-and-build`` then hashes the column, writes Parquet locally, uploads it + to ``$DATABRICKS_STAGING_VOLUME`` through the Files API and runs ``INSERT OVERWRITE ... read_files`` + on the warehouse: the rows are in the table with the column hashed, the staged files are deleted, + and the table keeps its definition, grants, row filter and mask (no statement but the load); +* a second run overwrites the rows, it does not append; +* a plan afterwards is empty: loading data does not drift the table. + +Not proven here: what real Databricks accepts of ``read_files``, ``INSERT OVERWRITE`` over a managed +table that has a mask and a row filter, or the Files API's limits; that is the manual live tier. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Any, Dict, Iterator, List, Tuple + +import pytest + +from fluid_build.build_runners import _masking as masking +from fluid_build.iac import runner + +from ._databricks_cli import ( # noqa: F401 — plugin_cache is a fixture + apply_contract, + isolate_workspace, + make_contract, + plugin_cache, + write_contract, +) +from ._fake_databricks import FakeDatabricks + +pytestmark = [ + pytest.mark.integration, + pytest.mark.provider, + pytest.mark.databricks, + pytest.mark.skipif(runner.tofu_path() is None, reason="tofu binary not installed"), +] + +duckdb = pytest.importorskip("duckdb") +pytest.importorskip("databricks.sdk", reason="the [databricks] extra") + +TABLE = "main.sales.orders" +VOLUME = "main.staging.landing" +SALT = "e2e-landing-salt-0123456789" +FIRST: List[Tuple[int, float, str]] = [(1, 10.5, "emea"), (2, 20.0, "apac"), (3, 4.25, "emea")] +SECOND: List[Tuple[int, float, str]] = [(7, 1.5, "amer"), (8, 2.5, "amer")] + + +@pytest.fixture +def fake() -> Iterator[FakeDatabricks]: + server = FakeDatabricks().start() + server.add_volume(VOLUME) + try: + yield server + finally: + server.stop() + + +@pytest.fixture +def workspace(monkeypatch, tmp_path: Path, fake: FakeDatabricks, plugin_cache: str) -> Path: + isolate_workspace( + monkeypatch, + tmp_path, + host=fake.endpoint, + warehouse_id=fake.warehouse_id, + plugin_cache=plugin_cache, + ) + monkeypatch.setenv("DATABRICKS_STAGING_VOLUME", "/Volumes/" + VOLUME.replace(".", "/")) + monkeypatch.setenv(masking.DEFAULT_SALT_ENV, SALT) + # A directory of its own: the runner refuses a contract directory that holds more than the + # product (here the test's HOME and credential file), as it should. + product = tmp_path / "product" + product.mkdir() + return product + + +def _csv(root: Path, rows: List[Tuple[int, float, str]]) -> Path: + path = root / "orders.csv" + path.write_text( + "id,amount,region\n" + "".join(f"{i},{a},{r}\n" for i, a, r in rows), encoding="utf-8" + ) + return path + + +def _contract(root: Path) -> Dict[str, Any]: + contract = make_contract() + expose = contract["exposes"][0] + # varchar, not string: the run-time schema gate compares the declared type with DuckDB's own + for column in expose["contract"]["schema"]: + if column["type"] == "string": + column["type"] = "varchar" + expose["policy"]["privacy"] = {"masking": [{"column": "region", "strategy": "hash"}]} + expose["policy"]["authz"]["columnRestrictions"] = [ + {"principal": "group:analysts", "columns": ["amount"], "access": "deny"} + ] + expose["policy"]["authz"]["rowFilters"] = [{"principal": "group:analysts", "where": "id > 0"}] + contract["builds"] = [ + { + "id": "ingest_orders", + "pattern": "acquisition", + "engine": "duckdb", + "properties": { + "source": { + "kind": "filesystem", + "mode": "full_refresh", + "connection": {"uri": str(root / "orders.csv")}, + "reader": {"format": "csv", "options": {"header": True}}, + "streams": ["orders"], + }, + "sink": {"format": "parquet"}, + }, + "outputs": ["orders"], + } + ] + return contract + + +def _sha(value: str) -> str: + return hashlib.sha256((SALT + value).encode("utf-8")).hexdigest() + + +def _by_id(fake: FakeDatabricks) -> List[Dict[str, Any]]: + return sorted(fake.rows(TABLE), key=lambda r: r["id"]) + + +def test_apply_creates_the_table_then_the_build_lands_masked_bronze_in_it(workspace, fake, capfd): + _csv(workspace, FIRST) + path = write_contract(workspace, _contract(workspace)) + + # 1. apply: the masking is accepted (the runner applies it); the table, grants, mask and filter exist + rc, out = apply_contract(capfd, path) + assert rc == 0, out + assert "databricks-governance-unmapped" not in out + assert fake.rows(TABLE) == [], "apply creates the table empty" + assert fake.tables[TABLE]["row_filter"], "the row filter is bound" + assert any(c.get("mask") for c in fake.tables[TABLE]["columns"]), "and the column mask" + grants = dict(fake.grants[("table", TABLE)]) + definition_before = repr(fake.tables[TABLE]) + sql_before = len(fake.sql_log) + + # 2. the build: hash at landing, upload, INSERT OVERWRITE, delete the staged files + rc, out = apply_contract(capfd, path, "--mode", "amend-and-build") + assert rc == 0, out + rows = _by_id(fake) + assert [(r["id"], r["amount"]) for r in rows] == [(i, a) for i, a, _ in FIRST] + assert [r["region"] for r in rows] == [_sha(r) for _, _, r in FIRST] + assert "emea" not in repr(fake.rows(TABLE)), "the cleartext never left the machine" + assert fake.volume_files() == [], "no landed data is left in the volume" + uploads = [p for m, p in fake.file_ops if m == "PUT"] + assert len(uploads) == 1 and uploads[0].startswith( + "/Volumes/main/staging/landing/dbx_fake_orders/" + ) + assert not (workspace / "out").exists() or list((workspace / "out").glob("*")) == [] + assert fake.unknown == [] + + # the table keeps its definition, grants, row filter and mask: the load is the only statement + assert repr(fake.tables[TABLE]) == definition_before + assert dict(fake.grants[("table", TABLE)]) == grants + assert fake.replaced == [] + after = fake.sql_log[sql_before:] + loads = [s for s in after if "INSERT OVERWRITE" in s] + assert len(loads) == 1 and "read_files(" in loads[0] + assert [s for s in after if "INSERT OVERWRITE" not in s] == [], "nothing else ran" + + # 3. loading data does not drift the table + rc, out = apply_contract(capfd, path, "--mode", "dry-run") + assert rc == 0, out + assert "tofu plan: +0 ~0 -0" in out + + +def test_a_second_run_replaces_the_rows(workspace, fake, capfd): + _csv(workspace, FIRST) + path = write_contract(workspace, _contract(workspace)) + assert apply_contract(capfd, path)[0] == 0 + assert apply_contract(capfd, path, "--mode", "amend-and-build")[0] == 0 + assert len(fake.rows(TABLE)) == 3 + + _csv(workspace, SECOND) + marker = fake.tables[TABLE]["marker"] + rc, out = apply_contract(capfd, path, "--mode", "amend-and-build") + assert rc == 0, out + assert [r["id"] for r in _by_id(fake)] == [7, 8], "overwritten, not appended" + assert fake.volume_files() == [] + assert fake.tables[TABLE]["marker"] == marker, "the table was never replaced" diff --git a/tests/iac/test_iac_databricks_masking_at_landing.py b/tests/iac/test_iac_databricks_masking_at_landing.py new file mode 100644 index 000000000..041b55b97 --- /dev/null +++ b/tests/iac/test_iac_databricks_masking_at_landing.py @@ -0,0 +1,286 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``policy.privacy.masking`` on a Databricks expose: accepted when forge-cli applies it. + +The acquisition runner masks at landing, before the rows leave the machine +(``build_runners/_masking.py``), and lands them in the Databricks table +(``build_runners/_databricks_load.py``). So an expose every build of which is a DuckDB +acquisition build keeps its ``policy.privacy.masking``, as on AWS. Anything else that writes the +table (dbt, another engine, no build at all) applies no mask, and the emitter keeps refusing the +field, with the reason: applied anyway, the table would hold the values the contract promises to +mask. + +``fluid validate`` and ``fluid apply`` read the verdict from the one ``_compile`` pass, so each +case below is checked through both. +""" + +from __future__ import annotations + +import copy +from typing import Any, Dict, List, Optional + +import pytest + +from fluid_build.iac import IAC_PLUGINS +from fluid_build.iac.base import UnsupportedBindingError +from fluid_build.iac.providers import databricks as dbx + +pytestmark = [pytest.mark.unit, pytest.mark.databricks] + +MASKING = [{"column": "msisdn", "strategy": "hash"}] + + +@pytest.fixture(autouse=True) +def _warehouse(monkeypatch): + monkeypatch.setenv(dbx.WAREHOUSE_ENV, "wh-0123456789abcdef") + + +def _build( + build_id: str = "ingest", + *, + engine: str = "duckdb", + pattern: Optional[str] = "acquisition", + outputs: Optional[List[str]] = ("orders",), +) -> Dict[str, Any]: + build: Dict[str, Any] = { + "id": build_id, + "engine": engine, + "properties": { + "source": {"kind": "filesystem", "mode": "full_refresh"}, + "sink": {"format": "parquet"}, + }, + } + if pattern: + build["pattern"] = pattern + if outputs is not None: + build["outputs"] = list(outputs) + return build + + +def _contract( + *builds: Dict[str, Any], masking: Any = MASKING, **expose_extra: Any +) -> Dict[str, Any]: + expose: Dict[str, Any] = { + "exposeId": "orders", + "kind": "table", + "binding": { + "platform": "databricks", + "format": "delta_table", + "location": {"database": "main", "schema": "bronze", "table": "orders"}, + }, + "policy": {"authz": {"readers": ["group:analysts"]}, "privacy": {"masking": masking}}, + "contract": { + "schema": [{"name": "id", "type": "bigint"}, {"name": "msisdn", "type": "string"}] + }, + } + expose.update(expose_extra) + contract: Dict[str, Any] = { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "bronze.orders", + "name": "Orders", + "exposes": [expose], + } + if builds: + contract["builds"] = list(builds) + return contract + + +def _emit(contract: Dict[str, Any]) -> Optional[UnsupportedBindingError]: + try: + IAC_PLUGINS["databricks"].emit(copy.deepcopy(contract)) + except UnsupportedBindingError as exc: + return exc + return None + + +def _validate(contract: Dict[str, Any]) -> List[str]: + return dbx.validate_databricks_binding(copy.deepcopy(contract))[0] + + +def _both(contract: Dict[str, Any]) -> Optional[str]: + """The refusal text, asserting that validate and emit agree on whether there is one.""" + refusal = _emit(contract) + errors = _validate(contract) + assert bool(errors) == (refusal is not None), (errors, refusal) + if refusal is None: + return None + assert refusal.kind == "databricks-governance-unmapped" or refusal.kind == "governance-unmapped" + assert "policy.privacy.masking" in str(refusal) + assert any("policy.privacy.masking" in e for e in errors) + return str(refusal) + + +# ── accepted: every build that writes the expose applies the mask ───────── + + +def test_a_duckdb_acquisition_build_that_writes_the_expose_makes_masking_acceptable(): + assert _both(_contract(_build())) is None + + +def test_a_duckdb_build_that_names_no_outputs_is_taken_to_write_it(): + assert _both(_contract(_build(outputs=None))) is None + + +def test_two_duckdb_acquisition_builds_both_apply_it(): + assert _both(_contract(_build("a"), _build("b"))) is None + + +def test_a_build_that_writes_another_expose_does_not_matter(): + other = _build("other", engine="dbt", pattern="embedded-logic", outputs=["something_else"]) + assert _both(_contract(_build(), other)) is None + + +def test_the_engine_is_read_case_insensitively(): + assert _both(_contract(_build(engine=" DuckDB "))) is None + + +def test_masking_and_column_restrictions_are_both_enforced(): + contract = _contract(_build()) + contract["exposes"][0]["policy"]["authz"]["columnRestrictions"] = [ + {"principal": "group:analysts", "columns": ["msisdn"], "access": "deny"} + ] + assert _both(contract) is None + + +def test_the_table_is_emitted_exactly_as_without_masking(): + """Masking is the runner's, not the table's: the module has no trace of it.""" + with_mask = _contract(_build()) + without = _contract(_build()) + without["exposes"][0]["policy"]["privacy"] = {} + a = IAC_PLUGINS["databricks"].emit(copy.deepcopy(with_mask)) + b = IAC_PLUGINS["databricks"].emit(copy.deepcopy(without)) + assert a == b + + +# ── refused, with the reason ────────────────────────────────────────────── + + +def test_a_contract_with_no_build_that_writes_the_expose_is_refused(): + text = _both(_contract()) + assert "machine. No build writes it" in text and "DuckDB acquisition" in text + + +def test_a_dbt_build_writing_the_expose_is_refused_naming_it(): + text = _both(_contract(_build("models", engine="dbt", pattern="embedded-logic"))) + assert "'models'" in text and "dbt" in text + assert "DuckDB acquisition" in text + + +def test_a_dbt_build_beside_a_duckdb_one_is_still_refused(): + text = _both(_contract(_build(), _build("models", engine="dbt-databricks", pattern=None))) + assert "'models'" in text and "'ingest'" not in text + + +@pytest.mark.parametrize( + "engine", ["dlt", "airbyte", "meltano", "kafka-connect", "debezium", "spark"] +) +def test_another_engine_applies_no_mask_here(engine): + text = _both(_contract(_build("other", engine=engine))) + assert "'other'" in text and engine in text + + +def test_a_duckdb_build_that_is_not_an_acquisition_build_applies_no_mask(): + """Embedded SQL on DuckDB does not run ``_masking`` and cannot land a Databricks table.""" + text = _both(_contract(_build("sql", engine="duckdb", pattern="embedded-logic"))) + assert "'sql'" in text + + +def test_a_build_with_no_engine_applies_no_mask(): + build = _build("plain") + del build["engine"] + assert "'plain'" in _both(_contract(build)) + + +# ── the rest of the governance block is unchanged ───────────────────────── + + +@pytest.mark.parametrize( + "extra, field", + [ + ({"policy": {"privacy": {"rowLevelPolicy": {"a": 1}}}}, "policy.privacy.rowLevelPolicy"), + ( + { + "binding": { + "platform": "databricks", + "format": "delta_table", + "location": {"database": "main", "schema": "bronze", "table": "orders"}, + "governance": {"lakeFormation": {"grants": []}}, + } + }, + "binding.governance", + ), + ], +) +def test_row_level_policy_and_binding_governance_stay_refused(extra, field): + contract = _contract(_build(), masking=[]) + contract["exposes"][0].update(extra) + refusal = _emit(contract) + assert refusal is not None and field in str(refusal) + assert any(field in e for e in _validate(contract)) + + +def test_row_level_policy_beside_accepted_masking_is_still_refused_alone(): + contract = _contract(_build()) + contract["exposes"][0]["policy"]["privacy"]["rowLevelPolicy"] = {"a": 1} + refusal = str(_emit(contract)) + assert "policy.privacy.rowLevelPolicy" in refusal + assert "policy.privacy.masking" not in refusal, "the accepted field is not named as refused" + + +@pytest.mark.parametrize("masking", [None, [], {}]) +def test_an_empty_masking_block_is_not_masking(masking): + assert _both(_contract(masking=masking)) is None + + +@pytest.mark.parametrize( + "source, sink, fragment", + [ + ({"mode": "incremental_append"}, {"format": "parquet"}, "full_refresh"), + ({"mode": "full_refresh"}, {"format": "csv"}, "sink format"), + ({}, {"format": "parquet"}, "modes"), + ], +) +def test_a_landing_the_runner_would_refuse_is_refused_before_the_table_exists( + source, sink, fragment +): + build = _build() + build["properties"] = {"source": {"kind": "filesystem", **source}, "sink": sink} + refusal = _emit(_contract(build)) + assert refusal is not None and refusal.kind == "databricks-landing-unsupported" + assert fragment in str(refusal) and "'ingest'" in str(refusal) + + +def test_a_placeholder_mode_is_left_to_the_emitter(): + build = _build() + build["properties"]["source"]["mode"] = "{{ env.MODE }}" + errors, _ = dbx.validate_databricks_binding(_contract(build)) + assert not [e for e in errors if "landing" in e] + # at apply the contract is substituted; a variable still unset is refused there + refusal = _emit(_contract(build)) + assert refusal is not None and refusal.kind == "databricks-landing-unsupported" + + +@pytest.mark.parametrize( + "masking", + [ + [{"column": "msisdn", "strategy": "k_anonymity"}], + [{"column": "msisdn", "strategy": "no-such-strategy"}], + {"column": "msisdn", "strategy": "hash"}, + ], +) +def test_a_masking_rule_the_runner_cannot_apply_is_refused(masking): + refusal = _emit(_contract(_build(), masking=masking)) + assert refusal is not None and refusal.kind == "databricks-masking-invalid" diff --git a/tests/iac/test_iac_databricks_retention.py b/tests/iac/test_iac_databricks_retention.py new file mode 100644 index 000000000..0d5a86d9c --- /dev/null +++ b/tests/iac/test_iac_databricks_retention.py @@ -0,0 +1,856 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``lifecycle.expire`` on Databricks: accepted for a table rebuilt in full, Delta history limited. + +Pure functions only. The spec is the module docstring of +``iac/providers/databricks_retention.py``: a dbt ``table`` model or forge-cli's DuckDB +acquisition rewrites the whole table, so ``expire`` is accepted when the contract's retention +covers the Delta history (30 days by default); anything else is refused. +""" + +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any, Dict, List, Optional + +import pytest + +from fluid_build.iac import IAC_PLUGINS +from fluid_build.iac.base import UnsupportedBindingError +from fluid_build.iac.providers import databricks as dbx +from fluid_build.iac.providers import databricks_retention as ret +from fluid_build.iac.source import contract_source + +pytestmark = [pytest.mark.unit, pytest.mark.databricks] + +WAREHOUSE = "wh-0123456789abcdef" +DAY = 86400 + + +# ── durations ─────────────────────────────────────────────────────────────── +@pytest.mark.parametrize( + "text,seconds", + [ + ("P30D", 30 * DAY), + ("P365D", 365 * DAY), + ("P6Y", 6 * 365 * DAY), + ("P1Y", 365 * DAY), # the shortest year: never read as longer than it is + ("P1M", 28 * DAY), # the shortest month + ("P2M", 56 * DAY), + ("P1Y2M3W4D", (365 + 56 + 21 + 4) * DAY), + ("P4W", 28 * DAY), + ("PT720H", 30 * DAY), + ("PT36H", 36 * 3600), + ("P1DT12H30M15S", DAY + 12 * 3600 + 30 * 60 + 15), + ("P0D", 0), + ("PT0S", 0), + (" P7D ", 7 * DAY), + ], +) +def test_iso_duration(text, seconds): + assert ret.iso_duration_seconds(text) == seconds + + +@pytest.mark.parametrize( + "text", + [ + "", + "P", + "PT", + "P1DT", # a T with no time part + "30 days", + "p30d", + "P-1D", + "P1.5D", + "P1,5D", + "P1H", # hours need the T + "PT1D", + "P1D1Y", # out of order + "P" + "9" * 13 + "D", + None, + 30, + ], +) +def test_iso_duration_rejects(text): + with pytest.raises(ValueError): + ret.iso_duration_seconds(text) + + +@pytest.mark.parametrize( + "text,seconds", + [ + ("interval 30 days", 30 * DAY), + ("INTERVAL 7 DAYS", 7 * DAY), + ("interval 1 week", 7 * DAY), + ("interval 1 week 12 hours", 7 * DAY + 12 * 3600), + ("interval 48 hours", 48 * 3600), + ("30 days", 30 * DAY), + ("interval 1500 milliseconds", 2), # rounded up, never down + ], +) +def test_interval(text, seconds): + assert ret.interval_seconds(text) == seconds + + +@pytest.mark.parametrize( + "text", ["", "interval", "interval 1 month", "interval 1 year", "interval -1 days", "soon", 5] +) +def test_interval_rejects(text): + with pytest.raises(ValueError): + ret.interval_seconds(text) + + +def test_history_is_the_larger_of_the_two_properties_and_defaults_to_30_days(): + assert ret.history_seconds({}) == 30 * DAY + assert ret.history_seconds({ret.LOG_RETENTION: "interval 10 days"}) == 10 * DAY + assert ret.history_seconds({ret.LOG_RETENTION: "interval 2 days"}) == 7 * DAY + assert ret.history_seconds({ret.DELETED_FILE_RETENTION: "interval 90 days"}) == 90 * DAY + # names match case-insensitively + assert ret.history_seconds({"DELTA.LOGRETENTIONDURATION": "interval 400 days"}) == 400 * DAY + with pytest.raises(ValueError): + ret.history_seconds({ret.LOG_RETENTION: "forever"}) + + +# ── the contract ──────────────────────────────────────────────────────────── +def _set(target: Any, path: str, value: Any) -> None: + keys = path.split(".") + for key in keys[:-1]: + target = target[int(key)] if isinstance(target, list) else target.setdefault(key, {}) + if isinstance(target, list): + target[int(keys[-1])] = value + else: + target[keys[-1]] = value + + +def _contract(**overrides: Any) -> Dict[str, Any]: + """``orders`` on Databricks with ``expire`` after P365D, written by a local dbt build.""" + contract: Dict[str, Any] = { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "sales.orders", + "name": "Orders", + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": { + "platform": "databricks", + "format": "delta_table", + "location": {"database": "main", "schema": "sales", "table": "orders"}, + }, + "lifecycle": {"retention": "P365D", "expire": True}, + "contract": {"schema": [{"name": "id", "type": "bigint"}]}, + } + ], + "builds": [ + { + "id": "transform", + "engine": "dbt", + "repository": "./dbt", + "outputs": ["orders"], + "properties": {"model": "orders"}, + "execution": {"runtime": {"platform": "databricks"}}, + } + ], + } + for path, value in overrides.items(): + _set(contract, path.replace("__", "."), value) + return contract + + +def _acquisition(mode: Any = "full_refresh", **build: Any) -> Dict[str, Any]: + source = {"kind": "filesystem", **({"mode": mode} if mode is not None else {})} + out = { + "id": "land", + "engine": "duckdb", + "pattern": "acquisition", + "outputs": ["orders"], + **build, + } + out["properties"] = {"source": source, **(build.get("properties") or {})} + return out + + +def _project( + tmp_path: Path, + sql: str = "select 1 as id\n", + *, + project_models: str = "", + yaml_config: Optional[str] = None, + name: str = "orders", + folder: str = "", + extra: Optional[Dict[str, str]] = None, +) -> Path: + root = tmp_path / "dbt" + root.mkdir(exist_ok=True) + (root / "dbt_project.yml").write_text(f"name: shop\n{project_models}", encoding="utf-8") + models = root / "models" / folder + models.mkdir(parents=True, exist_ok=True) + (models / f"{name}.sql").write_text(sql, encoding="utf-8") + if yaml_config is not None: + (models / "schema.yml").write_text( + f"models:\n - name: {name}\n config:\n{yaml_config}", encoding="utf-8" + ) + for rel, text in (extra or {}).items(): + path = root / "models" / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return root + + +def _compile( + contract: Dict[str, Any], tmp_path: Optional[Path] = None, *, validating: bool = False +) -> dbx._Compiled: + source = tmp_path / "contract.fluid.yaml" if tmp_path is not None else None + with contract_source(source): + return dbx._compile(contract, {dbx.WAREHOUSE_ENV: WAREHOUSE}, validating=validating) + + +def _kinds(compiled: dbx._Compiled) -> List[str]: + return [refusal.kind for refusal in compiled.refusals] + + +def _accepted(compiled: dbx._Compiled) -> dbx._Table: + assert not compiled.refusals, [f"{r.kind}: {r}" for r in compiled.refusals] + (table,) = compiled.tables + assert table.retention is not None + return table + + +def _refused(compiled: dbx._Compiled, kind: str) -> UnsupportedBindingError: + assert f"databricks-retention-{kind}" in _kinds(compiled), _kinds(compiled) + assert not compiled.tables, "a refused expose emits no table" + return next(r for r in compiled.refusals if r.kind == f"databricks-retention-{kind}") + + +# ── a dbt model that is rebuilt in full ───────────────────────────────────── +class TestDbtTable: + def test_sql_config_table_is_accepted(self, tmp_path): + _project(tmp_path, "{{ config(materialized='table') }}\nselect 1 as id\n") + table = _accepted(_compile(_contract(), tmp_path)) + assert table.retention.period == "P365D" and table.retention.seconds == 365 * DAY + + def test_double_quotes_whitespace_control_and_other_args(self, tmp_path): + _project( + tmp_path, + '{{- config(\n materialized="table",\n partition_by=["id"],\n tags=["a"]\n) -}}\n' + "select 1 as id\n", + ) + _accepted(_compile(_contract(), tmp_path)) + + def test_yaml_config_table_is_accepted(self, tmp_path): + _project(tmp_path, yaml_config=" materialized: table\n") + _accepted(_compile(_contract(), tmp_path)) + + def test_project_config_with_the_plus_prefix(self, tmp_path): + _project(tmp_path, project_models="models:\n shop:\n +materialized: table\n") + _accepted(_compile(_contract(), tmp_path)) + + def test_project_config_without_the_plus_prefix(self, tmp_path): + _project(tmp_path, project_models="models:\n shop:\n materialized: table\n") + _accepted(_compile(_contract(), tmp_path)) + + def test_root_level_project_config(self, tmp_path): + _project(tmp_path, project_models="models:\n +materialized: table\n") + _accepted(_compile(_contract(), tmp_path)) + + def test_folder_config(self, tmp_path): + _project( + tmp_path, + folder="marts", + project_models="models:\n shop:\n +materialized: view\n marts:\n" + " +materialized: table\n", + ) + _accepted(_compile(_contract(), tmp_path)) + + def test_config_named_for_the_model_itself(self, tmp_path): + _project( + tmp_path, + folder="marts", + project_models="models:\n shop:\n marts:\n +materialized: view\n" + " orders:\n +materialized: table\n", + ) + _accepted(_compile(_contract(), tmp_path)) + + # dbt: config() in the model > the model's YAML config > dbt_project.yml, and the + # most specific path in dbt_project.yml. + @pytest.mark.parametrize( + "project,yaml,sql,accepted", + [ + ("incremental", "table", None, True), # YAML over the project + ("table", "incremental", None, False), + ("view", "view", "table", True), # SQL over both + ("table", "table", "incremental", False), + ("incremental", "incremental", "table", True), + (None, "view", "table", True), + (None, "table", "view", False), + ], + ) + def test_precedence(self, tmp_path, project, yaml, sql, accepted): + _project( + tmp_path, + (f"{{{{ config(materialized='{sql}') }}}}\n" if sql else "") + "select 1 as id\n", + project_models=(f"models:\n shop:\n +materialized: {project}\n" if project else ""), + yaml_config=f" materialized: {yaml}\n" if yaml else None, + ) + compiled = _compile(_contract(), tmp_path) + if accepted: + _accepted(compiled) + else: + assert compiled.refusals and not compiled.tables + + def test_the_more_specific_project_path_wins(self, tmp_path): + _project( + tmp_path, + folder="marts", + project_models="models:\n shop:\n marts:\n +materialized: incremental\n" + " +materialized: table\n", + ) + _refused(_compile(_contract(), tmp_path), "incremental") + + def test_later_config_call_wins(self, tmp_path): + _project( + tmp_path, + "{{ config(materialized='incremental') }}\n{{ config(materialized='table') }}\n" + "select 1 as id\n", + ) + _accepted(_compile(_contract(), tmp_path)) + + def test_a_commented_out_or_raw_config_is_not_read(self, tmp_path): + _project( + tmp_path, + "{# {{ config(materialized='incremental') }} #}\n" + "{% raw %}{{ config(materialized='incremental') }}{% endraw %}\n" + "{{ config(materialized='table') }}\nselect 1 as id\n", + ) + _accepted(_compile(_contract(), tmp_path)) + + def test_the_model_is_found_by_alias(self, tmp_path): + _project( + tmp_path, + "{{ config(materialized='table', alias='orders') }}\nselect 1 as id\n", + name="stg_orders_final", + ) + # dbt selects by the model's name (its file), not its alias. + contract = _contract(builds__0__properties={"select": "stg_orders_final"}) + _accepted(_compile(contract, tmp_path)) + _refused(_compile(_contract(), tmp_path), "not-selected") + + def test_the_model_is_the_one_named_for_the_table_not_its_neighbours(self, tmp_path): + _project( + tmp_path, + "{{ config(materialized='table') }}\nselect 1 as id\n", + extra={"other.sql": "{{ config(materialized='incremental') }}\nselect 1\n"}, + ) + _accepted(_compile(_contract(), tmp_path)) + + def test_a_table_is_found_case_insensitively(self, tmp_path): + _project(tmp_path, "{{ config(materialized='table') }}\nselect 1 as id\n", name="Orders") + # The relation is found without case; dbt's selection is case-sensitive. + _accepted(_compile(_contract(builds__0__properties={"model": "Orders"}), tmp_path)) + _refused(_compile(_contract(), tmp_path), "not-selected") + + +class TestEnvironmentPlaceholders: + """``fluid validate`` reads the contract before ``{{ env.X }}`` is substituted; the emitter + after. A placeholder is left to the emitter, never refused for its own text.""" + + def _check(self, contract, table="orders"): + return ret.check_expire( + "exposes[0]", contract["exposes"][0], table, contract, validating=True + ) + + def test_a_placeholder_table_name_is_left_to_the_emitter(self, tmp_path): + _project(tmp_path, "{{ config(materialized='table') }}\nselect 1 as id\n") + with contract_source(tmp_path / "contract.fluid.yaml"): + retention, refusals = self._check(_contract(), table="{{ env.tbl }}") + assert refusals == [] and retention is not None + + def test_a_placeholder_retention_is_left_to_the_emitter_and_promises_nothing(self): + contract = _contract( + builds=[_acquisition()], + exposes__0__lifecycle={"retention": "{{ env.RETENTION }}", "expire": True}, + ) + assert self._check(contract) == (None, []) + + def test_a_placeholder_source_mode_is_left_to_the_emitter(self): + assert self._check(_contract(builds=[_acquisition("{{ env.MODE }}")]))[1] == [] + + def test_the_emitter_still_refuses_the_value_once_substituted(self): + contract = _contract(builds=[_acquisition("incremental_append")]) + _refused(_compile(contract), "incremental") + + def test_a_retention_of_hours_names_a_whole_day_count_that_covers_it(self, tmp_path): + _project( + tmp_path, + "{{ config(materialized='table', tblproperties=" + "{'delta.logRetentionDuration': 'interval 721 hours'}) }}\nselect 1 as id\n", + ) + contract = _contract(exposes__0__lifecycle={"retention": "P30D", "expire": True}) + refusal = _refused(_compile(contract, tmp_path), "too-short") + assert "at least P31D" in " ".join(refusal.remediation) + + +class TestJinjaScan: + """``sql_config`` reads a model's tags in linear time: a model file is the dbt author's, + but CI runs ``fluid validate`` on a pull request's.""" + + @pytest.mark.parametrize("opener", ["{{", "{%", "{#", "{% raw %}"]) + def test_a_file_of_unclosed_openers_is_read_quickly(self, opener): + import time + + started = time.monotonic() + assert ret.sql_config(opener * 200_000) == {} + assert time.monotonic() - started < 2.0 + + def test_tags_comments_and_raw_blocks_are_read_as_the_regexes_did(self): + text = ( + "{# {{ config(materialized='view') }} #}" + "{% raw %}{{ config(materialized='view') }}{% endraw %}" + "{{ config(materialized='table') }} {{ unclosed" + ) + assert ret.sql_config(text) == {"materialized": "table"} + + +class TestDbtRefused: + @pytest.mark.parametrize("kind", ["incremental", "snapshot"]) + def test_a_model_that_keeps_old_rows(self, tmp_path, kind): + _project(tmp_path, f"{{{{ config(materialized='{kind}') }}}}\nselect 1 as id\n") + refusal = _refused(_compile(_contract(), tmp_path), "incremental") + assert kind in str(refusal) and "materialized='table'" in " ".join(refusal.remediation) + + @pytest.mark.parametrize("kind", ["view", "ephemeral", "materialized_view", "Table", "x"]) + def test_a_model_that_is_not_a_table(self, tmp_path, kind): + _project(tmp_path, f"{{{{ config(materialized='{kind}') }}}}\nselect 1 as id\n") + _refused(_compile(_contract(), tmp_path), "not-table") + + def test_dbts_default_is_a_view(self, tmp_path): + _project(tmp_path) + refusal = _refused(_compile(_contract(), tmp_path), "not-table") + assert "view" in str(refusal) + + @pytest.mark.parametrize( + "sql", + [ + "{{ config(materialized=var('m')) }}\nselect 1", + "{{ config(materialized=mat) }}\nselect 1", + "{% if var('x') %}{{ config(materialized='table') }}{% endif %}\nselect 1", + "{% for x in y %}{{ config(materialized='table') }}{% endfor %}\nselect 1", + "{{ config(**opts) }}\nselect 1", + "{{ config('table') }}\nselect 1", + "{{ config(materialized='table' }}\nselect 1", + "{{ config(schema='other', materialized='table') }}\nselect 1", + "{{ config(database='other', materialized='table') }}\nselect 1", + "{{ config(enabled=false, materialized='table') }}\nselect 1", + ], + ) + def test_what_cannot_be_determined_is_refused_with_the_fix(self, tmp_path, sql): + _project(tmp_path, sql) + refusal = _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + assert "materialized='table'" in " ".join(refusal.remediation) + + def test_a_config_after_a_closed_block_is_still_read(self, tmp_path): + _project( + tmp_path, + "{% set x = 1 %}{% if x %}{% endif %}{{ config(materialized='table') }}\nselect 1", + ) + _accepted(_compile(_contract(), tmp_path)) + + def test_a_project_jinja_value_is_not_a_literal(self, tmp_path): + _project( + tmp_path, + project_models="models:\n shop:\n +materialized: \"{{ var('m') }}\"\n", + ) + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_a_git_repository_is_not_readable_here(self, tmp_path): + contract = _contract(builds__0__repository="https://github.com/o/r?tag=v1&path=dbt") + refusal = _refused(_compile(contract, tmp_path), "dbt-unresolved") + assert "Git repository" in str(refusal) and "?tag" not in str(refusal) + assert "local dbt project" in " ".join(refusal.remediation) + + def test_no_project_at_the_repository(self, tmp_path): + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_no_model_creates_the_table(self, tmp_path): + _project(tmp_path, "{{ config(materialized='table') }}\nselect 1", name="customers") + refusal = _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + assert "'orders'" in str(refusal) + + def test_two_models_creating_it(self, tmp_path): + _project( + tmp_path, + "{{ config(materialized='table') }}\nselect 1", + extra={"b.sql": "{{ config(alias='orders') }}\nselect 1"}, + ) + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_an_alias_that_is_not_a_literal_cannot_be_ruled_out(self, tmp_path): + _project( + tmp_path, + "{{ config(materialized='table') }}\nselect 1", + extra={"b.sql": "{{ config(alias=var('a')) }}\nselect 1"}, + ) + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_a_python_model(self, tmp_path): + _project(tmp_path, name="other", extra={"orders.py": "def model(dbt, s): pass\n"}) + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_a_model_path_outside_the_project(self, tmp_path): + _project(tmp_path, "{{ config(materialized='table') }}\nselect 1") + (tmp_path / "dbt" / "dbt_project.yml").write_text( + "name: shop\nmodel-paths: ['../x']\n", encoding="utf-8" + ) + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_a_broken_dbt_project_yml(self, tmp_path): + _project(tmp_path) + (tmp_path / "dbt" / "dbt_project.yml").write_text("name: [", encoding="utf-8") + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_a_linked_model_is_not_followed(self, tmp_path): + _project(tmp_path, name="other") + outside = tmp_path / "outside.sql" + outside.write_text("{{ config(materialized='table') }}\nselect 1", encoding="utf-8") + (tmp_path / "dbt" / "models" / "orders.sql").symlink_to(outside) + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_a_model_path_through_a_linked_folder_is_not_followed(self, tmp_path): + _project(tmp_path, name="other") + outside = tmp_path / "outside" / "sub" + outside.mkdir(parents=True) + (outside / "orders.sql").write_text( + "{{ config(materialized='table') }}\nselect 1", encoding="utf-8" + ) + (tmp_path / "dbt" / "linked").symlink_to(tmp_path / "outside") + (tmp_path / "dbt" / "dbt_project.yml").write_text( + "name: shop\nmodel-paths: ['linked/sub']\n", encoding="utf-8" + ) + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_validate_without_a_contract_file_reads_nothing(self): + # As for the job emitter: nothing to read yet, so nothing refused for the project. + _accepted(_compile(_contract(), None, validating=True)) + # The emit path knows no location, and says so. + _refused(_compile(_contract(), None), "dbt-unresolved") + + +def _selecting(properties: Dict[str, Any], **overrides: Any) -> Dict[str, Any]: + return _contract(builds__0__properties=properties, **overrides) + + +class TestSelection: + """The guarantee needs the build to run the model: its ``--select`` must include it.""" + + @pytest.fixture(autouse=True) + def _project(self, tmp_path): + _project( + tmp_path, + "{{ config(materialized='table') }}\nselect 1 as id\n", + folder="marts", + extra={"other.sql": "select 1\n"}, + ) + + @pytest.mark.parametrize( + "properties", + [ + {"model": "orders"}, # +orders+ + {"select": "orders"}, + {"select": ["orders"]}, + {"select": "orders+"}, + {"select": "+orders"}, + {"select": "2+orders"}, + {"select": "orders+3"}, + {"select": "@orders"}, + {"models": "orders"}, + {"select": "fqn:orders"}, + {"select": "model:orders"}, + {"select": "shop.marts.orders"}, + {"select": "shop.marts"}, + {"select": "shop.*"}, + {"select": "shop"}, # the project's name selects its every node + {"select": "marts"}, # dbt matches the fqn without the project's name too + {"select": "marts.orders"}, + {"select": "marts.*"}, + {"select": "shop.m*.orders"}, + {"select": "*"}, + {"select": "path:models/marts"}, + {"select": "path:models/marts/"}, + {"select": "models/marts/orders.sql"}, + {"select": ["other", "orders"]}, + {"select": "other orders"}, + {"select": "orders,path:models/marts"}, + {"select": "+orders+,shop.marts"}, + {}, # no selector: dbt builds everything + {"model": "other"}, # outputs > 1 : no selector, the whole project + ], + ) + def test_a_selection_that_includes_the_model_is_accepted(self, tmp_path, properties): + contract = _selecting(properties) + if properties == {"model": "other"}: + contract["builds"][0]["outputs"] = ["orders", "other"] + contract["exposes"].append(copy.deepcopy(contract["exposes"][0])) + contract["exposes"][1].update(exposeId="other") + contract["exposes"][1]["binding"]["location"]["table"] = "other" + contract["exposes"][1].pop("lifecycle") + compiled = _compile(contract, tmp_path) + assert not compiled.refusals, [f"{r.kind}: {r}" for r in compiled.refusals] + + @pytest.mark.parametrize( + "properties", + [ + {"select": "other"}, + {"select": "customers"}, + {"select": "shop.orders"}, # dbt's fqn is a prefix: marts is skipped + {"select": "ord*"}, # dbt globs the whole rest of the fqn: shop.marts.orders + {"select": "ma*ts"}, + {"select": "*.core"}, + {"select": "Orders"}, # case-sensitive + {"select": "models"}, # no slash: a name, not the path + {"select": ["other", "customers"]}, + {"select": "shop.other"}, + {"select": "path:models/staging"}, + {"select": "models/other.sql"}, + {"select": "orders,other"}, # intersection of two names + {"select": "stg_*"}, + ], + ) + def test_a_selection_that_leaves_it_out_is_refused(self, tmp_path, properties): + refusal = _refused(_compile(_selecting(properties), tmp_path), "not-selected") + assert "'orders'" in str(refusal) and "model 'orders'" in str(refusal) + assert "does not select it" in str(refusal) + assert "properties.select" in " ".join(refusal.remediation) + + @pytest.mark.parametrize( + "properties", + [ + {"select": "tag:nightly"}, + {"select": "config.materialized:table"}, + {"select": "tag:nightly,orders"}, + {"model": "other"}, # +other+ + {"select": "+other"}, # other's ancestors: only dbt's graph knows + {"select": "other+"}, + {"select": "result:error"}, + ], + ) + def test_a_selection_only_the_graph_or_state_can_answer_is_refused(self, tmp_path, properties): + refusal = _refused(_compile(_selecting(properties), tmp_path), "not-selected") + assert "cannot evaluate" in str(refusal) + + def test_a_flag_like_selector_is_dropped_as_the_runner_drops_it(self, tmp_path): + # build_selectors removes it, leaving no selector: the same set dbt would run. + _accepted_or_not = _compile(_selecting({"select": ["--full-refresh"]}), tmp_path) + assert not _accepted_or_not.refusals + + def test_the_selection_is_named_in_the_message(self, tmp_path): + refusal = _refused( + _compile(_selecting({"select": ["other", "tag:x"]}), tmp_path), "not-selected" + ) + assert "other tag:x" in str(refusal) + + def test_not_selected_is_checked_before_the_materialization(self, tmp_path): + _project(tmp_path, "select 1\n", folder="marts") # a view, and not selected + compiled = _compile(_selecting({"select": "other"}), tmp_path) + assert "databricks-retention-not-selected" in _kinds(compiled) + assert "databricks-retention-not-table" not in _kinds(compiled) + + def test_an_acquisition_has_no_dbt_selection(self): + _accepted(_compile(_contract(builds=[_acquisition(properties={"select": "other"})]))) + + +class TestDeltaProperties: + def _table(self, tmp_path, props: str, retention: str = "P365D"): + _project( + tmp_path, + "{{ config(materialized='table', tblproperties=" + props + ") }}\nselect 1 as id\n", + ) + return _compile( + _contract(exposes__0__lifecycle={"retention": retention, "expire": True}), tmp_path + ) + + def test_a_longer_history_than_the_contract_is_refused(self, tmp_path): + compiled = self._table(tmp_path, "{'delta.logRetentionDuration': 'interval 400 days'}") + refusal = _refused(compiled, "too-short") + assert "interval 400 days (declared)" in str(refusal) + assert "7 days (Delta's default)" in str(refusal) + + def test_a_history_inside_the_contract_is_accepted(self, tmp_path): + compiled = self._table( + tmp_path, + "{'delta.logRetentionDuration': 'interval 400 days'," + " 'delta.deletedFileRetentionDuration': 'interval 7 days'}", + "P2Y", + ) + _accepted(compiled) + + def test_the_deleted_file_retention_counts_too(self, tmp_path): + compiled = self._table( + tmp_path, "{'delta.deletedFileRetentionDuration': 'interval 500 days'}" + ) + _refused(compiled, "too-short") + + def test_a_property_that_is_not_an_interval_is_refused(self, tmp_path): + compiled = self._table(tmp_path, "{'delta.logRetentionDuration': 'a long time'}") + _refused(compiled, "dbt-unresolved") + + +# ── the retention itself ──────────────────────────────────────────────────── +class TestPeriod: + @pytest.fixture(autouse=True) + def _dbt_table(self, tmp_path): + _project(tmp_path, "{{ config(materialized='table') }}\nselect 1 as id\n") + + @pytest.mark.parametrize("period", ["P30D", "P31D", "P365D", "P6Y", "P1Y", "P2M", "PT720H"]) + def test_a_retention_that_covers_the_history_is_accepted(self, tmp_path, period): + contract = _contract(exposes__0__lifecycle={"retention": period, "expire": True}) + _accepted(_compile(contract, tmp_path)) + + @pytest.mark.parametrize("period", ["P7D", "P1D", "P29D", "P4W", "P1M", "PT719H", "P0D"]) + def test_a_retention_shorter_than_30_days_is_refused(self, tmp_path, period): + contract = _contract(exposes__0__lifecycle={"retention": period, "expire": True}) + refusal = _refused(_compile(contract, tmp_path), "too-short") + assert "30 days" in str(refusal) and "P30D" in " ".join(refusal.remediation) + + @pytest.mark.parametrize("period", ["30 days", "P", "p30d", "P1.5D", 30, "P1DT"]) + def test_a_retention_that_is_not_an_iso_duration_is_refused(self, tmp_path, period): + contract = _contract(exposes__0__lifecycle={"retention": period, "expire": True}) + _refused(_compile(contract, tmp_path), "period") + + def test_expire_without_a_retention_is_refused(self, tmp_path): + contract = _contract(exposes__0__lifecycle={"expire": True}) + _refused(_compile(contract, tmp_path), "period") + + @pytest.mark.parametrize( + "lifecycle", [{"retention": "P7D"}, {"retention": "P7D", "expire": False}, {}] + ) + def test_without_expire_nothing_is_checked(self, tmp_path, lifecycle): + compiled = _compile(_contract(exposes__0__lifecycle=lifecycle), tmp_path) + assert not compiled.refusals and compiled.tables[0].retention is None + + +# ── the writers ───────────────────────────────────────────────────────────── +class TestWriters: + def test_a_duckdb_acquisition_is_accepted(self): + contract = _contract(builds=[_acquisition()]) + assert _accepted(_compile(contract)).retention.seconds == 365 * DAY + + def test_an_acquisition_without_outputs_is_still_the_writer(self): + build = _acquisition() + del build["outputs"] + _accepted(_compile(_contract(builds=[build]))) + + @pytest.mark.parametrize( + "mode", ["incremental_append", "incremental_merge", "incremental_dedup", "cdc", "streaming"] + ) + def test_an_acquisition_that_does_not_rewrite_the_table_is_refused(self, mode): + refusal = _refused(_compile(_contract(builds=[_acquisition(mode)])), "incremental") + assert repr(mode) in str(refusal) and "full_refresh" in str(refusal) + + def test_an_acquisition_without_a_source_mode_is_refused(self): + _refused(_compile(_contract(builds=[_acquisition(None)])), "unmapped") + + def test_the_mode_accepted_is_the_one_the_landing_runs(self): + from fluid_build.build_runners import _databricks_load as load + + for mode in load.LOAD_MODES: + assert load.unsupported_reason(mode, "parquet", 1) is None + _accepted(_compile(_contract(builds=[_acquisition(mode)]))) + + def test_an_acquisition_with_a_short_retention_is_still_refused(self): + contract = _contract( + builds=[_acquisition()], exposes__0__lifecycle={"retention": "P7D", "expire": True} + ) + _refused(_compile(contract), "too-short") + + @pytest.mark.parametrize( + "build", + [ + {"engine": "dlt", "pattern": "acquisition"}, + {"engine": "airbyte", "pattern": "acquisition"}, + {"engine": "duckdb", "pattern": "embedded-logic"}, + {"engine": "python"}, + {"engine": "spark"}, + {"engine": "flink"}, + ], + ) + def test_a_writer_not_known_to_rewrite_the_table_is_refused(self, build): + contract = _contract(builds=[{"id": "w", "outputs": ["orders"], **build}]) + refusal = _refused(_compile(contract), "unmapped") + assert "orders" in str(refusal) or "build 'w'" in str(refusal) + + def test_no_build_writes_it(self): + contract = _contract(builds=[]) + _refused(_compile(contract), "unmapped") + contract = _contract(builds=[{**_acquisition(), "outputs": ["something_else"]}]) + _refused(_compile(contract), "unmapped") + + def test_every_writer_must_rewrite_the_table(self, tmp_path): + contract = _contract( + builds=[_acquisition(), {"id": "w", "engine": "python", "outputs": ["orders"]}] + ) + _refused(_compile(contract, tmp_path), "unmapped") + + def test_a_dbt_writer_beside_an_acquisition(self, tmp_path): + _project(tmp_path, "{{ config(materialized='incremental') }}\nselect 1 as id\n") + contract = _contract() + contract["builds"].insert(0, _acquisition()) + _refused(_compile(contract, tmp_path), "incremental") + + +# ── validate and the emitter are one pass ─────────────────────────────────── +CASES = { + "table": ("{{ config(materialized='table') }}\nselect 1 as id\n", {}, None), + "incremental": ("{{ config(materialized='incremental') }}\nselect 1\n", {}, "incremental"), + "view": ("select 1\n", {}, "not-table"), + "var": ("{{ config(materialized=var('m')) }}\nselect 1\n", {}, "dbt-unresolved"), + "short": ( + "{{ config(materialized='table') }}\nselect 1\n", + {"exposes__0__lifecycle": {"retention": "P7D", "expire": True}}, + "too-short", + ), +} + + +@pytest.mark.parametrize("label", sorted(CASES)) +def test_validate_and_the_emitter_agree(tmp_path, monkeypatch, label): + sql, overrides, kind = CASES[label] + monkeypatch.setenv(dbx.WAREHOUSE_ENV, WAREHOUSE) + _project(tmp_path, sql) + contract = _contract(**overrides) + with contract_source(tmp_path / "contract.fluid.yaml"): + errors, _warnings = dbx.validate_databricks_binding(copy.deepcopy(contract)) + if kind is None: + assert not errors + assert "databricks_sql_table" in IAC_PLUGINS["databricks"].emit(contract) + return + assert errors + with pytest.raises(UnsupportedBindingError) as exc: + IAC_PLUGINS["databricks"].emit(contract) + assert exc.value.kind == f"databricks-retention-{kind}" + assert str(exc.value) in errors[0] + + +def test_managed_objects_carry_the_accepted_retention(tmp_path): + _project(tmp_path, "{{ config(materialized='table') }}\nselect 1 as id\n") + with contract_source(tmp_path / "contract.fluid.yaml"): + tables, _jobs = dbx.managed_objects(_contract(), {dbx.WAREHOUSE_ENV: WAREHOUSE}) + (table,) = tables + assert (table.retention_period, table.retention_seconds) == ("P365D", 365 * DAY) + plain = _contract(exposes__0__lifecycle={}) + with contract_source(tmp_path / "contract.fluid.yaml"): + (table,), _ = dbx.managed_objects(plain, {dbx.WAREHOUSE_ENV: WAREHOUSE}) + assert table.retention_seconds is None From bf12a016fd52645b5c4b382a33dd5c17febedb83 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:55:45 +0100 Subject: [PATCH 21/26] refactor: one HTTP client in the base install, and redirects never carry a credential to another origin (#804) * refactor: one HTTP client in the base install, httpx The base install carried three HTTP stacks: httpx, requests and aiohttp. Every site now uses httpx, which is already a base dependency. - fluid_build/util/http_client.py: new_client and new_async_client keep requests' behaviour (up to 30 redirects, environment proxies, REQUESTS_CA_BUNDLE/CURL_CA_BUNDLE, TLS always verified). RetryTransport gives urllib3 Retry semantics. httpx's own transport retries cover only connection failures, and httpx-retries (MIT) would add a base dependency, so its shape is adapted here. - Ported: - util/network - observability/reporter: refusing a foreign redirect is now a response hook that runs on every hop - cli/_command_center: HEAD does not follow redirects and GET does, as before - cli/marketplace - the Command Center catalog connector: aiohttp becomes httpx.AsyncClient with aiohttp's limits - the Data Mesh Manager session: GET/PUT/DELETE retried on 429/5xx with backoff, honouring Retry-After - local mocks - util/network.safe_request reported status_code None for every 4xx and 5xx, because the error response is falsy. It now reports the status. - requests leaves [project] dependencies. It still installs through openlineage-python and tiktoken, so the package count drops when those leave the base install in 0.28.0. - Tests move from requests mocks to respx and httpx. The new tests/test_http_client.py covers redirects, retries, proxies and TLS against a loopback server. * fix: redirects are judged on the URL httpx follows, and the Data Mesh Manager key stays on its origin Security review of the httpx port: - refuse_foreign_redirect resolves the Location as httpx does. A Location with a scheme but no host (https:/x) used to pass as same-origin while httpx sent the key and body to the default port. - Retry-After is capped at the 120 s backoff ceiling; an unparseable or overflowing value is ignored rather than crashing the CLI. - The Data Mesh Manager session refuses a cross-origin redirect instead of forwarding x-api-key (older than the port); the error names the new origin and says to update DMM_API_URL. - A missing REQUESTS_CA_BUNDLE/CURL_CA_BUNDLE disables the reporter with a warning naming the variable; TLS is never relaxed. - The catalog connector keeps the system trust store aiohttp used. Drift from requests found by the defect hunt: - a non-JSON 200 from a registry falls back again instead of crashing; - params merge with the URL's own query string (merge_query); - safe_request raises only on 4xx/5xx and translates allow_redirects; - the two SSRF wiring tests fail again when the gate is removed; - the catalog connector's search reads the real SearchFilters fields (broken before the port, hidden by a test shim). * test: the redirect hook agrees with where httpx follows, for 427 Location spellings * chore: refresh .secrets.baseline line numbers * chore: no silent except in the ported HTTP sites --- .secrets.baseline | 8 +- .../catalog_registrars/_http_retry.py | 27 +- fluid_build/cli/_command_center.py | 43 +- .../cli/market_catalogs/command_center.py | 190 +++--- fluid_build/cli/marketplace.py | 186 +++--- fluid_build/observability/reporter.py | 110 ++-- .../datamesh_manager/datamesh_manager.py | 84 +-- fluid_build/providers/local/mocks.py | 6 +- fluid_build/util/http_client.py | 347 +++++++++++ fluid_build/util/network.py | 106 ++-- pyproject.toml | 3 +- .../build_runners/test_consumes_resolution.py | 6 +- tests/iac/conftest.py | 12 +- tests/iac/test_iac_aws_apply_engine_moto.py | 4 +- tests/iac/test_iac_moto_e2e.py | 4 +- tests/iac/test_iac_state_drift_moto.py | 4 +- .../iac/test_iac_state_key_migration_moto.py | 4 +- .../test_iac_cross_account_localstack.py | 10 +- .../test_redirect_hook_matches_httpx.py | 142 +++++ .../observability/test_reporter_redirects.py | 36 +- tests/test_circuit_breaker.py | 31 +- tests/test_command_center_client.py | 41 +- tests/test_command_center_client_gate.py | 46 +- tests/test_command_center_ssrf.py | 28 +- tests/test_datamesh_manager_publish_spec.py | 25 +- tests/test_http_client.py | 252 ++++++++ tests/test_http_client_hardening.py | 567 ++++++++++++++++++ tests/test_market_bundled_blueprints.py | 13 +- tests/test_market_connectors.py | 119 +++- tests/test_marketplace_branches.py | 164 ++--- tests/test_marketplace_ext.py | 213 +++---- tests/test_network.py | 53 ++ 32 files changed, 2096 insertions(+), 788 deletions(-) create mode 100644 fluid_build/util/http_client.py create mode 100644 tests/observability/test_redirect_hook_matches_httpx.py create mode 100644 tests/test_http_client.py create mode 100644 tests/test_http_client_hardening.py diff --git a/.secrets.baseline b/.secrets.baseline index f0631d42f..9735f4c82 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -556,7 +556,7 @@ "filename": "pyproject.toml", "hashed_secret": "fdb92d6db3585d9dec894d5eb5bba7996cecc1dc", "is_verified": false, - "line_number": 85 + "line_number": 84 } ], "scripts/verify_compose.yml": [ @@ -1586,7 +1586,7 @@ "filename": "tests/test_command_center_ssrf.py", "hashed_secret": "c212d24997054583ddd732a7fae6a73c98b07c3a", "is_verified": false, - "line_number": 95 + "line_number": 97 } ], "tests/test_config_manager.py": [ @@ -1707,7 +1707,7 @@ "filename": "tests/test_datamesh_manager_publish_spec.py", "hashed_secret": "829c3804401b0727f70f73d4415e162400cbe57b", "is_verified": false, - "line_number": 719 + "line_number": 720 } ], "tests/test_doctor_store_backend.py": [ @@ -2154,5 +2154,5 @@ } ] }, - "generated_at": "2026-11-03T19:58:17Z" + "generated_at": "2026-11-03T22:58:58Z" } diff --git a/fluid_build/build_runners/catalog_registrars/_http_retry.py b/fluid_build/build_runners/catalog_registrars/_http_retry.py index 110c2549f..5692db7e5 100644 --- a/fluid_build/build_runners/catalog_registrars/_http_retry.py +++ b/fluid_build/build_runners/catalog_registrars/_http_retry.py @@ -45,12 +45,10 @@ from __future__ import annotations -import email.utils import logging import random import time from dataclasses import dataclass, field -from datetime import datetime, timezone from typing import Callable, FrozenSet, Optional, TypeVar # Retryable HTTP status codes — mirrors DataHub's @@ -115,28 +113,9 @@ def retry_after_seconds(exc: BaseException) -> Optional[float]: if not isinstance(exc, httpx.HTTPStatusError): return None - raw = exc.response.headers.get("Retry-After") - if not raw: - return None - raw = raw.strip() - try: - secs = float(int(raw)) - return secs if secs >= 0 else None - except ValueError: - pass - # HTTP-date form → seconds from now. ``parsedate_to_datetime`` - # returns ``None`` on some Pythons and raises ``ValueError`` / - # ``TypeError`` on others for a malformed value — handle both. - try: - parsed = email.utils.parsedate_to_datetime(raw) - except (ValueError, TypeError): - return None - if parsed is None: - return None - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - delta = (parsed - datetime.now(timezone.utc)).total_seconds() - return max(0.0, delta) + from fluid_build.util.http_client import parse_retry_after + + return parse_retry_after(exc.response.headers.get("Retry-After")) def compute_delay(attempt: int, policy: RetryPolicy, retry_after: Optional[float] = None) -> float: diff --git a/fluid_build/cli/_command_center.py b/fluid_build/cli/_command_center.py index e2eb9ef15..038e80281 100644 --- a/fluid_build/cli/_command_center.py +++ b/fluid_build/cli/_command_center.py @@ -46,14 +46,23 @@ # the same host moved up to https): the reporter's redirect rule. A Command # Center that answers with a redirect to another origin, a private or a cloud # metadata address included, is not followed there. -_PROBE_HOOKS = {"response": refuse_foreign_redirect} +_PROBE_HOOKS = [refuse_foreign_redirect] -try: - import requests - REQUESTS_AVAILABLE = True -except ImportError: - REQUESTS_AVAILABLE = False +def _probe(method: str, url: str, *, timeout: float, **kwargs: Any) -> Any: + """One probe request: a fresh client, the redirect rule on every hop. + + Imported lazily: ``httpx`` stays off the cold ``fluid --help`` path. A HEAD + does not follow redirects (as ``requests.head`` did not); a GET does. + """ + from fluid_build.util.http_client import new_client + + with new_client( + timeout=timeout, + response_hooks=_PROBE_HOOKS, + follow_redirects=method.upper() != "HEAD", + ) as client: + return client.request(method, url, **kwargs) @dataclass @@ -182,10 +191,6 @@ def _check_availability(self) -> None: Performs a health check and queries available endpoints. Sets self.available and self.features. """ - if not REQUESTS_AVAILABLE: - self.logger.warning("requests library not available, skipping Command Center detection") - return - # The reporter's host gate, before any request leaves the process: # the URL is user/config-supplied, and the credential goes to it # elsewhere. The warning names the origin only. @@ -199,10 +204,12 @@ def _check_availability(self) -> None: self.available = False return + import httpx # lazy: off the cold --help path + try: # Quick health check with timeout health_url = f"{self.url}/health" - response = requests.get(health_url, timeout=2, hooks=_PROBE_HOOKS) + response = _probe("GET", health_url, timeout=2) if response.status_code == 200: self.available = True @@ -214,11 +221,11 @@ def _check_availability(self) -> None: self.logger.debug(f"Command Center health check failed: {response.status_code}") self.available = False - except requests.exceptions.Timeout: + except httpx.TimeoutException: self.logger.debug(f"Command Center health check timeout: {self.url}") self.available = False - except requests.exceptions.ConnectionError: + except (httpx.NetworkError, httpx.RemoteProtocolError, httpx.ProxyError): self.logger.debug(f"Command Center not reachable: {self.url}") self.available = False @@ -264,7 +271,7 @@ def _check_endpoint(self, endpoint: str) -> bool: url = f"{self.url}{endpoint}" # Try HEAD request first (faster) - response = requests.head(url, timeout=1, hooks=_PROBE_HOOKS) + response = _probe("HEAD", url, timeout=1) # 200 OK or 405 Method Not Allowed (HEAD not supported but endpoint exists) if response.status_code in (200, 405): @@ -275,7 +282,9 @@ def _check_endpoint(self, endpoint: str) -> bool: return False # For other status codes, try GET with limit - response = requests.get(url, params={"limit": 1}, timeout=1, hooks=_PROBE_HOOKS) + from fluid_build.util.http_client import merge_query + + response = _probe("GET", merge_query(url, {"limit": 1}), timeout=1) return response.status_code in (200, 401, 403) # Exists (may need auth) except Exception as e: @@ -290,12 +299,12 @@ def _get_version(self) -> str: Version string or "unknown" """ try: - response = requests.get(f"{self.url}/api/v1/version", timeout=1, hooks=_PROBE_HOOKS) + response = _probe("GET", f"{self.url}/api/v1/version", timeout=1) if response.status_code == 200: data = response.json() return data.get("version", "unknown") except Exception: - pass + pass # the version is informational; an unreachable server reads "unknown" return "unknown" diff --git a/fluid_build/cli/market_catalogs/command_center.py b/fluid_build/cli/market_catalogs/command_center.py index 9c8a44cd7..09ef66be9 100644 --- a/fluid_build/cli/market_catalogs/command_center.py +++ b/fluid_build/cli/market_catalogs/command_center.py @@ -68,15 +68,26 @@ async def _connect_impl(self) -> bool: self.base_url = cc.get_catalog_url() self.cc_client = cc - # Initialize aiohttp session. Every request on it follows a + # Initialize the HTTP session. Every request on it follows a # redirect only within the Command Center's origin: the client's # probes passed the host gate for that origin, not for wherever # the Command Center redirects (the reporter's redirect rule). - import aiohttp - - from fluid_build.observability.reporter import foreign_redirect_trace_config - - self.session = aiohttp.ClientSession(trace_configs=[foreign_redirect_trace_config()]) + import httpx + + from fluid_build.observability.reporter import refuse_foreign_redirect + from fluid_build.util.http_client import new_async_client + + # aiohttp's defaults, kept: 10 redirects, 300s total / 30s connect, + # and no proxy-environment lookup. + self.session = new_async_client( + timeout=httpx.Timeout(300.0, connect=30.0), + response_hooks=[refuse_foreign_redirect], + max_redirects=10, + trust_env=False, + # aiohttp's default context: the system CA store, so a corporate + # CA installed only there is still trusted. + system_trust=True, + ) self.logger.info(f"Connected to Command Center catalog: {self.base_url}") return True @@ -94,18 +105,18 @@ async def _search_data_products_impl(self, filters: SearchFilters) -> List[DataP # Build query parameters params = {} - if filters.query: - params["query"] = filters.query - if filters.domains: - params["domains"] = ",".join(filters.domains) - if filters.owners: - params["owners"] = ",".join(filters.owners) + if filters.text_query: + params["query"] = filters.text_query + if filters.domain: + params["domains"] = filters.domain + if filters.owner: + params["owners"] = filters.owner if filters.tags: params["tags"] = ",".join(filters.tags) - if filters.layers: - params["layers"] = ",".join([layer.value for layer in filters.layers]) - if filters.statuses: - params["statuses"] = ",".join([status.value for status in filters.statuses]) + if filters.layer: + params["layers"] = filters.layer.value + if filters.status: + params["statuses"] = filters.status.value if filters.min_quality_score: params["min_quality"] = filters.min_quality_score if filters.limit: @@ -114,69 +125,80 @@ async def _search_data_products_impl(self, filters: SearchFilters) -> List[DataP params["offset"] = filters.offset # Make request - async with self.session.get(self.base_url, params=params) as response: - if response.status != 200: - self.logger.warning(f"Command Center search failed: {response.status}") - return [] - - data = await response.json() - products = [] - - # Map Command Center schema to DataProductMetadata - for item in data.get("items", []): - try: - # Parse layer - layer = None - if "layer" in item: - try: - layer = DataProductLayer(item["layer"]) - except ValueError: - pass - - # Parse status - status = DataProductStatus.ACTIVE # Default - if "status" in item: - try: - status = DataProductStatus(item["status"]) - except ValueError: - pass - - # Create metadata - product = DataProductMetadata( - id=item["id"], - name=item["name"], - description=item.get("description", ""), - domain=item.get("domain", "unknown"), - owner=( - item.get("owner", {}).get("name", "unknown") - if isinstance(item.get("owner"), dict) - else item.get("owner", "unknown") - ), - layer=layer, - status=status, - tags=item.get("tags", []), - quality_score=item.get("quality_score", 0.0), - created_at=item.get("created_at"), - updated_at=item.get("updated_at"), - data_location=item.get("data_location", ""), - schema_definition=item.get("schema", {}), - sla=item.get("sla", {}), - documentation_url=item.get("documentation_url"), - metadata={ - "source": "command_center", - "cc_url": f"{self.base_url}/{item['id']}", - "version": item.get("version", "unknown"), - **item.get("metadata", {}), - }, - ) - - products.append(product) - - except Exception as e: - self.logger.warning(f"Failed to parse product {item.get('id')}: {e}") - continue - - return products + from fluid_build.util.http_client import merge_query + + response = await self.session.get(merge_query(self.base_url, params)) + if response.status_code != 200: + self.logger.warning(f"Command Center search failed: {response.status_code}") + return [] + + data = response.json() + products = [] + + # Map Command Center schema to DataProductMetadata + for item in data.get("items", []): + try: + # Parse layer + layer = None + if "layer" in item: + try: + layer = DataProductLayer(item["layer"]) + except ValueError: + self.logger.debug( + "Command Center product %r has an unknown layer %r; left unset", + item.get("id"), + item["layer"], + ) + + # Parse status + status = DataProductStatus.ACTIVE # Default + if "status" in item: + try: + status = DataProductStatus(item["status"]) + except ValueError: + self.logger.debug( + "Command Center product %r has an unknown status %r; kept %s", + item.get("id"), + item["status"], + status.value, + ) + + # Create metadata + product = DataProductMetadata( + id=item["id"], + name=item["name"], + description=item.get("description", ""), + domain=item.get("domain", "unknown"), + owner=( + item.get("owner", {}).get("name", "unknown") + if isinstance(item.get("owner"), dict) + else item.get("owner", "unknown") + ), + layer=layer, + status=status, + tags=item.get("tags", []), + quality_score=item.get("quality_score", 0.0), + created_at=item.get("created_at"), + updated_at=item.get("updated_at"), + data_location=item.get("data_location", ""), + schema_definition=item.get("schema", {}), + sla=item.get("sla", {}), + documentation_url=item.get("documentation_url"), + metadata={ + "source": "command_center", + "cc_url": f"{self.base_url}/{item['id']}", + "version": item.get("version", "unknown"), + **item.get("metadata", {}), + }, + ) + + products.append(product) + + except Exception as e: + self.logger.warning(f"Failed to parse product {item.get('id')}: {e}") + continue + + return products except Exception as e: self.logger.error(f"Command Center search failed: {e}") @@ -187,11 +209,11 @@ async def _get_catalog_stats_impl(self) -> Dict[str, Any]: try: stats_url = f"{self.base_url}/stats" - async with self.session.get(stats_url) as response: - if response.status == 200: - return await response.json() - else: - return {"total_products": 0, "total_domains": 0, "avg_quality_score": 0.0} + response = await self.session.get(stats_url) + if response.status_code == 200: + return response.json() + else: + return {"total_products": 0, "total_domains": 0, "avg_quality_score": 0.0} except Exception as e: self.logger.warning(f"Failed to get Command Center stats: {e}") diff --git a/fluid_build/cli/marketplace.py b/fluid_build/cli/marketplace.py index 67ed28e3a..fbde4f1ce 100644 --- a/fluid_build/cli/marketplace.py +++ b/fluid_build/cli/marketplace.py @@ -33,36 +33,12 @@ from pathlib import Path from typing import Any, Callable, Dict, List, Optional -# NOTE: ``requests`` (~107 modules incl urllib3/certifi/charset_normalizer) is -# resolved lazily — at import time it would land on the cold ``fluid --help`` -# path (this module is imported at registration). ``_resolve_requests`` performs -# the import on first use and caches it into the module global, and the PEP 562 -# ``__getattr__`` exposes ``requests`` as a module attribute so the -# ``patch("…cli.marketplace.requests")`` test seam keeps working. See the A++ -# Light CLI startup card. -_REQUESTS_UNRESOLVED = object() # sentinel: import not yet attempted - - -def _resolve_requests(): - """Import ``requests`` lazily, caching the module (or ``None``) in globals. - - Honors a test ``patch("…cli.marketplace.requests")``: once the module - attribute is set, that value is returned unchanged. - """ - current = globals().get("requests", _REQUESTS_UNRESOLVED) - if current is not _REQUESTS_UNRESOLVED: - return current - try: - import requests as _requests - except ImportError: - _requests = None # type: ignore[assignment] - globals()["requests"] = _requests - return _requests +# NOTE: ``httpx`` is imported lazily, inside the functions that make a request: +# this module is imported at registration, and ``httpx`` must stay off the cold +# ``fluid --help`` path. The HTTP calls go through ``fluid_build.util.http_client``. def __getattr__(name: str): - if name == "requests": - return _resolve_requests() if name == "get_command_center_client": from ._command_center import get_command_center_client @@ -85,10 +61,10 @@ def __getattr__(name: str): from ._common import CLIError from .console import cprint -# NOTE: ``get_command_center_client`` lives in ``cli._command_center``, which -# imports ``requests`` (~107 modules) at module scope. Importing it here eagerly -# would pull ``requests`` onto the cold ``fluid --help`` path (this module is -# imported at registration). It's resolved lazily via ``__getattr__`` below — and +# NOTE: ``get_command_center_client`` lives in ``cli._command_center``. +# Importing it here eagerly would pull the Command Center client onto the cold +# ``fluid --help`` path (this module is imported at registration). It's resolved +# lazily via ``__getattr__`` above — and # exposing it as a module attribute keeps the # ``patch("…cli.marketplace.get_command_center_client")`` test seam working. See # the A++ Light CLI startup card. @@ -110,19 +86,32 @@ def __getattr__(name: str): console = Console() if RICH_AVAILABLE else None -def _requests_exception_type(name: str) -> type[BaseException] | None: - """Return a requests exception class, tolerating patched test doubles.""" - requests = _resolve_requests() - exceptions = getattr(requests, "exceptions", None) - exc_type = getattr(exceptions, name, None) - if isinstance(exc_type, type) and issubclass(exc_type, BaseException): - return exc_type - return None +def _http_errors() -> tuple: + """What a failed marketplace request raises (``requests.RequestException``'s role).""" + import httpx + + return (httpx.HTTPError, httpx.InvalidURL) + + +def _response_errors() -> tuple: + """:func:`_http_errors` plus a body that is not JSON. + + ``requests`` wrapped a ``.json()`` decode failure in a ``RequestException``; + ``httpx`` lets the ``ValueError`` (``json.JSONDecodeError``) through. + """ + return _http_errors() + (ValueError,) + + +def _is_timeout(exc: BaseException) -> bool: + import httpx + + return isinstance(exc, httpx.TimeoutException) + +def _is_connection_error(exc: BaseException) -> bool: + import httpx -def _is_requests_exception(exc: BaseException, name: str) -> bool: - exc_type = _requests_exception_type(name) - return bool(exc_type and isinstance(exc, exc_type)) + return isinstance(exc, (httpx.NetworkError, httpx.RemoteProtocolError, httpx.ProxyError)) class NamedRegistryURL(str): @@ -139,19 +128,19 @@ class NamedRegistryURL(str): __slots__ = () -# The headers ``requests`` sends on every request of its own accord -# (``requests.utils.default_headers``, -# https://github.com/psf/requests/blob/main/src/requests/utils.py). Any other -# header may carry a credential, and on a hop to another host ``requests`` -# re-derives only ``Authorization`` (``Session.rebuild_auth``) and ``Cookie`` -# (``Session.resolve_redirects``): an ``X-API-Key`` would go on with it. So a -# request with any other header counts as carrying one. -_DEFAULT_REQUEST_HEADERS = frozenset({"user-agent", "accept-encoding", "accept", "connection"}) +# The headers ``httpx`` sends on every request of its own accord (``Host``, +# plus the client defaults in ``httpx._client``). Any other header may carry a +# credential, and on a hop to another origin ``httpx`` strips only +# ``Authorization``: an ``X-API-Key`` would go on with it. So a request with +# any other header counts as carrying one. +_DEFAULT_REQUEST_HEADERS = frozenset( + {"host", "user-agent", "accept-encoding", "accept", "connection"} +) def _carries_anything(request: Any) -> bool: - """Whether a prepared request has a body or a header beyond requests' defaults.""" - return request.body is not None or any( + """Whether a request has a body or a header beyond the client's defaults.""" + return bool(request.content) or any( name.lower() not in _DEFAULT_REQUEST_HEADERS for name in request.headers ) @@ -160,28 +149,20 @@ def _named_registry_redirect_hook() -> Callable[..., Any]: """A response hook for one call to a named registry: ``refuse_foreign_redirect`` if the call's first request carries a body or a header of its own. - A call whose first request has neither (only ``requests``' default + A call whose first request has neither (only the client's default headers) is followed wherever it goes, as main did: nothing in it is the operator's. One with a body (the instantiate POST, the operator's - parameters) or any other header (Authorization from ``auth`` or - ``~/.netrc``, X-API-Key, a cookie the caller passes) still stops at the - origin. - - Judged once, on the first request: ``requests`` runs a call's response - hooks on every hop, the first response's first. A later hop's prepared - request may carry headers ``requests`` adds of its own accord, none of - them the operator's for the next origin, and each recomputed for each - next URL: the ``Cookie`` it builds from the call's own jar, which holds - the cookies this redirect chain set (``Session.resolve_redirects``; - ``requests.get`` uses a fresh session); for an http:// URL, the - ``Proxy-Authorization`` of the operator's proxy - (``Session.rebuild_proxies``), which goes to the proxy, not the origin; - and an ``Authorization`` that ``~/.netrc`` binds to the hop's own host - (``Session.rebuild_auth``, which drops it when the host changes). - Judging every hop afresh read the first two as the operator's and - stopped a registry that set a cookie, or was reached through an - authenticated proxy, before it moved host. One hook per call: the - verdict belongs to that call's redirect chain. + parameters) or any other header (Authorization from ``~/.netrc``, + X-API-Key, a cookie the caller passes) still stops at the origin. + + Judged once, on the first request: ``httpx`` runs a call's response hooks + on every hop, the first response's first. A later hop's request may carry a + ``Cookie`` the client builds of its own accord from the cookies this + redirect chain set, none of them the operator's for the next origin; + judging every hop afresh read that as the operator's and stopped a registry + that set a cookie before it moved host. One hook per call (and one client + per call, as ``requests.get`` used a fresh session): the verdict belongs to + that call's redirect chain. """ from fluid_build.observability.reporter import refuse_foreign_redirect @@ -198,46 +179,45 @@ def hook(response: Any, *args: Any, **kwargs: Any) -> Any: def _marketplace_request( - request_func, url: str, logger: logging.Logger, *, named_registry: bool = False, **kwargs + method: str, url: str, logger: logging.Logger, *, named_registry: bool = False, **kwargs ): """Run a marketplace HTTP request with bounded timeout and one transient retry. - A redirect is followed only within the marketplace's origin (the - reporter's redirect rule, ``refuse_foreign_redirect``, as for the Command - Center client's probes): neither a search nor the instantiate POST, which - carries the operator's parameters, goes on to another origin. The refusal - is raised as a ``RequestException``, which every caller reports. + Redirects are followed (as ``requests`` did) but only within the + marketplace's origin (the reporter's redirect rule, + ``refuse_foreign_redirect``, as for the Command Center client's probes): + neither a search nor the instantiate POST, which carries the operator's + parameters, goes on to another origin. The refusal is raised as an + ``httpx.HTTPError``, which every caller reports. ``named_registry`` says ``url`` is in a registry the operator named (:class:`NamedRegistryURL`), not the Command Center: there a request with no body and no header of its own follows a redirect to another origin, as on main (:func:`_named_registry_redirect_hook`). """ + import httpx + from fluid_build.observability.reporter import ( CredentialRedirectRefused, refuse_foreign_redirect, ) + from fluid_build.util.http_client import merge_query, new_client timeout = kwargs.pop("timeout", MARKETPLACE_HTTP_TIMEOUT_SECONDS) - request_error = _requests_exception_type("RequestException") or Exception - transient_errors = tuple( - exc_type - for exc_type in ( - _requests_exception_type("Timeout"), - _requests_exception_type("ConnectionError"), - ) - if exc_type is not None - ) + # ``requests`` merged ``params`` into the URL's own query (``?tenant=acme``); + # ``httpx`` would replace it. + url = merge_query(url, kwargs.pop("params", None)) for attempt in range(MARKETPLACE_HTTP_RETRIES + 1): # A retry is a new redirect chain, judged afresh. hook = _named_registry_redirect_hook() if named_registry else refuse_foreign_redirect try: - return request_func(url, timeout=timeout, hooks={"response": hook}, **kwargs) + with new_client(timeout=timeout, response_hooks=[hook]) as client: + return client.request(method, url, **kwargs) except CredentialRedirectRefused as exc: - raise request_error(f"not following the redirect: {exc}") from exc - except request_error as exc: - is_transient = bool(transient_errors and isinstance(exc, transient_errors)) + raise httpx.HTTPError(f"not following the redirect: {exc}") from exc + except _http_errors() as exc: + is_transient = _is_timeout(exc) or _is_connection_error(exc) if not is_transient or attempt >= MARKETPLACE_HTTP_RETRIES: raise logger.warning("Marketplace request failed transiently; retrying once: %s", exc) @@ -246,24 +226,25 @@ def _marketplace_request( def _marketplace_get(url: str, logger: logging.Logger, **kwargs): - requests = _resolve_requests() - return _marketplace_request(requests.get, url, logger, **kwargs) + return _marketplace_request("GET", url, logger, **kwargs) def _marketplace_post(url: str, logger: logging.Logger, **kwargs): - requests = _resolve_requests() - return _marketplace_request(requests.post, url, logger, **kwargs) + return _marketplace_request("POST", url, logger, **kwargs) def _format_marketplace_error(exc: BaseException) -> str: - if _is_requests_exception(exc, "Timeout"): + if _is_timeout(exc): return ( "Marketplace API request timed out after " f"{MARKETPLACE_HTTP_TIMEOUT_SECONDS}s. Check the API URL or try again." ) - if _is_requests_exception(exc, "ConnectionError"): + if _is_connection_error(exc): return "Could not connect to the marketplace API. Check Command Center or FLUID_API_URL." + if isinstance(exc, ValueError): + return "Marketplace API returned a response that is not valid JSON." + response = getattr(exc, "response", None) status_code = getattr(response, "status_code", None) if status_code: @@ -538,7 +519,6 @@ def _bundled_blueprints_matching(args) -> list: def search_blueprints(args, logger: logging.Logger, api_url: str) -> int: """Search marketplace blueprints (bundled + registry).""" - requests = _resolve_requests() # ``--format json`` (mirrors the catalog discovery path in market.py): emit a # clean machine-parseable array on stdout and suppress every decorative print # so a script piping the output gets pure JSON, no rich-table contamination. @@ -582,7 +562,7 @@ def search_blueprints(args, logger: logging.Logger, api_url: str) -> int: data = response.json() registry = data.get("items", []) registry_total = data.get("total", len(registry)) - except requests.exceptions.RequestException as e: + except _response_errors() as e: message = _format_marketplace_error(e) # A bundled-only result is still useful; only hard-fail when there # is nothing at all to show. @@ -669,7 +649,6 @@ def search_blueprints(args, logger: logging.Logger, api_url: str) -> int: def show_blueprint_info(args, logger: logging.Logger, api_url: str) -> int: """Show detailed blueprint information.""" - requests = _resolve_requests() console.print(f"[cyan]ℹ️ Fetching blueprint: {args.blueprint_id}...[/cyan]\n") from fluid_build.cli._market_bundled_blueprints import get_bundled_blueprint, is_bundled @@ -690,7 +669,7 @@ def show_blueprint_info(args, logger: logging.Logger, api_url: str) -> int: ) response.raise_for_status() bp = response.json() - except requests.exceptions.RequestException as e: + except _response_errors() as e: message = _format_marketplace_error(e) console.print(f"[red]❌ {message}[/red]") logger.error("Failed to get blueprint info: %s", message, exc_info=True) @@ -787,7 +766,6 @@ def instantiate_blueprint(args, logger: logging.Logger, api_url: str) -> int: render_bundled_contract, ) - requests = _resolve_requests() console.print(f"[cyan]⚙️ Instantiating blueprint: {args.blueprint_id}...[/cyan]\n") bundled_bp = get_bundled_blueprint(args.blueprint_id) if is_bundled(args.blueprint_id) else None @@ -805,7 +783,7 @@ def instantiate_blueprint(args, logger: logging.Logger, api_url: str) -> int: ) response.raise_for_status() bp = response.json() - except requests.exceptions.RequestException as e: + except _response_errors() as e: message = _format_marketplace_error(e) console.print(f"[red]❌ Failed to fetch blueprint: {message}[/red]") logger.error("Failed to fetch blueprint: %s", message, exc_info=True) @@ -859,7 +837,7 @@ def instantiate_blueprint(args, logger: logging.Logger, api_url: str) -> int: result = response.json() contract = result["contract"] cost_estimate = result.get("cost_estimate", "N/A") - except requests.exceptions.RequestException as e: + except _response_errors() as e: message = _format_marketplace_error(e) console.print(f"[red]❌ Failed to instantiate blueprint: {message}[/red]") logger.error("Blueprint instantiation failed: %s", message, exc_info=True) diff --git a/fluid_build/observability/reporter.py b/fluid_build/observability/reporter.py index b0eafd546..098eea7cc 100644 --- a/fluid_build/observability/reporter.py +++ b/fluid_build/observability/reporter.py @@ -23,12 +23,7 @@ import time from enum import Enum from typing import Any, Dict, List, Mapping, Optional, Tuple -from urllib.parse import urljoin, urlparse, urlsplit - -try: - import requests -except ImportError: - requests = None # type: ignore +from urllib.parse import urlparse, urlsplit # Reuse the canonical SSRF post-DNS-resolution gate (RFC1918, # link-local 169.254.0.0/16 — AWS/GCP metadata — loopback, reserved; @@ -171,8 +166,8 @@ def credential_redirect_refusal(source: str, target: str) -> Optional[str]: ``None`` when it may: ``target`` (absolute) is on the same origin (scheme, host and port), or is the same host moved from http to https on the - default ports, the one cross-origin hop for which ``requests`` keeps - ``Authorization`` (``should_strip_auth``). Every other hop is refused + default ports, the one cross-origin hop for which the HTTP client keeps + ``Authorization``. Every other hop is refused rather than followed with the credential stripped: the request body is the run's report, which is not for another origin either. An https request is never followed down to http. @@ -193,62 +188,45 @@ def credential_redirect_refusal(source: str, target: str) -> Optional[str]: def refuse_foreign_redirect(response: Any, *_args: Any, **_kwargs: Any) -> Any: - """A ``requests`` response hook: raise :class:`CredentialRedirectRefused` for - a redirect :func:`credential_redirect_refusal` refuses, before it is followed. + """An ``httpx`` ``response`` event hook: raise :class:`CredentialRedirectRefused` + for a redirect :func:`credential_redirect_refusal` refuses, before it is followed. - ``requests`` runs a request's response hooks on every hop, so passing - ``hooks={"response": refuse_foreign_redirect}`` to one call guards that - call; :func:`guard_credential_redirects` installs it on a whole session. + ``httpx`` runs a client's ``response`` hooks on every hop, so a client built + with ``response_hooks=[refuse_foreign_redirect]`` + (:func:`fluid_build.util.http_client.new_client`) guards every request it + makes, and the hook sees each redirect before the client follows it. """ if not response.is_redirect: return response - target = urljoin(response.url, response.headers.get("location", "")) - refusal = credential_redirect_refusal(response.url, target) + import httpx # a hook only runs inside an httpx client; keep it off the cold path + + source = str(response.url) + location = response.headers.get("location", "") + try: + parsed = httpx.URL(location) + except httpx.InvalidURL: + parsed = None + if parsed is None: + refusal: Optional[str] = "it redirects to a Location that is not a valid URL" + elif parsed.scheme and not parsed.host: + # ``httpx`` copies the request's host onto such a Location (``https:/x``, + # ``http:``) and drops its port; never judge or follow that form. + refusal = "it redirects to a Location with a scheme but no host" + else: + # The URL the client will follow: the Location resolved against the + # request URL (the public ``URL.join``), not ``urljoin`` on the response URL. + refusal = credential_redirect_refusal(source, str(response.request.url.join(parsed))) if refusal is None: return response - response.close() + try: + response.close() + except RuntimeError: + # An async client's stream cannot be closed synchronously; the async + # wrapper (util.http_client.new_async_client) closes it with aclose(). + pass raise CredentialRedirectRefused(refusal) -def guard_credential_redirects(session: Any) -> Any: - """Make ``session`` (a ``requests.Session``) refuse a redirect its credentials may not follow. - - ``requests`` follows redirects by default and, when the host changes, drops - only ``Authorization``: a credential in any other header (``X-API-Key``) - went on to wherever the endpoint redirected. A response hook - (:func:`refuse_foreign_redirect`) sees every hop before ``requests`` - follows it, and raises :class:`CredentialRedirectRefused` for one - :func:`credential_redirect_refusal` refuses. Returns ``session``. - """ - session.hooks["response"].append(refuse_foreign_redirect) - return session - - -def foreign_redirect_trace_config() -> Any: - """An ``aiohttp.TraceConfig`` that refuses what :func:`refuse_foreign_redirect` refuses. - - The same rule for an ``aiohttp.ClientSession``, which follows redirects by - default: built with ``trace_configs=[foreign_redirect_trace_config()]``, a - session raises :class:`CredentialRedirectRefused` for every hop - :func:`credential_redirect_refusal` refuses, on every request it makes, - before ``aiohttp`` follows it. - """ - import aiohttp - - async def _on_redirect(_session: Any, _context: Any, params: Any) -> None: - response = params.response - source = str(response.url) - location = response.headers.get("Location") or response.headers.get("URI") or "" - refusal = credential_redirect_refusal(source, urljoin(source, location)) - if refusal is not None: - response.close() - raise CredentialRedirectRefused(refusal) - - trace = aiohttp.TraceConfig() - trace.on_request_redirect.append(_on_redirect) - return trace - - class CircuitState(Enum): """Circuit breaker states.""" @@ -339,7 +317,7 @@ def _on_failure(self, error: Exception): self.last_failure_time = time.time() if self.state == CircuitState.HALF_OPEN: - # The type only: a requests exception's text repeats the URL. + # The type only: an HTTP client exception's text repeats the URL. logger.warning( "Circuit breaker transitioning HALF_OPEN → OPEN (recovery failed: %s)", type(error).__name__, @@ -409,13 +387,20 @@ def __init__(self, config: CommandCenterConfig): # Session for connection pooling. Every request carries the credential, # so a redirect is followed only within the endpoint's own origin. self.session: Optional[Any] = None - if self.enabled and requests: - self.session = guard_credential_redirects(requests.Session()) + if self.enabled: + from fluid_build.util.http_client import CaBundleError, new_client + headers = {"Content-Type": "application/json"} if config.api_key: headers["X-API-Key"] = config.api_key headers.update(config.headers or {}) - self.session.headers.update(headers) + try: + self.session = new_client(headers=headers, response_hooks=[refuse_foreign_redirect]) + except CaBundleError as exc: + # An unusable CA bundle must not take the command down, and TLS + # is not relaxed to get past it: reporting is off for this run. + logger.warning("Command Center reporter disabled: %s.", exc) + self.enabled = False def start(self): """Start background worker thread.""" @@ -423,11 +408,6 @@ def start(self): logger.debug("Command Center not configured, reporter disabled") return - if not requests: - logger.warning("requests library not installed, Command Center integration disabled") - self.enabled = False - return - # SSRF gate: ``config.url`` is user/config-supplied and every # event POST carries the X-API-Key header. Refuse to start the # reporter against a host that resolves to a private/link-local/ @@ -669,7 +649,7 @@ def _send_event(self, event: Dict[str, Any]): # Its text names the origins only. outcome = f"not followed, {e}" else: - # The type and the status only: a requests exception's text + # The type and the status only: an HTTP client exception's text # repeats the URL, and nothing here needs the response body. status = getattr(getattr(e, "response", None), "status_code", None) outcome = str(status or type(e).__name__) diff --git a/fluid_build/providers/datamesh_manager/datamesh_manager.py b/fluid_build/providers/datamesh_manager/datamesh_manager.py index 10ff76643..522218fbe 100644 --- a/fluid_build/providers/datamesh_manager/datamesh_manager.py +++ b/fluid_build/providers/datamesh_manager/datamesh_manager.py @@ -54,25 +54,13 @@ ) if TYPE_CHECKING: - import requests as requests_typing + import httpx as httpx_typing - RequestsSession = requests_typing.Session - RequestsResponse = requests_typing.Response + HttpClient = httpx_typing.Client + HttpResponse = httpx_typing.Response else: - RequestsSession = Any - RequestsResponse = Any - -try: - import requests - from requests.adapters import HTTPAdapter - from urllib3.util.retry import Retry - - REQUESTS_AVAILABLE = True -except ImportError: # pragma: no cover - requests = None # type: ignore[assignment] - HTTPAdapter = None # type: ignore[assignment,misc] - Retry = None # type: ignore[assignment,misc] - REQUESTS_AVAILABLE = False + HttpClient = Any + HttpResponse = Any LOG = logging.getLogger(__name__) @@ -184,13 +172,7 @@ def __init__( default=False, ) - if not REQUESTS_AVAILABLE: - raise ProviderError( - "The 'requests' library is required for the Data Mesh Manager provider.\n" - "Install it with: pip install requests" - ) - - self._session_instance: Optional[RequestsSession] = None + self._session_instance: Optional[HttpClient] = None # ---- BaseProvider abstract methods ------------------------------------ @@ -1246,19 +1228,26 @@ def _derive_team_id(fluid: Mapping[str, Any]) -> str: # ---- HTTP helpers ----------------------------------------------------- - def _session(self) -> RequestsSession: + def _session(self) -> HttpClient: + # Follows redirects, honours the proxy environment and retries 429/5xx on + # GET/PUT/DELETE (not POST) with exponential backoff -- what the + # ``requests`` session with a urllib3 ``Retry`` did. See util/http_client. if self._session_instance is None: - s = requests.Session() - retry = Retry( - total=3, - backoff_factor=0.5, - status_forcelist=[429, 500, 502, 503, 504], - allowed_methods=["GET", "PUT", "DELETE"], + from fluid_build.observability.reporter import refuse_foreign_redirect + from fluid_build.util.http_client import new_client + + # Every request carries ``x-api-key``, which httpx (unlike + # ``Authorization``) keeps across origins: a redirect is followed + # only within the API's own origin. + self._session_instance = new_client( + response_hooks=[refuse_foreign_redirect], + retry={ + "total": 3, + "backoff_factor": 0.5, + "status_forcelist": [429, 500, 502, 503, 504], + "allowed_methods": ["GET", "PUT", "DELETE"], + }, ) - adapter = HTTPAdapter(max_retries=retry) - s.mount("https://", adapter) - s.mount("http://", adapter) - self._session_instance = s return self._session_instance def _headers(self) -> Dict[str, str]: @@ -1274,7 +1263,11 @@ def _request( path: str, *, json_body: Any = None, - ) -> RequestsResponse: + ) -> HttpResponse: + import httpx + + from fluid_build.observability.reporter import CredentialRedirectRefused + url = f"{self.api_url}{path}" self._log.debug("%s %s", method, url) try: @@ -1285,11 +1278,24 @@ def _request( json=json_body, timeout=_TIMEOUT, ) - except requests.ConnectionError as exc: + except CredentialRedirectRefused as exc: + # Not retried (the hook runs above the retry transport) and not + # followed: the key and body are for the configured origin only. + raise ProviderError( + f"Refused to follow a redirect from the Entropy Data API: {exc}. " + "The API key and request body were not sent there; if the API moved, " + "update DMM_API_URL." + ) from exc + except ( + httpx.NetworkError, + httpx.RemoteProtocolError, + httpx.ProxyError, + httpx.ConnectTimeout, # requests' ConnectTimeout was a ConnectionError first + ) as exc: raise ProviderError(f"Connection failed: {url} — {exc}") from exc - except requests.Timeout as exc: + except httpx.TimeoutException as exc: raise ProviderError(f"Request timed out: {url}") from exc - except requests.RequestException as exc: + except (httpx.HTTPError, httpx.InvalidURL) as exc: raise ProviderError(f"HTTP request failed: {url} — {exc}") from exc if resp.status_code >= 400: diff --git a/fluid_build/providers/local/mocks.py b/fluid_build/providers/local/mocks.py index c2957d81c..74cf134b2 100644 --- a/fluid_build/providers/local/mocks.py +++ b/fluid_build/providers/local/mocks.py @@ -163,13 +163,15 @@ def stop(self) -> Dict[str, Any]: def create_bucket(self, bucket_name: str) -> Dict[str, Any]: """Create GCS bucket in fake server.""" - import requests + import httpx url = f"{self.endpoint_url}/storage/v1/b" data = {"name": bucket_name} try: - response = requests.post(url, json=data) + # requests.post followed redirects and set no timeout; keep the + # redirects and bound the wait. + response = httpx.post(url, json=data, follow_redirects=True, timeout=30) return {"success": response.status_code in [200, 201], "bucket": bucket_name} except Exception as e: self.logger.error(f"Failed to create bucket: {e}") diff --git a/fluid_build/util/http_client.py b/fluid_build/util/http_client.py new file mode 100644 index 000000000..6964cfd95 --- /dev/null +++ b/fluid_build/util/http_client.py @@ -0,0 +1,347 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The one plain ``httpx`` client factory for callers that used ``requests``. + +This is NOT the SSRF-guarded client (:func:`fluid_build.util.safe_http.safe_httpx_client` +-- DNS pinning, private-address refusal). The sites that use this module +carry their own host gates (the Command Center host gate, the operator-named +registry rule, a fixed vendor API URL) and must keep reaching loopback and +internal hosts; they only needed the transport swapped from ``requests``. + +What it keeps from ``requests`` so behaviour does not change for users: + +* **Redirects are followed** (``requests`` follows by default; ``httpx`` does + not) up to 30 hops (``requests.Session.max_redirects``; ``httpx`` defaults to + 20). Callers that must stop a redirect do so in a ``response`` event hook, + which ``httpx`` runs on every hop before following it. +* **Environment proxies** (``HTTP(S)_PROXY``, ``NO_PROXY``) -- ``trust_env``. +* **A custom CA bundle** from ``REQUESTS_CA_BUNDLE`` / ``CURL_CA_BUNDLE``, + which ``requests`` honoured and ``httpx`` does not (it reads + ``SSL_CERT_FILE`` / ``SSL_CERT_DIR``). Verification stays ON in every case. +* **Retries** with ``urllib3.util.retry.Retry`` semantics, as + :class:`RetryTransport`. + +Borrow-before-build receipts: ``httpx.HTTPTransport(retries=N)`` retries +connection failures only (never a status code, never a read error), so it +cannot stand in for ``Retry(status_forcelist=...)``. The ``httpx-retries`` +package (MIT, will-ockmore/httpx-retries) is the maintained transport-wrapper +answer, but it is another base dependency in a change whose point is fewer; +this module adapts its shape (a transport wrapper with a urllib3-style +``Retry``) in ~80 lines. ``tenacity`` was weighed and not taken for the same +reason, and ``build_runners/catalog_registrars/_http_retry.py`` is an +operation-level loop living above this layer (``util`` must not import +``build_runners``), with different semantics (4 attempts, jitter). + +``httpx`` is imported at module scope: import this module lazily, from the +function that makes the request, so the cold ``fluid --help`` path stays free +of it. +""" + +from __future__ import annotations + +import email.utils +import os +import ssl +import time +from datetime import datetime, timezone +from typing import Any, Callable, Iterable, Mapping, Optional, Union + +import httpx + +#: ``requests.Session.max_redirects``. +MAX_REDIRECTS = 30 + +#: urllib3 ``Retry.DEFAULT_BACKOFF_MAX``. +_BACKOFF_MAX = 120.0 + +#: urllib3 ``Retry.RETRY_AFTER_STATUS_CODES``: the only statuses whose +#: ``Retry-After`` header is honoured. +_RETRY_AFTER_STATUSES = frozenset({413, 429, 503}) + +_IDEMPOTENT_METHODS = frozenset({"HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"}) + + +class RetryError(httpx.TransportError): + """Retries ran out on a retryable status (``requests.exceptions.RetryError``). + + ``requests`` did not hand back the last 5xx/429 response once + ``Retry(status_forcelist=...)`` was exhausted; it raised. This keeps that. + """ + + +class CaBundleError(OSError): + """``REQUESTS_CA_BUNDLE`` / ``CURL_CA_BUNDLE`` names a path that cannot be loaded. + + An ``OSError``, as ``requests`` raised it (per request); the text names the + variable. Verification is never switched off or swapped for certifi to get + past it. + """ + + +def tls_verify() -> Union[bool, ssl.SSLContext]: + """The ``verify=`` value ``requests`` would have used, for ``httpx``. + + ``True`` (certifi) unless ``REQUESTS_CA_BUNDLE`` or ``CURL_CA_BUNDLE`` names + a file or directory, which ``requests`` used in place of its default. Never + ``False``: nothing here switches verification off. + """ + name = "REQUESTS_CA_BUNDLE" if os.environ.get("REQUESTS_CA_BUNDLE") else "CURL_CA_BUNDLE" + bundle = os.environ.get(name) + if not bundle: + return True + try: + if os.path.isdir(bundle): + return ssl.create_default_context(capath=bundle) + return ssl.create_default_context(cafile=bundle) + except (OSError, ssl.SSLError) as exc: + raise CaBundleError( + f"{name} is set to {bundle!r}, which cannot be loaded as a CA bundle ({exc})" + ) from exc + + +def merge_query(url: str, params: Optional[Mapping[str, Any]]) -> str: + """``url`` with ``params`` merged into its own query string, as ``requests`` did. + + ``httpx`` replaces a URL's query with ``params=``; ``requests`` kept the + URL's own pairs and appended the new ones. Empty ``params`` leave ``url`` + untouched (``httpx`` would erase its query). + """ + if not params: + return str(url) + return str(httpx.URL(str(url)).copy_merge_params(params)) + + +def parse_retry_after(raw: Optional[str]) -> Optional[float]: + """Seconds from a ``Retry-After`` value: delta-seconds or an HTTP-date (RFC 9110). + + ``None`` when absent, negative or unparseable. Shared with + ``build_runners.catalog_registrars._http_retry``. + """ + if not raw: + return None + raw = raw.strip() + try: + secs = float(int(raw)) + return secs if secs >= 0 else None + except OverflowError: + # An integer too large for a float (hundreds of digits): unusable. + return None + except ValueError: + pass # not a number of seconds; try the HTTP-date form below + try: + parsed = email.utils.parsedate_to_datetime(raw) + except (ValueError, TypeError, OverflowError): + return None + if parsed is None: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + try: + return max(0.0, (parsed - datetime.now(timezone.utc)).total_seconds()) + except OverflowError: + return None + + +class RetryTransport(httpx.BaseTransport): + """``urllib3.util.retry.Retry`` semantics around an ``httpx`` transport. + + * connect failures (``ConnectError`` / ``ConnectTimeout``) are retried for + every method -- nothing was sent; + * read-side failures (``ReadError``, ``ReadTimeout``, ``RemoteProtocolError``, + ``WriteError``) are retried only for ``allowed_methods`` -- the request + may have been processed; + * a response whose status is in ``status_forcelist`` is retried for + ``allowed_methods``; when retries run out :class:`RetryError` is raised; + * the sleep between attempts is ``backoff_factor * 2 ** (n - 1)`` (none + before the first retry, capped at 120s), or the ``Retry-After`` header on + 413/429/503, whichever the response carries. + + ``total`` is the number of retries, so ``total + 1`` attempts at most. + """ + + def __init__( + self, + *, + total: int = 3, + backoff_factor: float = 0.0, + status_forcelist: Iterable[int] = (), + allowed_methods: Iterable[str] = _IDEMPOTENT_METHODS, + transport: Optional[httpx.BaseTransport] = None, + sleep: Callable[[float], None] = time.sleep, + ) -> None: + self.total = total + self.backoff_factor = backoff_factor + self.status_forcelist = frozenset(status_forcelist) + self.allowed_methods = frozenset(m.upper() for m in allowed_methods) + self._sleep = sleep + self._transport = transport or httpx.HTTPTransport() + + def _backoff(self, retry_number: int) -> float: + if retry_number <= 1: + return 0.0 + return min(_BACKOFF_MAX, self.backoff_factor * (2 ** (retry_number - 1))) + + def _delay_after_error( + self, exc: httpx.TransportError, replayable: bool, retries: int + ) -> float: + """The sleep before retrying after ``exc``; re-raises ``exc`` when it is final.""" + connect_failure = isinstance(exc, (httpx.ConnectError, httpx.ConnectTimeout)) + read_failure = isinstance( + exc, + (httpx.ReadError, httpx.ReadTimeout, httpx.WriteError, httpx.RemoteProtocolError), + ) + retryable = connect_failure or (read_failure and replayable) + if not retryable or retries >= self.total: + raise exc + return self._backoff(retries + 1) + + def _delay_after_response( + self, request: httpx.Request, response: httpx.Response, replayable: bool, retries: int + ) -> Optional[float]: + """The sleep before retrying after ``response``; ``None`` to hand it back.""" + if not replayable or response.status_code not in self.status_forcelist: + return None + if retries >= self.total: + response.close() + raise RetryError(f"too many {response.status_code} error responses", request=request) + delay = self._backoff(retries + 1) + if response.status_code in _RETRY_AFTER_STATUSES: + advertised = parse_retry_after(response.headers.get("Retry-After")) + if advertised is not None: + # Capped like the backoff (httpx-retries' ``max_backoff_wait``): a + # server does not get to park the CLI for days. + delay = min(_BACKOFF_MAX, advertised) + response.close() + return delay + + def handle_request(self, request: httpx.Request) -> httpx.Response: + replayable = request.method.upper() in self.allowed_methods + retries = 0 + while True: + try: + response = self._transport.handle_request(request) + except httpx.TransportError as exc: + delay = self._delay_after_error(exc, replayable, retries) + else: + maybe_delay = self._delay_after_response(request, response, replayable, retries) + if maybe_delay is None: + return response + delay = maybe_delay + retries += 1 + if delay > 0: + self._sleep(delay) + + def close(self) -> None: + self._transport.close() + + +class _Client(httpx.Client): + """``httpx.Client`` whose every transport -- direct and per-proxy -- retries. + + A ``transport=`` argument would make ``httpx`` ignore the proxy environment + (``allow_env_proxies = trust_env and transport is None``), so the retry + wrapper is applied at the two factory methods ``httpx`` builds transports + through instead. + """ + + def __init__(self, *, retry: Optional[Mapping[str, Any]] = None, **kwargs: Any) -> None: + self._retry_options = dict(retry) if retry is not None else None + super().__init__(**kwargs) + + def _wrap(self, transport: httpx.BaseTransport) -> httpx.BaseTransport: + if self._retry_options is None: + return transport + return RetryTransport(transport=transport, **self._retry_options) + + def _init_transport(self, *args: Any, **kwargs: Any) -> httpx.BaseTransport: + return self._wrap(super()._init_transport(*args, **kwargs)) + + def _init_proxy_transport(self, *args: Any, **kwargs: Any) -> httpx.BaseTransport: + return self._wrap(super()._init_proxy_transport(*args, **kwargs)) + + +def new_client( + *, + timeout: Union[float, httpx.Timeout, None] = None, + headers: Optional[Mapping[str, str]] = None, + response_hooks: Iterable[Callable[[httpx.Response], Any]] = (), + retry: Optional[Mapping[str, Any]] = None, + follow_redirects: bool = True, + **kwargs: Any, +) -> httpx.Client: + """An ``httpx.Client`` that behaves like a ``requests`` session. + + Follows redirects (``follow_redirects=False`` where the site did not), + trusts the proxy environment, verifies TLS (:func:`tls_verify`), and runs + each ``response_hooks`` entry on every hop before it is followed. ``retry`` + is the keyword arguments of :class:`RetryTransport` (``total``, + ``backoff_factor``, ``status_forcelist``, ``allowed_methods``, ``sleep``). + """ + return _Client( + retry=retry, + timeout=timeout, + headers=dict(headers) if headers else None, + follow_redirects=follow_redirects, + max_redirects=MAX_REDIRECTS, + verify=tls_verify(), + event_hooks={"response": list(response_hooks)}, + trust_env=True, + **kwargs, + ) + + +def new_async_client( + *, + timeout: Union[float, httpx.Timeout, None] = None, + response_hooks: Iterable[Callable[[httpx.Response], Any]] = (), + follow_redirects: bool = True, + max_redirects: int = MAX_REDIRECTS, + trust_env: bool = True, + system_trust: bool = False, + **kwargs: Any, +) -> httpx.AsyncClient: + """The async twin of :func:`new_client` (``httpx.AsyncClient``). + + ``response_hooks`` stay plain functions: they are wrapped in the coroutine + ``httpx`` requires of an async client's event hooks, so one hook (for + example ``refuse_foreign_redirect``) serves the sync and async clients. + + ``system_trust=True`` verifies against the system CA store + (``ssl.create_default_context()``, what ``aiohttp`` used) instead of + certifi / ``REQUESTS_CA_BUNDLE``, for a caller that must keep trusting a + corporate CA installed only system-wide. Verification stays on. + """ + + def _as_async(hook: Callable[[httpx.Response], Any]) -> Callable[[httpx.Response], Any]: + async def _hook(response: httpx.Response) -> None: + try: + hook(response) + except Exception: + await response.aclose() + raise + + return _hook + + return httpx.AsyncClient( + timeout=timeout, + follow_redirects=follow_redirects, + max_redirects=max_redirects, + verify=( + ssl.create_default_context() if system_trust else tls_verify() if trust_env else True + ), + event_hooks={"response": [_as_async(h) for h in response_hooks]}, + trust_env=trust_env, + **kwargs, + ) diff --git a/fluid_build/util/network.py b/fluid_build/util/network.py index 35fd8ad2f..96e63f5f1 100644 --- a/fluid_build/util/network.py +++ b/fluid_build/util/network.py @@ -35,18 +35,6 @@ if TYPE_CHECKING: pass -try: - import requests as requests_module - from requests.adapters import HTTPAdapter - from urllib3.util.retry import Retry - - HAS_REQUESTS = True -except ImportError: - HAS_REQUESTS = False - requests_module = None # type: ignore - HTTPAdapter = None # type: ignore - Retry = None # type: ignore - from ..errors import NetworkError, wrap_error logger = logging.getLogger(__name__) @@ -194,35 +182,41 @@ def create_session_with_retries( max_retries: int = DEFAULT_MAX_RETRIES, backoff_factor: float = DEFAULT_BACKOFF_FACTOR, status_forcelist: tuple = (500, 502, 503, 504, 429), -) -> Any: # returns requests.Session +) -> Any: # returns httpx.Client """ - Create requests session with automatic retry configuration. + Create an httpx client with automatic retry configuration. + + Redirects are followed and the proxy environment is honoured, as the + ``requests`` session this replaces did. Retries follow + ``urllib3.util.retry.Retry`` semantics (see ``util.http_client``). Args: max_retries: Maximum number of retries - backoff_factor: Backoff multiplier (delay = backoff_factor * (2 ** retry_count)) + backoff_factor: Backoff multiplier (delay = backoff_factor * (2 ** (retry_count - 1)), + no delay before the first retry) status_forcelist: HTTP status codes that trigger retry Returns: - Configured requests.Session + Configured httpx.Client """ - if not HAS_REQUESTS: - raise ImportError("requests library not installed") - - session = requests_module.Session() - - retry_strategy = Retry( - total=max_retries, - backoff_factor=backoff_factor, - status_forcelist=status_forcelist, - allowed_methods=["HEAD", "GET", "PUT", "POST", "DELETE", "OPTIONS", "TRACE"], + from .http_client import new_client + + return new_client( + retry={ + "total": max_retries, + "backoff_factor": backoff_factor, + "status_forcelist": status_forcelist, + "allowed_methods": ["HEAD", "GET", "PUT", "POST", "DELETE", "OPTIONS", "TRACE"], + } ) - adapter = HTTPAdapter(max_retries=retry_strategy) - session.mount("http://", adapter) - session.mount("https://", adapter) - return session +_REQUESTS_ONLY_KWARGS = { + "verify": "pass the CA bundle through the client (REQUESTS_CA_BUNDLE / CURL_CA_BUNDLE " + "or util.http_client.new_client) -- verify= is a client option in httpx", + "cert": "client certificates are a client option in httpx (new_client(cert=...))", + "stream": "use session.stream(...) for a streamed response in httpx", +} def safe_request( @@ -234,7 +228,7 @@ def safe_request( session: Optional[Any] = None, circuit_breaker: Optional[CircuitBreaker] = None, **kwargs, -) -> Any: # returns requests.Response +) -> Any: # returns httpx.Response """ Make HTTP request with safety features. @@ -243,9 +237,9 @@ def safe_request( url: Request URL timeout: Read timeout in seconds connect_timeout: Connection timeout in seconds - session: Optional requests session (creates one with retries if not provided) + session: Optional httpx client (creates one with retries if not provided) circuit_breaker: Optional circuit breaker - **kwargs: Additional arguments passed to requests + **kwargs: Additional arguments passed to httpx Returns: Response object @@ -253,8 +247,7 @@ def safe_request( Raises: NetworkError: If request fails after retries """ - if not HAS_REQUESTS: - raise ImportError("requests library not installed") + import httpx # Set timeouts if timeout is None: @@ -262,15 +255,23 @@ def safe_request( if connect_timeout is None: connect_timeout = DEFAULT_CONNECT_TIMEOUT - # Use tuple (connect_timeout, read_timeout) - timeout_tuple = (connect_timeout, timeout) + # connect_timeout bounds the connection, timeout the read (and write/pool) + request_timeout = httpx.Timeout(timeout, connect=connect_timeout) # Create session if not provided if session is None: session = create_session_with_retries() + # ``requests`` keyword names that ``httpx`` spells differently: translate the + # one that maps one-to-one, refuse the rest rather than ignore them silently. + if "allow_redirects" in kwargs: + kwargs["follow_redirects"] = kwargs.pop("allow_redirects") + for name, instead in _REQUESTS_ONLY_KWARGS.items(): + if name in kwargs: + raise TypeError(f"safe_request() no longer takes {name!r}: {instead}") + # Add timeout to kwargs - kwargs["timeout"] = timeout_tuple + kwargs["timeout"] = request_timeout # Execute request try: @@ -279,12 +280,15 @@ def safe_request( else: response = session.request(method, url, **kwargs) - # Raise for HTTP errors - response.raise_for_status() + # Raise for 4xx/5xx only, as ``requests`` did: ``httpx``'s + # ``raise_for_status()`` also raises on a 1xx/3xx (a 304, a 301 with no + # ``Location``). + if response.is_error: + response.raise_for_status() return response - except requests_module.exceptions.Timeout as e: + except httpx.TimeoutException as e: raise NetworkError( f"Request timed out after {timeout}s: {url}", context={"url": url, "method": method, "timeout": timeout}, @@ -296,7 +300,7 @@ def safe_request( original_error=e, ) - except requests_module.exceptions.ConnectionError as e: + except (httpx.NetworkError, httpx.RemoteProtocolError, httpx.ProxyError) as e: raise NetworkError( f"Connection failed: {url}", context={"url": url, "method": method}, @@ -308,8 +312,8 @@ def safe_request( original_error=e, ) - except requests_module.exceptions.HTTPError as e: - status_code = e.response.status_code if e.response else None + except httpx.HTTPStatusError as e: + status_code = e.response.status_code raise NetworkError( f"HTTP {status_code} error: {url}", context={"url": url, "method": method, "status_code": status_code}, @@ -331,38 +335,38 @@ def safe_request( # Convenience functions -def safe_get(url: str, **kwargs) -> Any: # returns requests.Response +def safe_get(url: str, **kwargs) -> Any: # returns httpx.Response """Safe GET request with timeouts and retries""" return safe_request("GET", url, **kwargs) -def safe_post(url: str, **kwargs) -> Any: # returns requests.Response +def safe_post(url: str, **kwargs) -> Any: # returns httpx.Response """Safe POST request with timeouts and retries""" return safe_request("POST", url, **kwargs) -def safe_put(url: str, **kwargs) -> Any: # returns requests.Response +def safe_put(url: str, **kwargs) -> Any: # returns httpx.Response """Safe PUT request with timeouts and retries""" return safe_request("PUT", url, **kwargs) -def safe_patch(url: str, **kwargs) -> Any: # returns requests.Response +def safe_patch(url: str, **kwargs) -> Any: # returns httpx.Response """Safe PATCH request with timeouts and retries""" return safe_request("PATCH", url, **kwargs) -def safe_delete(url: str, **kwargs) -> Any: # returns requests.Response +def safe_delete(url: str, **kwargs) -> Any: # returns httpx.Response """Safe DELETE request with timeouts and retries""" return safe_request("DELETE", url, **kwargs) # Global instances for common use -_default_session: Optional[Any] = None # requests.Session +_default_session: Optional[Any] = None # httpx.Client _default_rate_limiter: Optional[RateLimiter] = None _default_circuit_breaker: Optional[CircuitBreaker] = None -def get_default_session() -> Any: # returns requests.Session +def get_default_session() -> Any: # returns httpx.Client """Get or create default session""" global _default_session if _default_session is None: diff --git a/pyproject.toml b/pyproject.toml index 55312b7fa..ba879cc60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,9 +44,8 @@ dependencies = [ "pydantic>=2.7", "typing-extensions>=4.8", "jinja2>=3.1.6", # security: >=3.1.6 closes CVE-2025-27516 (sandbox escape via |attr) - "httpx>=0.27", # For FluidCommandCenter catalog provider + "httpx>=0.27", # The one HTTP client: Command Center, Data Mesh Manager, marketplace, reporter "h11>=0.16", # security: pin transitive h11 >=0.16 to close CVE-2025-43859 (chunked request smuggling) - "requests>=2.32.5", # For Data Mesh Manager catalog publishing and shared HTTP helpers # LiteLLM is the canonical LLM backend (was an opt-in extra). It # replaces ~1,300 lines of per-provider wire-format code with one # unified API across openai / anthropic / gemini / bedrock / azure / diff --git a/tests/build_runners/test_consumes_resolution.py b/tests/build_runners/test_consumes_resolution.py index 7b6157a4b..5d9de688f 100644 --- a/tests/build_runners/test_consumes_resolution.py +++ b/tests/build_runners/test_consumes_resolution.py @@ -931,7 +931,7 @@ def s3_endpoint(monkeypatch) -> Iterator[str]: import sys import time - import requests + import httpx port = _free_port() endpoint = f"http://127.0.0.1:{port}" @@ -944,9 +944,9 @@ def s3_endpoint(monkeypatch) -> Iterator[str]: deadline = time.monotonic() + 30 while True: try: - requests.get(f"{endpoint}/moto-api/", timeout=1) + httpx.get(f"{endpoint}/moto-api/", timeout=1) break - except requests.RequestException: + except httpx.RequestError: if proc.poll() is not None or time.monotonic() > deadline: pytest.skip("moto server did not start") time.sleep(0.2) diff --git a/tests/iac/conftest.py b/tests/iac/conftest.py index f93a38f91..ee0bb16c2 100644 --- a/tests/iac/conftest.py +++ b/tests/iac/conftest.py @@ -791,13 +791,13 @@ def _localstack_ready() -> tuple[bool, str]: if os.environ.get("FLUID_IAC_LIVE_LOCALSTACK", "").strip().lower() not in _TRUE: return False, "live LocalStack tests are opt-in — set FLUID_IAC_LIVE_LOCALSTACK=1" try: - import requests # noqa: F401 — verify present + import httpx # noqa: F401 — verify present except ImportError: - return False, "the `requests` package is not installed" + return False, "the `httpx` package is not installed" try: - import requests + import httpx - resp = requests.get(f"{LOCALSTACK_ENDPOINT}/_localstack/health", timeout=5) + resp = httpx.get(f"{LOCALSTACK_ENDPOINT}/_localstack/health", timeout=5) if resp.status_code != 200: return False, f"LocalStack health at {LOCALSTACK_ENDPOINT} returned {resp.status_code}" except Exception as exc: # noqa: BLE001 @@ -873,11 +873,11 @@ def _localstack_reset(endpoint: str = LOCALSTACK_ENDPOINT) -> None: (S3 buckets, Glue databases, Lambdas, …) so the next test sees an empty account. The default account ID stays ``000000000000``. """ - import requests + import httpx with contextlib.suppress(Exception): # POST is the documented method; GET works on some Pro versions. - requests.post(f"{endpoint}/_localstack/state/reset", timeout=15) + httpx.post(f"{endpoint}/_localstack/state/reset", timeout=15) class LocalStackProject: diff --git a/tests/iac/test_iac_aws_apply_engine_moto.py b/tests/iac/test_iac_aws_apply_engine_moto.py index dfe22f8bf..7d8d530f2 100644 --- a/tests/iac/test_iac_aws_apply_engine_moto.py +++ b/tests/iac/test_iac_aws_apply_engine_moto.py @@ -82,7 +82,7 @@ def moto_endpoint(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: the backends. The ``/moto-api/reset`` admin endpoint clears all state, giving each test the clean account it expects. """ - import requests + import httpx from moto.server import ThreadedMotoServer server = ThreadedMotoServer(port=0, verbose=False) @@ -94,7 +94,7 @@ def moto_endpoint(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: # Reset before yielding — also covers state leaked in by a previous # test running in the same process. with contextlib.suppress(Exception): - requests.post(f"{endpoint}/moto-api/reset", timeout=5) + httpx.post(f"{endpoint}/moto-api/reset", timeout=5) yield endpoint finally: server.stop() diff --git a/tests/iac/test_iac_moto_e2e.py b/tests/iac/test_iac_moto_e2e.py index 6df41b33b..a8c683d2f 100644 --- a/tests/iac/test_iac_moto_e2e.py +++ b/tests/iac/test_iac_moto_e2e.py @@ -131,7 +131,7 @@ def moto_endpoint(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: ``/moto-api/reset`` admin endpoint clears them, giving each test the clean account it expects. """ - import requests + import httpx from moto.server import ThreadedMotoServer server = ThreadedMotoServer(port=0, verbose=False) @@ -141,7 +141,7 @@ def moto_endpoint(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: endpoint = f"http://127.0.0.1:{port}" monkeypatch.setenv("AWS_ENDPOINT_URL", endpoint) with contextlib.suppress(Exception): - requests.post(f"{endpoint}/moto-api/reset", timeout=5) + httpx.post(f"{endpoint}/moto-api/reset", timeout=5) yield endpoint finally: server.stop() diff --git a/tests/iac/test_iac_state_drift_moto.py b/tests/iac/test_iac_state_drift_moto.py index f7628a67b..61d8fc801 100644 --- a/tests/iac/test_iac_state_drift_moto.py +++ b/tests/iac/test_iac_state_drift_moto.py @@ -81,7 +81,7 @@ def plugin_cache(tmp_path_factory: pytest.TempPathFactory) -> str: @pytest.fixture def moto(monkeypatch, tmp_path: Path, plugin_cache: str) -> Iterator[str]: """A fresh moto server that both ``tofu`` and boto3 are pointed at.""" - import requests + import httpx from moto.server import ThreadedMotoServer server = ThreadedMotoServer(port=0, verbose=False) @@ -90,7 +90,7 @@ def moto(monkeypatch, tmp_path: Path, plugin_cache: str) -> Iterator[str]: _, port = server.get_host_and_port() endpoint = f"http://127.0.0.1:{port}" with contextlib.suppress(Exception): - requests.post(f"{endpoint}/moto-api/reset", timeout=5) + httpx.post(f"{endpoint}/moto-api/reset", timeout=5) for var in ("AWS_PROFILE", "AWS_SESSION_TOKEN", "FLUID_STATE_BACKEND", "FLUID_PROVIDER"): monkeypatch.delenv(var, raising=False) monkeypatch.setenv("AWS_ENDPOINT_URL", endpoint) diff --git a/tests/iac/test_iac_state_key_migration_moto.py b/tests/iac/test_iac_state_key_migration_moto.py index e0d14f6c1..3e10eee32 100644 --- a/tests/iac/test_iac_state_key_migration_moto.py +++ b/tests/iac/test_iac_state_key_migration_moto.py @@ -95,7 +95,7 @@ def plugin_cache(tmp_path_factory: pytest.TempPathFactory) -> str: @pytest.fixture def moto(monkeypatch, tmp_path: Path, plugin_cache: str) -> Iterator[str]: """A fresh moto server holding the state bucket; tofu and boto3 aim at it.""" - import requests + import httpx from moto.server import ThreadedMotoServer server = ThreadedMotoServer(port=0, verbose=False) @@ -104,7 +104,7 @@ def moto(monkeypatch, tmp_path: Path, plugin_cache: str) -> Iterator[str]: _, port = server.get_host_and_port() endpoint = f"http://127.0.0.1:{port}" with contextlib.suppress(Exception): - requests.post(f"{endpoint}/moto-api/reset", timeout=5) + httpx.post(f"{endpoint}/moto-api/reset", timeout=5) for var in ("AWS_PROFILE", "AWS_SESSION_TOKEN", "FLUID_STATE_BACKEND", "FLUID_PROVIDER"): monkeypatch.delenv(var, raising=False) monkeypatch.setenv("AWS_ENDPOINT_URL", endpoint) diff --git a/tests/integration/test_iac_cross_account_localstack.py b/tests/integration/test_iac_cross_account_localstack.py index c74e221f5..dcfc45df6 100644 --- a/tests/integration/test_iac_cross_account_localstack.py +++ b/tests/integration/test_iac_cross_account_localstack.py @@ -140,15 +140,15 @@ def _gate() -> tuple[bool, str]: return False, f"fluid_build.iac unavailable: {exc}" if runner.tofu_path() is None: return False, "the `tofu` (OpenTofu) binary is not on PATH" - for mod in ("boto3", "requests"): + for mod in ("boto3", "httpx"): try: __import__(mod) except ImportError: return False, f"the `{mod}` package is not installed" - import requests + import httpx try: - resp = requests.get(f"{ENDPOINT}/_localstack/health", timeout=5) + resp = httpx.get(f"{ENDPOINT}/_localstack/health", timeout=5) except Exception as exc: # noqa: BLE001 return False, f"LocalStack not reachable at {ENDPOINT}: {exc}" if resp.status_code != 200: @@ -188,10 +188,10 @@ def _client(service: str, key: str = KEY_A): def _reset() -> None: """Wipe every emulator backend so each test starts from empty.""" - import requests + import httpx try: - requests.post(f"{ENDPOINT}/_localstack/state/reset", timeout=20) + httpx.post(f"{ENDPOINT}/_localstack/state/reset", timeout=20) except Exception: # noqa: BLE001 - best-effort isolation pass diff --git a/tests/observability/test_redirect_hook_matches_httpx.py b/tests/observability/test_redirect_hook_matches_httpx.py new file mode 100644 index 000000000..d966dd19a --- /dev/null +++ b/tests/observability/test_redirect_hook_matches_httpx.py @@ -0,0 +1,142 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The redirect hook must judge the URL httpx will actually follow. + +``refuse_foreign_redirect`` resolves a ``Location`` itself; httpx resolves it +again in ``Client._redirect_url`` before following it. If the two ever +disagree, a hop the hook judged same-origin can land somewhere else with the +credential (an ``https:/x`` Location once did: httpx keeps the host and drops +the port). This compares the hook's verdict with where httpx goes, for every +pair below. ``_redirect_url`` is private on purpose: the test is a canary for +an httpx upgrade that changes redirect resolution, and should fail loudly then. +""" + +from __future__ import annotations + +import httpx +import pytest + +from fluid_build.observability.reporter import CredentialRedirectRefused, refuse_foreign_redirect + +pytestmark = [pytest.mark.unit] + +_DEFAULT_PORT = {"http": 80, "https": 443} + +_SOURCES = [ + "http://127.0.0.1:8001/a/b", + "https://cc.example.com:8443/a", + "http://cc.example.com/a", + "https://cc.example.com/a", + "http://[::1]:8001/a", + "https://[2001:db8::1]:8443/a", + "http://user:pw@cc.example.com:8080/a", # pragma: allowlist secret +] + +_LOCATIONS = [ + # hostless forms httpx completes with the request's host and no port + "http:/x", + "http:", + "http:///x", + "HTTP:/x", + "https:/x", + "https:", + "https:///x", + "hTtPs:", + "http:?q", + "http:#f", + "http:x", + "http:8002/x", + # backslashes, scheme-relative and userinfo tricks + "http:\\\\evil.com/x", + "/\\evil.com", + "\\\\evil.com", + "//evil.com", + "///evil.com", + "////evil.com", + "https://cc.example.com:8443@evil/", + "https://cc.example.com:8443@evil.com:8443/", + "http://evil.com\\@cc.example.com/", + "http://cc.example.com:80@evil.com/", + "//cc.example.com:8443@evil.com/", + # IPv6 and numeric host spellings + "http://[::1]:8001/", + "http://[::1]/", + "http://[0:0:0:0:0:0:0:1]:8001/", + "http://[::ffff:127.0.0.1]:8001/", + "http://[::ffff:a9fe:a9fe]/", + "http://[fe80::1%25en0]:8001/", + "http://127.0.0.1:8001/", + "http://127.1:8001/", + "http://0x7f000001:8001/", + "http://2130706433:8001/", + # case, trailing dot, ports and scheme changes + "http://CC.EXAMPLE.COM/", + "http://cc.example.com./", + "https://cc.example.com:443/", + "https://cc.example.com:8443/", + "https://cc.example.com:08443/", + "http://cc.example.com/", + "http://cc.example.com:8443/", + "https://cc.example.com/", + "http://cc.example.com:8080/", + "http://cc.example.com:80/", + "http://cc.example.com:0080/", + # whitespace, relative and odd schemes + "https://cc.example.com:8443\t.evil/", + " //evil.com", + "\t//evil.com", + "", + "?q=1", + "#x", + ".", + "..", + "../../../", + "ftp://cc.example.com/", + "file:///etc/passwd", + "javascript:alert(1)", + "data:,x", + "http://xn--e1afmkfd.com/", + "http://cc.example.com%2F@evil/", + "https://cc%2eexample%2ecom:8443/", + "https://cc.example.com:8443%40evil/", +] + + +def _origin(url: httpx.URL) -> tuple: + return (url.scheme, url.host, url.port or _DEFAULT_PORT.get(url.scheme)) + + +@pytest.mark.parametrize("source", _SOURCES) +def test_an_allowed_redirect_lands_where_the_hook_judged(source: str) -> None: + client = httpx.Client() + request = httpx.Request("GET", source) + src = _origin(request.url) + escaped = [] + for location in _LOCATIONS: + response = httpx.Response( + 302, headers=[(b"Location", location.encode("latin-1"))], request=request + ) + try: + refuse_foreign_redirect(response) + except CredentialRedirectRefused: + continue + try: + lands = _origin(client._redirect_url(request, response)) + except Exception: # noqa: BLE001 - httpx refuses to follow it either + continue + upgrade = src[0] == "http" and src[2] == 80 and lands == ("https", src[1], 443) + if lands != src and not upgrade: + escaped.append((location, lands)) + assert not escaped, f"hook allowed hops that httpx sends elsewhere: {escaped}" diff --git a/tests/observability/test_reporter_redirects.py b/tests/observability/test_reporter_redirects.py index 373a1b7d3..2b8314fc2 100644 --- a/tests/observability/test_reporter_redirects.py +++ b/tests/observability/test_reporter_redirects.py @@ -15,7 +15,7 @@ """A credential-bearing request never follows a redirect off its own origin. The Command Center reporter sends ``X-API-Key`` (and, for a bearer credential, -``Authorization``) as session headers. ``requests`` follows redirects by +``Authorization``) as session headers. ``httpx`` follows redirects by default and drops only ``Authorization`` when the host changes, so a Command Center (or anything in front of it) answering ``307 Location: `` used to receive the run, and the key, at the second address. Two real loopback @@ -28,13 +28,13 @@ from __future__ import annotations -import io import json import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, Dict, Iterator, List, Optional, Tuple import pytest +import respx pytestmark = pytest.mark.unit @@ -168,38 +168,18 @@ def test_the_reporter_still_follows_a_redirect_on_its_own_origin(two_origins): assert reporter.stats == {"sent": 2, "failed": 0} -class _TlsRedirect: - """A requests transport adapter standing in for the TLS hop: answers 307 to ``location``.""" - - def __init__(self, location: str) -> None: - self.location = location - - def send(self, request: Any, **_kwargs: Any) -> Any: - import requests - - response = requests.Response() - response.status_code = 307 - response.reason = "Temporary Redirect" - response.headers["Location"] = self.location - response.url = request.url - response.request = request - response.raw = io.BytesIO(b"") - response.connection = self - return response - - def close(self) -> None: - pass - - def test_the_reporter_does_not_follow_https_down_to_http_on_the_same_host(two_origins): _, plain = two_origins port = plain.url.rsplit(":", 1)[1] reporter = _reporter(f"https://127.0.0.1:{port}") # Only the TLS answer is staged; the http hop it redirects to is the real listener. - reporter.session.mount(f"https://127.0.0.1:{port}", _TlsRedirect(f"{plain.url}/api/v1/x")) - - _report_a_run(reporter) + with respx.mock(assert_all_called=False, assert_all_mocked=False) as router: + tls = router.route(scheme="https", host="127.0.0.1", port=int(port)).respond( + 307, headers={"Location": f"{plain.url}/api/v1/x"} + ) + _report_a_run(reporter) + assert tls.called assert plain.carrying(_KEY, _TOKEN) == [] assert plain.requests == [] assert reporter.stats["sent"] == 0 diff --git a/tests/test_circuit_breaker.py b/tests/test_circuit_breaker.py index 330d4f08b..fcba9f72e 100644 --- a/tests/test_circuit_breaker.py +++ b/tests/test_circuit_breaker.py @@ -149,10 +149,9 @@ def test_send_logs_noop_when_disabled(self): def test_send_logs_noop_when_empty_logs(self): config = self._make_config(configured=True) - with patch("fluid_build.observability.reporter.requests", MagicMock()): - reporter = CommandCenterReporter(config) - reporter.send_logs("123", []) - assert reporter.queue.empty() + reporter = CommandCenterReporter(config) + reporter.send_logs("123", []) + assert reporter.queue.empty() def test_send_metrics_noop_when_disabled(self): config = self._make_config(configured=False) @@ -160,8 +159,7 @@ def test_send_metrics_noop_when_disabled(self): reporter.send_metrics("123", [{"name": "duration", "value": 1.0}]) assert reporter.queue.empty() - @patch("fluid_build.observability.reporter.requests") - def test_enqueue_adds_to_queue(self, _mock_requests): + def test_enqueue_adds_to_queue(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) reporter._enqueue("POST", "/api/v1/test", {"key": "val"}) @@ -170,8 +168,7 @@ def test_enqueue_adds_to_queue(self, _mock_requests): assert event["method"] == "POST" assert event["endpoint"] == "/api/v1/test" - @patch("fluid_build.observability.reporter.requests") - def test_enqueue_drops_when_full(self, _mock_requests): + def test_enqueue_drops_when_full(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) reporter.queue = queue.Queue(maxsize=1) @@ -179,8 +176,7 @@ def test_enqueue_drops_when_full(self, _mock_requests): reporter._enqueue("POST", "/test2", {}) # should be dropped assert reporter.queue.qsize() == 1 - @patch("fluid_build.observability.reporter.requests") - def test_register_execution_with_git_info(self, _mock_requests): + def test_register_execution_with_git_info(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) reporter.register_execution( @@ -193,8 +189,7 @@ def test_register_execution_with_git_info(self, _mock_requests): assert event["payload"]["git_repo"] == "test" assert event["payload"]["git_commit"] == "abc123" - @patch("fluid_build.observability.reporter.requests") - def test_update_execution_payload(self, _mock_requests): + def test_update_execution_payload(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) reporter.update_execution( @@ -207,15 +202,13 @@ def test_update_execution_payload(self, _mock_requests): assert event["payload"]["status"] == "success" assert event["payload"]["progress"] == 100.0 - @patch("fluid_build.observability.reporter.requests") - def test_update_execution_empty_payload_not_enqueued(self, _mock_requests): + def test_update_execution_empty_payload_not_enqueued(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) reporter.update_execution(execution_id="abc") assert reporter.queue.empty() - @patch("fluid_build.observability.reporter.requests") - def test_send_event_post(self, _mock_requests): + def test_send_event_post(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) mock_response = MagicMock() @@ -227,8 +220,7 @@ def test_send_event_post(self, _mock_requests): reporter._send_event(event) reporter.session.post.assert_called_once() - @patch("fluid_build.observability.reporter.requests") - def test_send_event_patch(self, _mock_requests): + def test_send_event_patch(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) mock_response = MagicMock() @@ -239,8 +231,7 @@ def test_send_event_patch(self, _mock_requests): reporter._send_event(event) reporter.session.patch.assert_called_once() - @patch("fluid_build.observability.reporter.requests") - def test_send_event_unsupported_method_raises(self, _mock_requests): + def test_send_event_unsupported_method_raises(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) reporter.session = MagicMock() diff --git a/tests/test_command_center_client.py b/tests/test_command_center_client.py index b7ba3b99e..81eed64ee 100644 --- a/tests/test_command_center_client.py +++ b/tests/test_command_center_client.py @@ -16,7 +16,9 @@ from unittest.mock import MagicMock, mock_open, patch +import httpx import pytest +import respx from fluid_build.cli._command_center import ( CommandCenterClient, @@ -97,40 +99,27 @@ def test_detect_url_from_config_file(self, mock_path_cls, _mock_check): @patch.object(CommandCenterClient, "_detect_url", return_value="http://localhost:8000") def test_check_availability_success(self, _mock_detect): - mock_health_response = MagicMock() - mock_health_response.status_code = 200 - - mock_head_response = MagicMock() - mock_head_response.status_code = 200 - - mock_version_response = MagicMock() - mock_version_response.status_code = 200 - mock_version_response.json.return_value = {"version": "2.0.0"} - - with ( - patch( - "fluid_build.cli._command_center.requests.get", - side_effect=[mock_health_response, mock_version_response], - ), - patch( - "fluid_build.cli._command_center.requests.head", - return_value=mock_head_response, - ), - ): + with respx.mock(assert_all_called=False) as router: + router.get("http://localhost:8000/health").respond(200) + router.head(url__regex=r"http://localhost:8000/api/.*").respond(200) + router.get("http://localhost:8000/api/v1/version").respond( + 200, json={"version": "2.0.0"} + ) client = CommandCenterClient() assert client.available is True @patch.object(CommandCenterClient, "_detect_url", return_value="http://localhost:8000") def test_check_availability_failure(self, _mock_detect): - with patch( - "fluid_build.cli._command_center.requests.get", - side_effect=Exception("connection refused"), - ): + with respx.mock() as router: + router.get("http://localhost:8000/health").mock( + side_effect=httpx.ConnectError("connection refused") + ) client = CommandCenterClient() assert client.available is False @patch.object(CommandCenterClient, "_detect_url", return_value="http://localhost:8000") - def test_check_availability_requests_not_available(self, _mock_detect): - with patch("fluid_build.cli._command_center.REQUESTS_AVAILABLE", False): + def test_check_availability_health_not_ok(self, _mock_detect): + with respx.mock() as router: + router.get("http://localhost:8000/health").respond(503) client = CommandCenterClient() assert client.available is False diff --git a/tests/test_command_center_client_gate.py b/tests/test_command_center_client_gate.py index 9115db7a6..249a0e235 100644 --- a/tests/test_command_center_client_gate.py +++ b/tests/test_command_center_client_gate.py @@ -32,8 +32,8 @@ from typing import Any, Iterator, List, Tuple from unittest.mock import patch +import httpx import pytest -import requests from fluid_build.cli._command_center import CommandCenterClient @@ -347,7 +347,7 @@ def test_a_blueprint_get_does_not_follow_a_redirect_to_another_origin( cc, other = cc_redirecting_its_data api_url = _marketplace_url() - with pytest.raises(requests.exceptions.RequestException) as raised: + with pytest.raises(httpx.HTTPError) as raised: marketplace._marketplace_get(f"{api_url}{path}", logging.getLogger("t")) assert ("GET", f"{_MARKETPLACE}{path}") in cc.recorded @@ -389,7 +389,7 @@ def test_the_instantiate_post_does_not_resend_the_parameters_to_another_origin( cc, other = cc_redirecting_its_data api_url = _marketplace_url() - with pytest.raises(requests.exceptions.RequestException): + with pytest.raises(httpx.HTTPError): marketplace._marketplace_post( f"{api_url}/customer-360/instantiate", logging.getLogger("t"), @@ -441,7 +441,7 @@ async def _run(): try: return await connector._get_catalog_stats_impl() finally: - await connector.session.close() + await connector.session.aclose() stats = asyncio.run(_run()) @@ -460,10 +460,10 @@ def test_every_request_on_the_connector_session_refuses_the_foreign_redirect( async def _run(): connector = await _connected_catalog_connector() try: - async with connector.session.get(f"{connector.base_url}/search") as response: - return response.status + response = await connector.session.get(f"{connector.base_url}/search") + return response.status_code finally: - await connector.session.close() + await connector.session.aclose() with pytest.raises(CredentialRedirectRefused, match="another origin"): asyncio.run(_run()) @@ -488,7 +488,7 @@ async def _run(): try: return await connector._get_catalog_stats_impl() finally: - await connector.session.close() + await connector.session.aclose() try: assert asyncio.run(_run()) == {"total_products": 7} @@ -499,8 +499,8 @@ async def _run(): # ── a refused hop is never sent: no request, not even a connection ─────── # # The refusal is raised from the hook each library runs on a redirect response -# before it follows it: aiohttp's ``on_request_redirect`` trace signal for the -# catalog connector's session, requests' response hook for the marketplace. +# before it follows it: the ``response`` event hook (``refuse_foreign_redirect``) +# of the catalog connector's httpx AsyncClient and of the marketplace's client. # The second origin must see nothing at all, also for a 307 / 308, which # resends the method and the body (the operator's parameters). @@ -554,12 +554,12 @@ async def _run(): connector = await _connected_catalog_connector() try: kwargs = {} if method == "GET" else {"json": {"parameters": _PARAMETERS}} - async with connector.session.request( + response = await connector.session.request( method, f"{connector.base_url}/instantiate", **kwargs - ) as response: - return response.status + ) + return response.status_code finally: - await connector.session.close() + await connector.session.aclose() with pytest.raises(CredentialRedirectRefused, match="another origin"): asyncio.run(_run()) @@ -579,10 +579,8 @@ def test_a_marketplace_request_sends_nothing_to_the_redirect_target( url = f"{_marketplace_url()}/customer-360/instantiate" kwargs = {} if method == "get" else {"json": {"parameters": _PARAMETERS}} - with pytest.raises(requests.exceptions.RequestException, match="another origin"): - marketplace._marketplace_request( - getattr(requests, method), url, logging.getLogger("t"), **kwargs - ) + with pytest.raises(httpx.HTTPError, match="another origin"): + marketplace._marketplace_request(method.upper(), url, logging.getLogger("t"), **kwargs) assert (method.upper(), f"{_MARKETPLACE}/customer-360/instantiate") in cc.recorded assert other.recorded == [] and other.bodies == [] @@ -593,7 +591,7 @@ def test_the_second_origin_is_reachable_so_the_zero_above_is_real(cc_redirecting # The counting listener does see a connection and a request sent to it. _cc, other, accepted, _status = cc_redirecting_every_hop - assert requests.post(f"{other.url}/collect", json={"x": 1}, timeout=5).status_code == 200 + assert httpx.post(f"{other.url}/collect", json={"x": 1}, timeout=5).status_code == 200 assert other.recorded == [("POST", "/collect")] assert len(accepted) == 1 @@ -758,7 +756,7 @@ def test_a_named_registry_request_with_a_header_of_its_own_refuses_another_origi old, new = moved_registry - with pytest.raises(requests.exceptions.RequestException, match="another origin"): + with pytest.raises(httpx.HTTPError, match="another origin"): marketplace._marketplace_get( f"{old.url}{_MARKETPLACE}/x", logging.getLogger("t"), @@ -775,9 +773,9 @@ def test_a_named_registry_request_with_a_body_refuses_another_origin(moved_regis old, new = moved_registry - with pytest.raises(requests.exceptions.RequestException, match="another origin"): + with pytest.raises(httpx.HTTPError, match="another origin"): marketplace._marketplace_request( - getattr(requests, method), + method.upper(), f"{old.url}{_MARKETPLACE}/x/instantiate", logging.getLogger("t"), named_registry=True, @@ -890,7 +888,7 @@ def test_a_named_registry_request_of_the_operators_still_stops_after_a_same_orig old, new = sticky_moved_registry - with pytest.raises(requests.exceptions.RequestException, match="another origin"): + with pytest.raises(httpx.HTTPError, match="another origin"): marketplace._marketplace_get( f"{old.url}{_MARKETPLACE}/x", logging.getLogger("t"), named_registry=True, **kwargs ) @@ -952,7 +950,7 @@ def test_a_named_registry_through_an_authenticated_proxy_follows_its_move(authen def test_a_named_registry_request_with_a_key_through_the_proxy_still_stops(authenticated_proxy): from fluid_build.cli import marketplace - with pytest.raises(requests.exceptions.RequestException, match="another origin"): + with pytest.raises(httpx.HTTPError, match="another origin"): marketplace._marketplace_get( f"{_PROXIED_OLD}{_MARKETPLACE}/x", logging.getLogger("t"), diff --git a/tests/test_command_center_ssrf.py b/tests/test_command_center_ssrf.py index fdb72bfa1..be319f3a5 100644 --- a/tests/test_command_center_ssrf.py +++ b/tests/test_command_center_ssrf.py @@ -27,6 +27,8 @@ from unittest.mock import MagicMock, patch +import respx + from fluid_build.cli._command_center import ( CommandCenterClient, ) @@ -152,31 +154,35 @@ def test_reporter_allowlist_opts_internal_back_in(self, monkeypatch): class TestClientCheckAvailabilityGate: def test_check_availability_refuses_metadata_url(self, monkeypatch): - """``_check_availability`` must bail before any ``requests`` - call when the configured URL is the metadata service.""" + """``_check_availability`` must bail before any HTTP request + when the configured URL is the metadata service.""" monkeypatch.setenv("FLUID_COMMAND_CENTER_URL", "http://169.254.169.254") - with patch("fluid_build.cli._command_center.requests") as mock_requests: + with respx.mock(assert_all_called=False) as router: + # A catch-all that answers 200: any attempted request is recorded + # (an unmocked request would be an error the client swallows, so + # ``router.calls`` would stay empty and prove nothing). + router.route().respond(200) client = CommandCenterClient() assert client.available is False # No HTTP probe should have been issued. - assert not mock_requests.get.called - assert not mock_requests.head.called + assert not router.calls def test_check_availability_refuses_rfc1918_url(self, monkeypatch): monkeypatch.setenv("FLUID_COMMAND_CENTER_URL", "http://10.0.0.9:8000") - with patch("fluid_build.cli._command_center.requests") as mock_requests: + with respx.mock(assert_all_called=False) as router: + router.route().respond(200) # records any attempted request client = CommandCenterClient() assert client.available is False - assert not mock_requests.get.called + assert not router.calls def test_check_availability_probes_localhost_default(self, monkeypatch): """Localhost (the default) is allowed — the probe IS issued.""" monkeypatch.delenv("FLUID_COMMAND_CENTER_URL", raising=False) monkeypatch.setenv("FLUID_DISABLE_CC_DETECTION", "false") - with patch("fluid_build.cli._command_center.requests") as mock_requests: - mock_requests.get.return_value = MagicMock(status_code=500) - mock_requests.exceptions = __import__("requests").exceptions # real exception classes + with respx.mock(assert_all_called=False) as router: + health = router.get(url__regex=r"http://[^/]+/health").respond(500) client = CommandCenterClient() # Localhost default must reach the probe (status 500 → unavailable, # but the request WAS attempted, proving the gate let it through). - assert mock_requests.get.called + assert health.called + assert client.available is False diff --git a/tests/test_datamesh_manager_publish_spec.py b/tests/test_datamesh_manager_publish_spec.py index 1d5084047..f0f9d6b6b 100644 --- a/tests/test_datamesh_manager_publish_spec.py +++ b/tests/test_datamesh_manager_publish_spec.py @@ -15,8 +15,9 @@ from types import SimpleNamespace from unittest.mock import MagicMock, patch +import httpx import pytest -import requests +import respx from fluid_build.cli.datamesh_manager import _cmd_publish, _publish_exit_code from fluid_build.providers.base import ProviderError @@ -704,14 +705,14 @@ def test_cmd_publish_fail_on_contract_error_returns_non_zero(): def test_request_wraps_retry_error_as_provider_error(): provider = DataMeshManagerProvider(api_key="dummy", api_url="https://api.entropy-data.com") - session = MagicMock() - session.request.side_effect = requests.exceptions.RetryError("too many 500 responses") - provider._session_instance = session - with pytest.raises(ProviderError) as excinfo: - provider._request("PUT", "/api/datacontracts/x.y", json_body={"id": "x.y"}) + with respx.mock() as router: + route = router.put("https://api.entropy-data.com/api/datacontracts/x.y").respond(500) + with pytest.raises(ProviderError) as excinfo: + provider._request("PUT", "/api/datacontracts/x.y", json_body={"id": "x.y"}) assert "HTTP request failed" in str(excinfo.value) + assert route.call_count > 1 # the 500s were retried before giving up def test_provider_rejects_plain_http_remote_api_url(): @@ -723,13 +724,13 @@ def test_provider_rejects_plain_http_remote_api_url(): def test_request_redacts_secret_like_error_body(): provider = DataMeshManagerProvider(api_key="dummy", api_url="https://api.entropy-data.com") - response = MagicMock(status_code=403, text='{"api_key":"ed_live_supersecret"}') - session = MagicMock() - session.request.return_value = response - provider._session_instance = session - with pytest.raises(ProviderError) as excinfo: - provider._request("PUT", "/api/dataproducts/x", json_body={"id": "x"}) + with respx.mock() as router: + router.put("https://api.entropy-data.com/api/dataproducts/x").mock( + return_value=httpx.Response(403, text='{"api_key":"ed_live_supersecret"}') + ) + with pytest.raises(ProviderError) as excinfo: + provider._request("PUT", "/api/dataproducts/x", json_body={"id": "x"}) message = str(excinfo.value) assert "ed_live_supersecret" not in message diff --git a/tests/test_http_client.py b/tests/test_http_client.py new file mode 100644 index 000000000..ed59afa6b --- /dev/null +++ b/tests/test_http_client.py @@ -0,0 +1,252 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""fluid_build.util.http_client: the redirect and retry semantics ``requests`` had. + +A real threaded loopback server, not a mock: the point is what goes over the wire. +""" + +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import httpx +import pytest + +from fluid_build.observability.reporter import CredentialRedirectRefused, refuse_foreign_redirect +from fluid_build.util.http_client import ( + MAX_REDIRECTS, + RetryError, + RetryTransport, + new_async_client, + new_client, + tls_verify, +) + + +class _Handler(BaseHTTPRequestHandler): + hits: dict = {} + script: dict = {} + + def log_message(self, *a): # silence + pass + + def _serve(self): + type(self).hits[self.path] = type(self).hits.get(self.path, 0) + 1 + n = type(self).hits[self.path] + length = int(self.headers.get("content-length") or 0) + body = self.rfile.read(length) if length else b"" + steps = type(self).script.get(self.path, [(200, {}, b"ok")]) + status, headers, payload = steps[min(n, len(steps)) - 1] + if status == "reset": + self.connection.close() + return + if self.path == "/echo": + payload = json.dumps({"method": self.command, "body": body.decode()}).encode() + self.send_response(status) + for k, v in headers.items(): + self.send_header(k, v) + self.send_header("content-length", str(len(payload))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(payload) + + do_GET = do_POST = do_PUT = do_HEAD = do_DELETE = _serve + + +@pytest.fixture() +def server(): + _Handler.hits = {} + _Handler.script = {} + srv = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + t = threading.Thread(target=srv.serve_forever, daemon=True) + t.start() + srv.base = f"http://127.0.0.1:{srv.server_address[1]}" + yield srv + srv.shutdown() + srv.server_close() + + +def test_get_with_timeout_and_post_json(server): + with new_client(timeout=5) as c: + assert c.get(server.base + "/x").text == "ok" + r = c.post(server.base + "/echo", json={"a": 1}) + assert json.loads(r.json()["body"]) == {"a": 1} + + +def test_redirects_are_followed_by_default_and_can_be_turned_off(server): + _Handler.script = {"/a": [(302, {"location": "/b"}, b"")], "/b": [(200, {}, b"landed")]} + with new_client() as c: + assert c.get(server.base + "/a").text == "landed" + with new_client(follow_redirects=False) as c: + assert c.get(server.base + "/a").status_code == 302 + + +def test_redirect_limit_is_requests_30_not_httpx_20(server): + assert MAX_REDIRECTS == 30 + _Handler.script = {"/loop": [(302, {"location": "/loop"}, b"")]} + with new_client() as c, pytest.raises(httpx.TooManyRedirects): + c.get(server.base + "/loop") + assert _Handler.hits["/loop"] == 31 + + +def test_post_302_becomes_get_like_requests(server): + _Handler.script = {"/p": [(302, {"location": "/echo"}, b"")]} + with new_client() as c: + assert c.post(server.base + "/p", json={"k": 1}).json()["method"] == "GET" + + +def test_response_hook_runs_on_every_hop_and_can_refuse(server): + other = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + threading.Thread(target=other.serve_forever, daemon=True).start() + try: + target = f"http://127.0.0.1:{other.server_address[1]}/secret" + _Handler.script = {"/go": [(302, {"location": target}, b"")]} + with new_client(response_hooks=[refuse_foreign_redirect]) as c: + with pytest.raises(CredentialRedirectRefused, match="another origin"): + c.get(server.base + "/go") + assert "/secret" not in _Handler.hits + finally: + other.shutdown() + other.server_close() + + +def test_same_origin_redirect_passes_the_hook(server): + _Handler.script = {"/a": [(302, {"location": "/b"}, b"")]} + with new_client(response_hooks=[refuse_foreign_redirect]) as c: + assert c.get(server.base + "/a").status_code == 200 + + +def test_async_client_refuses_with_the_real_exception(server): + import asyncio + + other = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + threading.Thread(target=other.serve_forever, daemon=True).start() + try: + target = f"http://127.0.0.1:{other.server_address[1]}/secret" + _Handler.script = {"/go": [(302, {"location": target}, b"")]} + + async def run(): + async with new_async_client(response_hooks=[refuse_foreign_redirect]) as c: + await c.get(server.base + "/go") + + with pytest.raises(CredentialRedirectRefused): + asyncio.run(run()) + assert "/secret" not in _Handler.hits + finally: + other.shutdown() + other.server_close() + + +def _retrying(**kw): + sleeps: list = [] + opts = {"total": 3, "backoff_factor": 0.5, "status_forcelist": [429, 500, 502, 503, 504]} + opts.update(kw) + return new_client(retry={**opts, "sleep": sleeps.append}), sleeps + + +def test_get_retries_a_503_then_succeeds_with_urllib3_backoff(server): + _Handler.script = {"/f": [(503, {}, b""), (503, {}, b""), (503, {}, b""), (200, {}, b"up")]} + c, sleeps = _retrying() + with c: + assert c.get(server.base + "/f").text == "up" + # urllib3: no sleep before the first retry, then factor * 2 ** (n - 1) + assert sleeps == [1.0, 2.0] + assert _Handler.hits["/f"] == 4 + + +def test_exhausted_status_retries_raise_like_requests_retryerror(server): + _Handler.script = {"/f": [(500, {}, b"")]} + c, _ = _retrying() + with c, pytest.raises(RetryError, match="500"): + c.get(server.base + "/f") + assert _Handler.hits["/f"] == 4 # 1 + total retries + + +def test_post_is_not_retried_when_not_an_allowed_method(server): + _Handler.script = {"/f": [(503, {}, b""), (200, {}, b"up")]} + c, _ = _retrying(allowed_methods=["GET", "PUT", "DELETE"]) + with c: + assert c.post(server.base + "/f", json={}).status_code == 503 + assert _Handler.hits["/f"] == 1 + + +def test_retry_after_is_honoured_on_429_and_503_only(server): + _Handler.script = { + "/a": [(429, {"retry-after": "7"}, b""), (200, {}, b"")], + "/b": [(500, {"retry-after": "7"}, b""), (200, {}, b"")], + } + c, sleeps = _retrying() + with c: + c.get(server.base + "/a") + c.get(server.base + "/b") + assert sleeps == [7.0] # /b: first retry, no backoff, header ignored on a 500 + + +def test_connection_reset_on_get_is_retried(server): + _Handler.script = {"/r": [("reset", {}, b""), (200, {}, b"back")]} + c, _ = _retrying() + with c: + assert c.get(server.base + "/r").text == "back" + assert _Handler.hits["/r"] == 2 + + +def test_connect_failure_is_retried_for_every_method_then_raises(): + attempts = [] + + class Refuse(httpx.BaseTransport): + def handle_request(self, request): + attempts.append(request.method) + raise httpx.ConnectError("refused", request=request) + + t = RetryTransport(total=2, transport=Refuse(), sleep=lambda s: None) + with httpx.Client(transport=t) as c, pytest.raises(httpx.ConnectError): + c.post("http://example.invalid/") + assert attempts == ["POST"] * 3 + + +def test_read_failure_on_a_non_allowed_method_is_not_retried(): + attempts = [] + + class Drop(httpx.BaseTransport): + def handle_request(self, request): + attempts.append(1) + raise httpx.ReadError("reset", request=request) + + t = RetryTransport(total=3, allowed_methods=["GET"], transport=Drop(), sleep=lambda s: None) + with httpx.Client(transport=t) as c, pytest.raises(httpx.ReadError): + c.post("http://example.invalid/") + assert len(attempts) == 1 + + +def test_retry_wraps_proxy_transports_too(monkeypatch): + monkeypatch.setenv("HTTPS_PROXY", "http://127.0.0.1:9") + monkeypatch.delenv("NO_PROXY", raising=False) + with new_client(retry={"total": 1}) as c: + mounts = [v for v in c._mounts.values() if v is not None] + assert mounts and all(isinstance(m, RetryTransport) for m in mounts) + assert isinstance(c._transport, RetryTransport) + + +def test_tls_verify_defaults_on_and_honours_requests_ca_bundle_env(monkeypatch, tmp_path): + import ssl + + monkeypatch.delenv("REQUESTS_CA_BUNDLE", raising=False) + monkeypatch.delenv("CURL_CA_BUNDLE", raising=False) + assert tls_verify() is True + monkeypatch.setenv("REQUESTS_CA_BUNDLE", str(tmp_path)) + ctx = tls_verify() + assert isinstance(ctx, ssl.SSLContext) and ctx.verify_mode == ssl.CERT_REQUIRED diff --git a/tests/test_http_client_hardening.py b/tests/test_http_client_hardening.py new file mode 100644 index 000000000..fe0846703 --- /dev/null +++ b/tests/test_http_client_hardening.py @@ -0,0 +1,567 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hardening of the shared HTTP client (security review of the httpx port). + +* a redirect ``Location`` with a scheme but no host never escapes the origin gate; +* ``Retry-After`` is capped and a huge value does not crash; +* the Data Mesh Manager ``x-api-key`` does not follow a redirect off its origin; +* an unusable CA bundle does not crash reporter construction, nor relax TLS; +* the catalog connector keeps trusting the system CA store. +""" + +from __future__ import annotations + +import asyncio +import logging +import ssl +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from types import SimpleNamespace +from typing import Any, Dict, List + +import httpx +import pytest + +from fluid_build.observability.reporter import CredentialRedirectRefused, refuse_foreign_redirect +from fluid_build.util.http_client import ( + new_async_client, + new_client, + parse_retry_after, +) + +pytestmark = pytest.mark.unit + +_KEY = "dmm-hardening-key-not-a-secret" # pragma: allowlist secret + + +class _Server: + def __init__(self) -> None: + self.log: List[Dict[str, Any]] = [] + self.routes: Dict[str, Any] = {} + outer = self + + class H(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def _do(self): + n = int(self.headers.get("content-length") or 0) + body = self.rfile.read(n).decode() if n else "" + outer.log.append( + { + "method": self.command, + "path": self.path, + "headers": {k.lower(): v for k, v in self.headers.items()}, + "body": body, + } + ) + status, headers = outer.routes.get(self.path.split("?")[0], (200, {})) + self.send_response(status) + for k, v in headers.items(): + self.send_header(k, v) + self.send_header("content-length", "2") + self.end_headers() + if self.command != "HEAD": + self.wfile.write(b"{}") + + do_GET = do_POST = do_PUT = do_DELETE = do_HEAD = _do + + self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), H) + self.base = f"http://127.0.0.1:{self.httpd.server_address[1]}" + threading.Thread(target=self.httpd.serve_forever, daemon=True).start() + + def close(self) -> None: + self.httpd.shutdown() + self.httpd.server_close() + + +@pytest.fixture() +def two_servers(): + a, b = _Server(), _Server() + try: + yield a, b + finally: + a.close() + b.close() + + +_HOSTLESS = ["https:/stolen", "http:", "http:///stolen", "HTTP:/stolen", "https:stolen"] + + +# ── 1. hostless redirect ────────────────────────────────────────────────── + + +@pytest.mark.parametrize("location", _HOSTLESS) +def test_hostless_location_is_refused_on_a_mock_transport_sync(location): + seen: List[httpx.Request] = [] + + def handler(req: httpx.Request) -> httpx.Response: + seen.append(req) + if req.url.port == 8443: + return httpx.Response(307, headers={"Location": location}) + return httpx.Response(200) + + with new_client( + headers={"X-API-Key": "k"}, + response_hooks=[refuse_foreign_redirect], + transport=httpx.MockTransport(handler), + ) as c: + with pytest.raises(CredentialRedirectRefused): + c.post("https://cc.example.com:8443/api/events", json={"r": 1}) + assert [r.url.port for r in seen] == [8443] + + +@pytest.mark.parametrize("location", _HOSTLESS) +def test_hostless_location_is_refused_on_a_mock_transport_async(location): + seen: List[httpx.Request] = [] + + def handler(req: httpx.Request) -> httpx.Response: + seen.append(req) + return httpx.Response(307, headers={"Location": location}) + + async def run(): + async with new_async_client( + response_hooks=[refuse_foreign_redirect], transport=httpx.MockTransport(handler) + ) as c: + await c.post("https://cc.example.com:8443/api/events", json={"r": 1}) + + with pytest.raises(CredentialRedirectRefused): + asyncio.run(run()) + assert len(seen) == 1 + + +@pytest.mark.parametrize("location", _HOSTLESS) +def test_hostless_location_is_refused_against_a_real_server_sync_and_async(two_servers, location): + a, b = two_servers + a.routes["/go"] = (307, {"Location": location}) + with new_client(response_hooks=[refuse_foreign_redirect]) as c: + with pytest.raises(CredentialRedirectRefused): + c.post(a.base + "/go", json={"r": 1}, headers={"X-API-Key": "k"}) + + async def run(): + async with new_async_client(response_hooks=[refuse_foreign_redirect]) as c: + await c.get(a.base + "/go", headers={"X-API-Key": "k"}) + + with pytest.raises(CredentialRedirectRefused): + asyncio.run(run()) + assert [r["path"] for r in a.log] == ["/go", "/go"] + assert b.log == [] + + +def test_legitimate_redirects_are_still_followed(two_servers): + a, _ = two_servers + a.routes["/rel"] = (302, {"Location": "/landed"}) + a.routes["/abs"] = (302, {"Location": a.base + "/landed"}) + a.routes["/dotrel"] = (302, {"Location": "../landed"}) + with new_client(response_hooks=[refuse_foreign_redirect]) as c: + for p in ("/rel", "/abs", "/dotrel"): + assert c.get(a.base + p).status_code == 200 + + +def test_default_port_is_filled_in_when_comparing_origins(): + def handler(req: httpx.Request) -> httpx.Response: + if req.url.path == "/start": + return httpx.Response(302, headers={"Location": "https://cc.example.com:443/next"}) + return httpx.Response(200) + + with new_client( + response_hooks=[refuse_foreign_redirect], transport=httpx.MockTransport(handler) + ) as c: + assert c.get("https://cc.example.com/start").status_code == 200 + + +# ── 2. Retry-After ──────────────────────────────────────────────────────── + + +def _retrying_mock(retry_after: str): + sleeps: List[float] = [] + calls: List[str] = [] + + def handler(req: httpx.Request) -> httpx.Response: + calls.append(req.method) + return httpx.Response(429, headers={"Retry-After": retry_after}) + + c = new_client() + # ``transport=`` would bypass the retry wrapper; install it as the factory does. + from fluid_build.util.http_client import RetryTransport + + c._transport = RetryTransport( + transport=httpx.MockTransport(handler), + total=3, + backoff_factor=0.5, + status_forcelist=[429], + allowed_methods=["GET"], + sleep=sleeps.append, + ) + return c, sleeps, calls + + +def test_a_huge_retry_after_is_capped_at_the_backoff_ceiling(): + c, sleeps, calls = _retrying_mock("86400000") + with c, pytest.raises(httpx.TransportError): + c.get("http://dmm.example/api/x") + assert sleeps and max(sleeps) == 120.0 + assert len(calls) == 4 + + +def test_a_small_retry_after_is_honoured_exactly(): + c, sleeps, _ = _retrying_mock("7") + with c, pytest.raises(httpx.TransportError): + c.get("http://dmm.example/api/x") + assert sleeps == [7.0, 7.0, 7.0] + + +def test_a_400_digit_retry_after_does_not_raise(): + assert parse_retry_after("9" * 400) is None + c, sleeps, _ = _retrying_mock("9" * 400) + with c, pytest.raises(httpx.TransportError): + c.get("http://dmm.example/api/x") + assert sleeps == [1.0, 2.0] # unparseable: plain backoff + + +def test_an_http_date_retry_after_still_works(): + from datetime import datetime, timedelta, timezone + from email.utils import format_datetime + + when = datetime.now(timezone.utc) + timedelta(seconds=30) + got = parse_retry_after(format_datetime(when, usegmt=True)) + assert got is not None and 20 <= got <= 31 + assert parse_retry_after("Fri, 31 Dec 9999 23:59:59 GMT") is not None + + +def test_the_catalog_registrar_retry_cap_still_holds(): + from fluid_build.build_runners.catalog_registrars._http_retry import ( + DEFAULT_MAX_DELAY, + RetryPolicy, + compute_delay, + ) + + assert compute_delay(1, RetryPolicy(), 86400000.0) == DEFAULT_MAX_DELAY + + +# ── 3. Data Mesh Manager redirect ───────────────────────────────────────── + + +@pytest.fixture() +def dmm(two_servers, monkeypatch): + a, b = two_servers + monkeypatch.setenv("DMM_API_URL", a.base) + monkeypatch.setenv("DMM_ALLOW_INSECURE_HTTP", "1") + from fluid_build.providers.datamesh_manager.datamesh_manager import DataMeshManagerProvider + + return DataMeshManagerProvider(api_key=_KEY), a, b + + +@pytest.mark.parametrize( + "method,status,body", + [("GET", 302, None), ("PUT", 302, {"a": 1}), ("PUT", 307, {"payload": "body"})], +) +def test_dmm_does_not_send_its_key_or_body_to_another_origin(dmm, method, status, body): + from fluid_build.providers.base import ProviderError + + prov, a, b = dmm + a.routes["/api/dataproducts/x"] = (status, {"Location": f"{b.base}/landed"}) + with pytest.raises(ProviderError) as ei: + prov._request(method, "/api/dataproducts/x", json_body=body) + msg = str(ei.value) + assert b.base.replace("http://", "") in msg and "DMM_API_URL" in msg + assert b.log == [] + assert len([r for r in a.log if r["path"] == "/api/dataproducts/x"]) == 1 # not retried + assert _KEY not in msg + + +def test_dmm_still_follows_a_same_origin_redirect(dmm): + prov, a, b = dmm + a.routes["/api/dataproducts/x"] = (307, {"Location": "/api/moved"}) + assert prov._request("GET", "/api/dataproducts/x").status_code == 200 + assert b.log == [] + + +# ── 4. missing CA bundle ────────────────────────────────────────────────── + + +@pytest.mark.parametrize("var", ["REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"]) +def test_a_missing_ca_bundle_does_not_crash_reporter_construction( + monkeypatch, tmp_path, caplog, var +): + from fluid_build.observability.config import CommandCenterConfig + from fluid_build.observability.reporter import CommandCenterReporter + + monkeypatch.delenv("REQUESTS_CA_BUNDLE", raising=False) + monkeypatch.delenv("CURL_CA_BUNDLE", raising=False) + monkeypatch.setenv(var, str(tmp_path / "does-not-exist.pem")) + cfg = CommandCenterConfig(url="https://cc.example.com", api_key=_KEY) + with caplog.at_level(logging.WARNING): + reporter = CommandCenterReporter(cfg) + assert reporter.enabled is False and reporter.session is None + reporter.start() + assert not reporter.running + assert var in caplog.text + + +def test_a_missing_ca_bundle_never_falls_back_to_unverified_tls(monkeypatch, tmp_path): + from fluid_build.util.http_client import CaBundleError + + monkeypatch.setenv("REQUESTS_CA_BUNDLE", str(tmp_path / "nope.pem")) + with pytest.raises(CaBundleError, match="REQUESTS_CA_BUNDLE"): + new_client() + + +# ── 5. system trust store for the catalog connector ────────────────────── + + +def test_system_trust_client_uses_the_default_verify_paths(monkeypatch): + real = ssl.create_default_context + calls: List[Any] = [] + + def spy(*a, **kw): + calls.append((a, kw)) + return real(*a, **kw) + + monkeypatch.setattr("fluid_build.util.http_client.ssl.create_default_context", spy) + client = new_async_client(trust_env=False, system_trust=True) + assert calls == [((), {})] # no cafile / capath: the system defaults + ctx = client._transport._pool._ssl_context + assert ctx.verify_mode == ssl.CERT_REQUIRED and ctx.check_hostname + asyncio.run(client.aclose()) + + +def test_the_catalog_connector_asks_for_system_trust(monkeypatch): + from fluid_build.cli.market_catalogs.command_center import CommandCenterConnector + + cc = SimpleNamespace( + available=True, + features=SimpleNamespace(catalog=True), + get_catalog_url=lambda: "https://cc.example.com/api/catalog", + ) + monkeypatch.setattr( + "fluid_build.cli._command_center.get_command_center_client", lambda **kw: cc + ) + seen: Dict[str, Any] = {} + real = new_async_client + + def spy(**kw): + seen.update(kw) + return real(**kw) + + monkeypatch.setattr("fluid_build.util.http_client.new_async_client", spy) + + async def run(): + conn = CommandCenterConnector({}, logging.getLogger("t")) + assert await conn._connect_impl() is True + await conn.session.aclose() + + asyncio.run(run()) + assert seen["system_trust"] is True + + +# ── 6. a non-JSON 200 from the marketplace is an error, not a crash ─────── + +_HTML = "not json" +_BP = "not-a-bundled-blueprint" + + +def _args(**kw): + base = dict( + query="zzz-matches-no-bundled-blueprint", + category=None, + tags=None, + maturity=None, + state="published", + sort="downloads", + limit=20, + format="json", + blueprint_id=_BP, + version=None, + params='{"a": 1}', + interactive=False, + output=None, + submit=False, + ) + base.update(kw) + return SimpleNamespace(**base) + + +def test_search_with_a_non_json_200_reports_instead_of_crashing(capsys): + import respx + + from fluid_build.cli.marketplace import search_blueprints + + with respx.mock(assert_all_called=False) as router: + router.get("http://api").respond(200, text=_HTML) + rc = search_blueprints(_args(), logging.getLogger("t"), "http://api") + assert rc == 1 + assert capsys.readouterr().out.strip() == "[]" + + +def test_info_with_a_non_json_200_reports_instead_of_crashing(): + import respx + + from fluid_build.cli.marketplace import show_blueprint_info + + with respx.mock(assert_all_called=False) as router: + router.get(f"http://api/{_BP}").respond(200, text=_HTML) + assert show_blueprint_info(_args(), logging.getLogger("t"), "http://api") == 1 + + +def test_instantiate_with_a_non_json_200_reports_instead_of_crashing(): + import respx + + from fluid_build.cli.marketplace import instantiate_blueprint + + with respx.mock(assert_all_called=False) as router: + router.get(f"http://api/{_BP}").respond(200, text=_HTML) + assert instantiate_blueprint(_args(), logging.getLogger("t"), "http://api") == 1 + with respx.mock(assert_all_called=False) as router: + router.get(f"http://api/{_BP}").respond(200, json={"id": _BP, "name": "X"}) + router.post(f"http://api/{_BP}/instantiate").respond(200, text=_HTML) + assert instantiate_blueprint(_args(), logging.getLogger("t"), "http://api") == 1 + + +# ── 7. params= merge into the URL's own query string ────────────────────── + + +def test_merge_query_keeps_the_urls_own_pairs_and_empty_params_keep_them_all(): + from fluid_build.util.http_client import merge_query + + out = httpx.URL(merge_query("https://reg.example/api?tenant=acme", {"limit": 1})) + assert dict(out.params) == {"tenant": "acme", "limit": "1"} + assert merge_query("https://reg.example/api?tenant=acme", {}) == ( + "https://reg.example/api?tenant=acme" + ) + + +def test_marketplace_search_keeps_the_registry_urls_query(): + import respx + + from fluid_build.cli.marketplace import NamedRegistryURL, search_blueprints + + url = NamedRegistryURL("https://reg.example/api/v1/blueprints?tenant=acme") + with respx.mock(assert_all_called=False) as router: + route = router.get(host="reg.example").respond(200, json={"items": [], "total": 0}) + search_blueprints(_args(format=None, query="q"), logging.getLogger("t"), url) + params = route.calls.last.request.url.params + assert params["tenant"] == "acme" and params["limit"] == "20" + + +def test_command_center_endpoint_probe_keeps_the_endpoints_query(monkeypatch): + import respx + + from fluid_build.cli._command_center import CommandCenterClient + + monkeypatch.setenv("FLUID_DISABLE_CC_DETECTION", "true") + client = CommandCenterClient() + client.url = "https://cc.example.com" + with respx.mock(assert_all_called=False) as router: + router.head(host="cc.example.com").respond(500) + get = router.get(host="cc.example.com").respond(200) + assert client._check_endpoint("/api/v1/x?tenant=acme") is True + params = get.calls.last.request.url.params + assert params["tenant"] == "acme" and params["limit"] == "1" + + +# ── 8. safe_request: errors are 4xx/5xx only; requests-only kwargs ──────── + + +def _safe(status, headers=None, **kw): + from fluid_build.util.network import safe_request + + def handler(req): + return httpx.Response(status, headers=headers or {}) + + session = httpx.Client(transport=httpx.MockTransport(handler)) + return safe_request("GET", "http://svc.example/x", session=session, **kw) + + +@pytest.mark.parametrize("status", [304, 301]) +def test_safe_request_does_not_raise_on_a_non_error_status(status): + assert _safe(status).status_code == status + + +def test_safe_request_still_raises_on_4xx_and_5xx(): + from fluid_build.errors import NetworkError + + for status in (404, 503): + with pytest.raises(NetworkError): + _safe(status) + + +def test_safe_request_translates_allow_redirects(): + from fluid_build.util.network import safe_request + + def handler(req): + if req.url.path == "/a": + return httpx.Response(302, headers={"Location": "/b"}) + return httpx.Response(200) + + with httpx.Client(transport=httpx.MockTransport(handler)) as s: + assert ( + safe_request("GET", "http://s/a", session=s, allow_redirects=False).status_code == 302 + ) + assert safe_request("GET", "http://s/a", session=s, allow_redirects=True).status_code == 200 + + +@pytest.mark.parametrize("name", ["verify", "cert", "stream"]) +def test_safe_request_refuses_requests_only_kwargs_loudly(name): + with pytest.raises(TypeError, match=name): + _safe(200, **{name: True}) + + +# ── 9. the catalog connector reads the real SearchFilters ───────────────── + + +def test_catalog_connector_search_uses_the_real_search_filters_and_keeps_the_query(): + from fluid_build.cli.market import ( + DataProductLayer, + DataProductStatus, + SearchFilters, + ) + from fluid_build.cli.market_catalogs.command_center import CommandCenterConnector + + seen: List[httpx.Request] = [] + + def handler(req): + seen.append(req) + return httpx.Response(200, json={"items": []}) + + async def run(): + conn = CommandCenterConnector({}, logging.getLogger("t")) + conn.base_url = "https://cc.example.com/api/catalog?tenant=acme" + conn.session = new_async_client(transport=httpx.MockTransport(handler)) + filters = SearchFilters( + text_query="orders", + domain="sales", + owner="bob", + layer=DataProductLayer.GOLD, + status=DataProductStatus.ACTIVE, + tags=["a", "b"], + ) + try: + return await conn._search_data_products_impl(filters) + finally: + await conn.session.aclose() + + assert asyncio.run(run()) == [] + params = seen[0].url.params + assert dict(params) == { + "tenant": "acme", + "query": "orders", + "domains": "sales", + "owners": "bob", + "tags": "a,b", + "layers": "gold", + "statuses": "active", + "limit": "50", + } diff --git a/tests/test_market_bundled_blueprints.py b/tests/test_market_bundled_blueprints.py index fc0920325..28d715621 100644 --- a/tests/test_market_bundled_blueprints.py +++ b/tests/test_market_bundled_blueprints.py @@ -180,22 +180,22 @@ def test_search_lists_bundled_with_no_registry(): ) with ( patch.object(marketplace_module, "console", MagicMock()), - patch.object(marketplace_module, "requests") as mock_requests, + patch("fluid_build.util.http_client.new_client") as mock_new_client, ): result = marketplace_module.search_blueprints(args, _logger(), None) assert result == 0 - mock_requests.get.assert_not_called() # api_url=None → registry skipped + mock_new_client.assert_not_called() # api_url=None → registry skipped def test_info_serves_bundled_offline(): args = SimpleNamespace(blueprint_id="fluid.starter") with ( patch.object(marketplace_module, "console", MagicMock()), - patch.object(marketplace_module, "requests") as mock_requests, + patch("fluid_build.util.http_client.new_client") as mock_new_client, ): result = marketplace_module.show_blueprint_info(args, _logger(), None) assert result == 0 - mock_requests.get.assert_not_called() + mock_new_client.assert_not_called() def test_instantiate_bundled_offline_writes_valid_contract(tmp_path): @@ -209,12 +209,11 @@ def test_instantiate_bundled_offline_writes_valid_contract(tmp_path): ) with ( patch.object(marketplace_module, "console", MagicMock()), - patch.object(marketplace_module, "requests") as mock_requests, + patch("fluid_build.util.http_client.new_client") as mock_new_client, ): result = marketplace_module.instantiate_blueprint(args, _logger(), None) assert result == 0 - mock_requests.get.assert_not_called() - mock_requests.post.assert_not_called() + mock_new_client.assert_not_called() written = json.loads(out.read_text()) assert written["id"] == "ops.offline-product" _assert_valid_074(written) diff --git a/tests/test_market_connectors.py b/tests/test_market_connectors.py index ef86e9903..7a390bd87 100644 --- a/tests/test_market_connectors.py +++ b/tests/test_market_connectors.py @@ -1034,6 +1034,13 @@ def test_status_string_not_list(self): # --------------------------------------------------------------------------- +def _cc_filters(query): + """A real ``SearchFilters`` (the connector must read its real field names).""" + from fluid_build.cli.market import SearchFilters + + return SearchFilters(text_query=query) + + class TestCommandCenterConnectorConnect: """``_connect_impl`` reaches ``fluid_build.cli._command_center``. @@ -1054,40 +1061,132 @@ def _client(self, *, catalog=True): cc.get_catalog_url.return_value = self._URL return cc - def _connect(self, cc): + def _connect(self, cc, *, then=None): + """Connect the real connector; ``then`` (a coroutine function taking the + connector) runs on the same loop, before the session is closed.""" from fluid_build.cli.market_catalogs.command_center import CommandCenterConnector + from fluid_build.util import http_client connector = CommandCenterConnector({}, logging.getLogger("test_cc_connector")) - # aiohttp is only a transitive dependency; stub it so this test pins - # the client import, not whether aiohttp happens to be installed. - fake_aiohttp = MagicMock() + + async def scenario(): + ok = await connector._connect_impl() + result = await then(connector) if then and ok else None + if getattr(connector, "session", None) is not None: + await connector.session.aclose() + return ok, result + with ( patch( "fluid_build.cli._command_center.get_command_center_client", return_value=cc, ) as factory, - patch.dict("sys.modules", {"aiohttp": fake_aiohttp}), + patch( + "fluid_build.util.http_client.new_async_client", + wraps=http_client.new_async_client, + ) as new_client, ): - ok = _run(connector._connect_impl()) - return ok, connector, factory, fake_aiohttp + ok, result = _run(scenario()) + return ok, connector, factory, new_client, result def test_connects_through_the_cli_command_center_client(self, caplog): + import httpx + + from fluid_build.observability.reporter import refuse_foreign_redirect + cc = self._client() with caplog.at_level(logging.INFO, logger="test_cc_connector"): - ok, connector, factory, fake_aiohttp = self._connect(cc) + ok, connector, factory, new_client, _ = self._connect(cc) assert "Failed to import Command Center client" not in caplog.text assert ok is True factory.assert_called_once() assert connector.cc_client is cc assert connector.base_url == self._URL - assert connector.session is fake_aiohttp.ClientSession.return_value + assert isinstance(connector.session, httpx.AsyncClient) + # The session refuses a redirect to another origin (the reporter's rule). + new_client.assert_called_once() + assert new_client.call_args.kwargs["response_hooks"] == [refuse_foreign_redirect] def test_catalog_unavailable_is_reported_as_such(self, caplog): with caplog.at_level(logging.INFO, logger="test_cc_connector"): - ok, _connector, factory, _aiohttp = self._connect(self._client(catalog=False)) + ok, _connector, factory, new_client, _ = self._connect(self._client(catalog=False)) assert ok is False factory.assert_called_once() + new_client.assert_not_called() assert "Failed to import Command Center client" not in caplog.text assert "Command Center catalog not available" in caplog.text + + def test_search_sends_the_query_and_maps_an_empty_catalog(self): + import httpx + import respx + + async def search(connector): + return await connector._search_data_products_impl(_cc_filters("orders")) + + with respx.mock(assert_all_called=False) as router: + route = router.get(self._URL).mock(return_value=httpx.Response(200, json={"items": []})) + ok, _connector, _factory, _new_client, products = self._connect( + self._client(), then=search + ) + + assert ok is True + assert route.call_count == 1 + assert route.calls.last.request.url.params["query"] == "orders" + assert products == [] + + def test_search_non_200_returns_no_products(self): + import httpx + import respx + + async def search(connector): + return await connector._search_data_products_impl(_cc_filters(None)) + + with respx.mock(assert_all_called=False) as router: + router.get(self._URL).mock(return_value=httpx.Response(503)) + ok, _connector, _factory, _new_client, products = self._connect( + self._client(), then=search + ) + + assert ok is True + assert products == [] + + def test_stats_come_back_as_the_catalog_json(self): + import httpx + import respx + + stats = {"total_products": 3, "total_domains": 2, "avg_quality_score": 0.9} + + async def get_stats(connector): + return await connector._get_catalog_stats_impl() + + with respx.mock(assert_all_called=False) as router: + router.get(f"{self._URL}/stats").mock(return_value=httpx.Response(200, json=stats)) + _ok, _c, _f, _n, result = self._connect(self._client(), then=get_stats) + + assert result == stats + + def test_search_refuses_a_redirect_to_another_origin(self): + import httpx + import respx + + async def search(connector): + return await connector._search_data_products_impl(_cc_filters(None)) + + with respx.mock(assert_all_called=False) as router: + router.get(self._URL).mock( + return_value=httpx.Response( + 302, headers={"Location": "https://elsewhere.example.test/collect"} + ) + ) + other = router.get("https://elsewhere.example.test/collect").mock( + return_value=httpx.Response(200, json={"items": []}) + ) + ok, _connector, _factory, _new_client, products = self._connect( + self._client(), then=search + ) + + assert ok is True + assert products == [] + assert other.call_count == 0 diff --git a/tests/test_marketplace_branches.py b/tests/test_marketplace_branches.py index 34b8d935f..c16032e2d 100644 --- a/tests/test_marketplace_branches.py +++ b/tests/test_marketplace_branches.py @@ -20,7 +20,9 @@ from types import SimpleNamespace from unittest.mock import MagicMock, patch +import httpx import pytest +import respx import fluid_build.cli.marketplace as marketplace_module from fluid_build.cli._common import CLIError @@ -31,6 +33,7 @@ register, run, ) +from fluid_build.util import http_client @pytest.fixture @@ -200,15 +203,26 @@ def test_public_fallback(self, mock_cc, logger): # ── search_blueprints ─────────────────────────────────────────────── +def _search_args(**overrides): + values = dict( + query="test", + category=None, + tags=None, + maturity=None, + state="published", + sort="downloads", + limit=20, + ) + values.update(overrides) + return SimpleNamespace(**values) + + class TestSearchBlueprints: @patch("fluid_build.cli.marketplace.console", MagicMock()) def test_search_success(self, logger): - import requests as real_requests - from fluid_build.cli.marketplace import search_blueprints - mock_resp = MagicMock() - mock_resp.json.return_value = { + body = { "items": [ { "id": "bp1", @@ -222,153 +236,83 @@ def test_search_success(self, logger): ], "total": 1, } - mock_resp.raise_for_status = MagicMock() - with patch("fluid_build.cli.marketplace.requests") as mock_requests: - mock_requests.get.return_value = mock_resp - mock_requests.exceptions = real_requests.exceptions - args = SimpleNamespace( - query="test", - category=None, - tags=None, - maturity=None, - state="published", - sort="downloads", - limit=20, - ) - result = search_blueprints(args, logger, "http://api") + with respx.mock(assert_all_called=False) as router: + route = router.get("http://api").mock(return_value=httpx.Response(200, json=body)) + with patch( + "fluid_build.util.http_client.new_client", wraps=http_client.new_client + ) as spy: + result = search_blueprints(_search_args(query="test"), logger, "http://api") assert result == 0 + assert route.call_count == 1 assert ( - mock_requests.get.call_args.kwargs["timeout"] - == marketplace_module.MARKETPLACE_HTTP_TIMEOUT_SECONDS + spy.call_args.kwargs["timeout"] == marketplace_module.MARKETPLACE_HTTP_TIMEOUT_SECONDS ) @patch("fluid_build.cli.marketplace.console", MagicMock()) def test_search_empty_results(self, logger): - import requests as real_requests - from fluid_build.cli.marketplace import search_blueprints - mock_resp = MagicMock() - mock_resp.json.return_value = {"items": [], "total": 0} - mock_resp.raise_for_status = MagicMock() - with patch("fluid_build.cli.marketplace.requests") as mock_requests: - mock_requests.get.return_value = mock_resp - mock_requests.exceptions = real_requests.exceptions - args = SimpleNamespace( - query="xyz", - category=None, - tags=None, - maturity=None, - state="published", - sort="downloads", - limit=20, + with respx.mock(assert_all_called=False) as router: + router.get("http://api").mock( + return_value=httpx.Response(200, json={"items": [], "total": 0}) ) - result = search_blueprints(args, logger, "http://api") + result = search_blueprints(_search_args(query="xyz"), logger, "http://api") assert result == 0 @patch("fluid_build.cli.marketplace.console", MagicMock()) def test_search_with_all_filters(self, logger): - import requests as real_requests - from fluid_build.cli.marketplace import search_blueprints - mock_resp = MagicMock() - mock_resp.json.return_value = {"items": [], "total": 0} - mock_resp.raise_for_status = MagicMock() - with patch("fluid_build.cli.marketplace.requests") as mock_requests: - mock_requests.get.return_value = mock_resp - mock_requests.exceptions = real_requests.exceptions - args = SimpleNamespace( - query="q", - category="analytics", - tags="tag1,tag2", - maturity="stable", - state="published", - sort="name", - limit=10, + args = _search_args( + query="q", + category="analytics", + tags="tag1,tag2", + maturity="stable", + sort="name", + limit=10, + ) + with respx.mock(assert_all_called=False) as router: + route = router.get("http://api").mock( + return_value=httpx.Response(200, json={"items": [], "total": 0}) ) result = search_blueprints(args, logger, "http://api") assert result == 0 + assert route.call_count == 1 @patch("fluid_build.cli.marketplace.console", MagicMock()) def test_search_request_error(self, logger): - import requests as real_requests - from fluid_build.cli.marketplace import search_blueprints - with patch("fluid_build.cli.marketplace.requests") as mock_requests: - mock_requests.get.side_effect = real_requests.exceptions.RequestException( - "Network error" - ) - mock_requests.exceptions = real_requests.exceptions - args = SimpleNamespace( - query="test", - category=None, - tags=None, - maturity=None, - state="published", - sort="downloads", - limit=20, - ) - result = search_blueprints(args, logger, "http://api") + with respx.mock(assert_all_called=False) as router: + router.get("http://api").mock(side_effect=httpx.HTTPError("Network error")) + result = search_blueprints(_search_args(), logger, "http://api") assert result == 1 def test_search_timeout_retries_and_returns_one(self, logger): - import requests as real_requests - from fluid_build.cli.marketplace import search_blueprints - args = SimpleNamespace( - query="test", - category=None, - tags=None, - maturity=None, - state="published", - sort="downloads", - limit=20, - ) with ( patch("fluid_build.cli.marketplace.console") as mock_console, - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, ): - mock_requests.get.side_effect = real_requests.exceptions.Timeout("read timed out") - mock_requests.exceptions = real_requests.exceptions - result = search_blueprints(args, logger, "http://api") + route = router.get("http://api").mock(side_effect=httpx.ReadTimeout("read timed out")) + result = search_blueprints(_search_args(), logger, "http://api") assert result == 1 - assert mock_requests.get.call_count == marketplace_module.MARKETPLACE_HTTP_RETRIES + 1 + assert route.call_count == marketplace_module.MARKETPLACE_HTTP_RETRIES + 1 assert "timed out after" in str(mock_console.print.call_args_list) def test_search_http_error_reports_status(self, logger): - import requests as real_requests - from fluid_build.cli.marketplace import search_blueprints - args = SimpleNamespace( - query="test", - category=None, - tags=None, - maturity=None, - state="published", - sort="downloads", - limit=20, - ) - response = MagicMock() - response.status_code = 503 - response.text = "service unavailable" - error = real_requests.exceptions.HTTPError("server error") - error.response = response - - mock_response = MagicMock() - mock_response.raise_for_status.side_effect = error - with ( patch("fluid_build.cli.marketplace.console") as mock_console, - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, ): - mock_requests.get.return_value = mock_response - mock_requests.exceptions = real_requests.exceptions - result = search_blueprints(args, logger, "http://api") + router.get("http://api").mock( + return_value=httpx.Response(503, text="service unavailable") + ) + result = search_blueprints(_search_args(), logger, "http://api") assert result == 1 assert "HTTP 503" in str(mock_console.print.call_args_list) diff --git a/tests/test_marketplace_ext.py b/tests/test_marketplace_ext.py index a00a928c1..68565ff32 100644 --- a/tests/test_marketplace_ext.py +++ b/tests/test_marketplace_ext.py @@ -15,12 +15,15 @@ """Tests for fluid_build.cli.marketplace.""" import argparse +import contextlib import json import logging from pathlib import Path from unittest.mock import MagicMock, patch +import httpx import pytest +import respx import fluid_build.cli.marketplace as marketplace_module from fluid_build.cli._common import CLIError @@ -29,6 +32,7 @@ interactive_parameter_wizard, show_blueprint_info, ) +from fluid_build.util import http_client # ── helpers ──────────────────────────────────────────────────────────── @@ -93,124 +97,115 @@ def _make_blueprint_json(**kwargs): # ── show_blueprint_info ─────────────────────────────────────────────── +API = "http://localhost:8000/api/v1/blueprints" + + +@contextlib.contextmanager +def _spy_client(): + """Record the keyword arguments each marketplace request opens its client with.""" + with patch("fluid_build.util.http_client.new_client", wraps=http_client.new_client) as spy: + yield spy + + class TestShowBlueprintInfo: def test_success_returns_zero(self): args = _args(blueprint_id="test-bp", version=None, show_template=False) - mock_response = MagicMock() - mock_response.json.return_value = _make_blueprint_json() with ( patch.object(marketplace_module, "console", MagicMock()), - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, + _spy_client() as spy, ): - mock_requests.get.return_value = mock_response - mock_requests.exceptions.RequestException = Exception - result = show_blueprint_info(args, _logger(), "http://localhost:8000/api/v1/blueprints") + route = router.get(f"{API}/test-bp").mock( + return_value=httpx.Response(200, json=_make_blueprint_json()) + ) + result = show_blueprint_info(args, _logger(), API) assert result == 0 + assert route.call_count == 1 assert ( - mock_requests.get.call_args.kwargs["timeout"] - == marketplace_module.MARKETPLACE_HTTP_TIMEOUT_SECONDS + spy.call_args.kwargs["timeout"] == marketplace_module.MARKETPLACE_HTTP_TIMEOUT_SECONDS ) def test_request_exception_returns_one(self): args = _args(blueprint_id="test-bp", version=None, show_template=False) - class FakeRequestError(Exception): - pass - with ( patch.object(marketplace_module, "console", MagicMock()), - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, ): - mock_requests.get.side_effect = FakeRequestError("connection refused") - mock_requests.exceptions.RequestException = FakeRequestError - result = show_blueprint_info(args, _logger(), "http://localhost:8000/api/v1/blueprints") + router.get(f"{API}/test-bp").mock(side_effect=httpx.HTTPError("connection refused")) + result = show_blueprint_info(args, _logger(), API) assert result == 1 def test_version_passed_in_params(self): args = _args(blueprint_id="test-bp", version="2.0.0", show_template=False) - mock_response = MagicMock() - mock_response.json.return_value = _make_blueprint_json() - captured_params = {} with ( patch.object(marketplace_module, "console", MagicMock()), - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, ): - mock_requests.get.side_effect = lambda url, params=None, **kwargs: ( - captured_params.update(params or {}), - mock_response, - )[-1] - mock_requests.exceptions.RequestException = Exception - show_blueprint_info(args, _logger(), "http://localhost:8000/api/v1/blueprints") + route = router.get(f"{API}/test-bp").mock( + return_value=httpx.Response(200, json=_make_blueprint_json()) + ) + show_blueprint_info(args, _logger(), API) - assert captured_params.get("version") == "2.0.0" + assert route.calls.last.request.url.params.get("version") == "2.0.0" def test_connection_error_returns_concise_message(self): - import requests as real_requests - args = _args(blueprint_id="test-bp", version=None, show_template=False) mock_console = MagicMock() with ( patch.object(marketplace_module, "console", mock_console), - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, ): - mock_requests.get.side_effect = real_requests.exceptions.ConnectionError( - "connection refused" + route = router.get(f"{API}/test-bp").mock( + side_effect=httpx.ConnectError("connection refused") ) - mock_requests.exceptions = real_requests.exceptions - result = show_blueprint_info(args, _logger(), "http://localhost:8000/api/v1/blueprints") + result = show_blueprint_info(args, _logger(), API) assert result == 1 - assert mock_requests.get.call_count == marketplace_module.MARKETPLACE_HTTP_RETRIES + 1 + assert route.call_count == marketplace_module.MARKETPLACE_HTTP_RETRIES + 1 assert "Could not connect to the marketplace API" in str(mock_console.print.call_args_list) def test_success_rate_greater_than_one_shown_as_percentage(self): args = _args(blueprint_id="test-bp", version=None, show_template=False) bp_data = _make_blueprint_json(success_rate=97.5) - mock_response = MagicMock() - mock_response.json.return_value = bp_data - printed_calls = [] mock_console = MagicMock() with ( patch.object(marketplace_module, "console", mock_console), - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, + _spy_client() as spy, ): - mock_requests.get.return_value = mock_response - mock_requests.exceptions.RequestException = Exception - result = show_blueprint_info(args, _logger(), "http://localhost:8000/api/v1/blueprints") + router.get(f"{API}/test-bp").mock(return_value=httpx.Response(200, json=bp_data)) + result = show_blueprint_info(args, _logger(), API) assert result == 0 assert ( - mock_requests.get.call_args.kwargs["timeout"] - == marketplace_module.MARKETPLACE_HTTP_TIMEOUT_SECONDS + spy.call_args.kwargs["timeout"] == marketplace_module.MARKETPLACE_HTTP_TIMEOUT_SECONDS ) def test_show_template_prints_syntax(self): args = _args(blueprint_id="test-bp", version=None, show_template=True) bp_data = _make_blueprint_json() bp_data["contract_template"] = "{{ project_id }}" - mock_response = MagicMock() - mock_response.json.return_value = bp_data with ( patch.object(marketplace_module, "console", MagicMock()), - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, + _spy_client() as spy, patch("fluid_build.cli.marketplace.Syntax", MagicMock()), ): - mock_requests.get.return_value = mock_response - mock_requests.exceptions.RequestException = Exception - result = show_blueprint_info(args, _logger(), "http://localhost:8000/api/v1/blueprints") + router.get(f"{API}/test-bp").mock(return_value=httpx.Response(200, json=bp_data)) + result = show_blueprint_info(args, _logger(), API) assert result == 0 assert ( - mock_requests.get.call_args.kwargs["timeout"] - == marketplace_module.MARKETPLACE_HTTP_TIMEOUT_SECONDS + spy.call_args.kwargs["timeout"] == marketplace_module.MARKETPLACE_HTTP_TIMEOUT_SECONDS ) @@ -218,25 +213,17 @@ def test_show_template_prints_syntax(self): class TestInstantiateBlueprint: - def _mock_requests(self, mock_requests, bp_data, instantiate_result=None): - mock_requests.exceptions.RequestException = Exception - bp_response = MagicMock() - bp_response.json.return_value = bp_data - - inst_response = MagicMock() - inst_response.json.return_value = instantiate_result or { - "contract": {"builds": []}, - "cost_estimate": "$0.10", - } - - def fake_get(url, **kwargs): - return bp_response - - def fake_post(url, **kwargs): - return inst_response - - mock_requests.get.side_effect = fake_get - mock_requests.post.side_effect = fake_post + def _mock_routes(self, router, bp_data, instantiate_result=None): + get_route = router.get(f"{API}/test-bp").mock( + return_value=httpx.Response(200, json=bp_data) + ) + post_route = router.post(f"{API}/test-bp/instantiate").mock( + return_value=httpx.Response( + 200, + json=instantiate_result or {"contract": {"builds": []}, "cost_estimate": "$0.10"}, + ) + ) + return get_route, post_route def test_no_params_no_interactive_returns_one(self): args = _args(blueprint_id="test-bp", params=None, interactive=False) @@ -244,12 +231,10 @@ def test_no_params_no_interactive_returns_one(self): with ( patch.object(marketplace_module, "console", MagicMock()), - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, ): - self._mock_requests(mock_requests, bp_data) - result = instantiate_blueprint( - args, _logger(), "http://localhost:8000/api/v1/blueprints" - ) + self._mock_routes(router, bp_data) + result = instantiate_blueprint(args, _logger(), API) assert result == 1 @@ -265,23 +250,18 @@ def test_params_as_json_string_returns_zero(self): with ( patch.object(marketplace_module, "console", MagicMock()), - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, + _spy_client() as spy, patch("fluid_build.cli.marketplace.Syntax", MagicMock()), ): - self._mock_requests(mock_requests, bp_data) - result = instantiate_blueprint( - args, _logger(), "http://localhost:8000/api/v1/blueprints" - ) + get_route, post_route = self._mock_routes(router, bp_data) + result = instantiate_blueprint(args, _logger(), API) assert result == 0 - assert ( - mock_requests.get.call_args.kwargs["timeout"] - == marketplace_module.MARKETPLACE_HTTP_TIMEOUT_SECONDS - ) - assert ( - mock_requests.post.call_args.kwargs["timeout"] - == marketplace_module.MARKETPLACE_HTTP_TIMEOUT_SECONDS - ) + assert get_route.call_count == 1 + assert post_route.call_count == 1 + timeouts = [call.kwargs["timeout"] for call in spy.call_args_list] + assert timeouts == [marketplace_module.MARKETPLACE_HTTP_TIMEOUT_SECONDS] * 2 def test_params_from_file(self, tmp_path): params_file = tmp_path / "params.json" @@ -297,52 +277,41 @@ def test_params_from_file(self, tmp_path): with ( patch.object(marketplace_module, "console", MagicMock()), - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, patch("fluid_build.cli.marketplace.Syntax", MagicMock()), ): - self._mock_requests(mock_requests, bp_data) - result = instantiate_blueprint( - args, _logger(), "http://localhost:8000/api/v1/blueprints" - ) + self._mock_routes(router, bp_data) + result = instantiate_blueprint(args, _logger(), API) assert result == 0 def test_fetch_failure_returns_one(self): args = _args(blueprint_id="test-bp", params='{"x": 1}', interactive=False) - class FakeError(Exception): - pass - with ( patch.object(marketplace_module, "console", MagicMock()), - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, ): - mock_requests.get.side_effect = FakeError("conn error") - mock_requests.exceptions.RequestException = FakeError - result = instantiate_blueprint( - args, _logger(), "http://localhost:8000/api/v1/blueprints" - ) + router.get(f"{API}/test-bp").mock(side_effect=httpx.HTTPError("conn error")) + result = instantiate_blueprint(args, _logger(), API) assert result == 1 def test_fetch_timeout_retries_and_returns_one(self): - import requests as real_requests - args = _args(blueprint_id="test-bp", params='{"x": 1}', interactive=False) mock_console = MagicMock() with ( patch.object(marketplace_module, "console", mock_console), - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, ): - mock_requests.get.side_effect = real_requests.exceptions.Timeout("read timed out") - mock_requests.exceptions = real_requests.exceptions - result = instantiate_blueprint( - args, _logger(), "http://localhost:8000/api/v1/blueprints" + route = router.get(f"{API}/test-bp").mock( + side_effect=httpx.ReadTimeout("read timed out") ) + result = instantiate_blueprint(args, _logger(), API) assert result == 1 - assert mock_requests.get.call_count == marketplace_module.MARKETPLACE_HTTP_RETRIES + 1 + assert route.call_count == marketplace_module.MARKETPLACE_HTTP_RETRIES + 1 assert "timed out after" in str(mock_console.print.call_args_list) def test_output_file_saved(self, tmp_path): @@ -358,13 +327,11 @@ def test_output_file_saved(self, tmp_path): with ( patch.object(marketplace_module, "console", MagicMock()), - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, patch("fluid_build.cli.marketplace.Syntax", MagicMock()), ): - self._mock_requests(mock_requests, bp_data) - result = instantiate_blueprint( - args, _logger(), "http://localhost:8000/api/v1/blueprints" - ) + self._mock_routes(router, bp_data) + result = instantiate_blueprint(args, _logger(), API) assert result == 0 assert output_file.exists() @@ -381,21 +348,15 @@ def test_instantiate_request_failure_returns_one(self): ) bp_data = _make_blueprint_json() - class FakeReqError(Exception): - response = None - with ( patch.object(marketplace_module, "console", MagicMock()), - patch("fluid_build.cli.marketplace.requests") as mock_requests, + respx.mock(assert_all_called=False) as router, ): - mock_requests.exceptions.RequestException = FakeReqError - bp_response = MagicMock() - bp_response.json.return_value = bp_data - mock_requests.get.return_value = bp_response - mock_requests.post.side_effect = FakeReqError("instantiate failed") - result = instantiate_blueprint( - args, _logger(), "http://localhost:8000/api/v1/blueprints" + router.get(f"{API}/test-bp").mock(return_value=httpx.Response(200, json=bp_data)) + router.post(f"{API}/test-bp/instantiate").mock( + side_effect=httpx.HTTPError("instantiate failed") ) + result = instantiate_blueprint(args, _logger(), API) assert result == 1 diff --git a/tests/test_network.py b/tests/test_network.py index e9d601ea8..e74011b43 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -16,7 +16,9 @@ import time +import httpx import pytest +import respx from fluid_build.errors import NetworkError from fluid_build.util.network import ( @@ -28,6 +30,8 @@ CircuitBreaker, CircuitState, RateLimiter, + create_session_with_retries, + safe_request, ) @@ -131,3 +135,52 @@ def test_default_values(self): assert DEFAULT_MAX_RETRIES == 3 assert DEFAULT_BACKOFF_FACTOR == 2 assert DEFAULT_RATE_LIMIT == 50 + + +class TestSafeRequest: + """``safe_request`` over httpx: success, HTTP status, timeout and connection errors.""" + + URL = "https://api.example.test/thing" + + def test_success_returns_response(self): + with respx.mock() as router: + router.get(self.URL).respond(200, json={"ok": True}) + with httpx.Client() as session: + response = safe_request("GET", self.URL, session=session) + assert response.status_code == 200 + assert response.json() == {"ok": True} + + def test_http_error_status_raises_network_error_with_status(self): + with respx.mock() as router: + router.get(self.URL).respond(404) + with httpx.Client() as session: + with pytest.raises(NetworkError, match="HTTP 404") as raised: + safe_request("GET", self.URL, session=session) + assert raised.value.context["status_code"] == 404 + + def test_timeout_raises_network_error(self): + with respx.mock() as router: + router.get(self.URL).mock(side_effect=httpx.ReadTimeout("slow")) + with httpx.Client() as session: + with pytest.raises(NetworkError, match="timed out"): + safe_request("GET", self.URL, session=session) + + def test_connection_error_raises_network_error(self): + with respx.mock() as router: + router.get(self.URL).mock(side_effect=httpx.ConnectError("refused")) + with httpx.Client() as session: + with pytest.raises(NetworkError, match="Connection failed"): + safe_request("GET", self.URL, session=session) + + def test_retrying_session_retries_a_transient_status(self): + with respx.mock() as router: + route = router.get(self.URL).mock( + side_effect=[httpx.Response(503), httpx.Response(200, json={"ok": True})] + ) + session = create_session_with_retries(max_retries=2, backoff_factor=0) + try: + response = safe_request("GET", self.URL, session=session) + finally: + session.close() + assert response.status_code == 200 + assert route.call_count == 2 From 356b8618a9aa316ad2b90372ecf893dc397231e6 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:56:51 +0100 Subject: [PATCH 22/26] fix(diff): close the --baseline gate gaps found while shipping #789 (#805) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(diff): validate both contracts before fluid diff --baseline compares them The diff rules read only the shapes a schema allows, so a contract no schema accepts could hide a change from --fail-on-breaking: a column retyped from decimal(18,2) to the number 18 passed with exit 0. compare_contracts now validates both sides first, as fluid plan does: the JSON Schema of each contract's own fluidVersion, offline, from the bundled schemas. It raises the new ContractValidationError, which lists every error of every invalid side. A pre-0.7 contract is refused with the upgrade fluid validate asks for. validate=False keeps the old behaviour, and the fluid forge --refine preview uses it, because the contract being refined may be the invalid one. fluid diff --baseline maps the error to diff_contract_invalid, exit 2 (diff(1)'s "trouble", never 0 or 1). It logs each schema error, names each invalid file with its fluid validate command, and prints or writes nothing. Prior art: datacontract-cli's breaking command parses both contracts into typed models and refuses an invalid one (HTTP 422). * fix(diff): carry a 0.7.x contract's quality, sensitivity and primary key A 0.7.x contract keeps them under exposes[].contract, where the diff did not look, so the gate missed a quality rule made failing, any sensitivity change, and a primary-key column made nullable (reported non-breaking). - Quality: contract.dq.rules (by id), contract.quality (by rule) and, from 0.7.6, each column's quality (by id) are read beside the pre-0.7 quality.tests. An escalation is judged by fluid_build.severity.is_error, so critical fails as error does: warn -> critical is now an escalation and critical -> error no longer is. - Sensitivity: a column whose sensitivity becomes or stops being pii or phi (the set docs_build and the MCP output port use) is column_pii_added / column_pii_removed, reported at .sensitivity. Any other change is the new column_sensitivity_changed (info). - Primary key: a column tagged primary-key (or primaryKey) is in the key, the tag the ODCS exporter turns into primaryKey: true, so one that stops being required is primary_key_nullable (breaking). The PK predicate and the sensitivity block move out of _primary_key_set and diff_columns, which keeps both under their pre-change complexity. * fix(diff): read the columns of a schema wrapped as {fields: [...]} _verify_reconcile, forge_copilot_drift_guard and tools/contract_tests accept an expose schema written as {"fields": [...]}; diff_columns read the mapping's keys instead and saw no columns, so a removed column went unreported. The wrapper is now unwrapped at schema and contract.schema alike, and its columns are reported at ...schema.fields.. No bundled schema accepts the wrapper, so fluid diff --baseline refuses such a contract as invalid; the unwrap serves the unvalidated callers, the fluid forge --refine preview among them. * fix(diff): print contract names as data in text, Markdown and JSON A column or expose name reached the terminal and Markdown raw. In text it could drive the terminal (an escape sequence, an OSC 8 link to another URL, a carriage return or line feed that forges a line, a bidi control that reorders it), and Rich read [bold] as a style, swapped :smile: for an emoji and raised on [/], which crashed the command. In Markdown a name could add a link, an image or an HTML comment that hides the rest of a PR comment, or mention someone. - render_text spells every non-printable or invisible character as repr spells it (\x1b, ‮) and doubles a backslash, and fluid diff prints it literally (cprint_literal) and soft-wrapped, as the drift report is, so each entry stays one line. - render_markdown puts the path and the description each in one code span. A backslash escape is not enough: GitHub's renderer still links \@user, \#1, \:smile\: and foo\@example\.com (checked against its /markdown API); a code span is literal on every host. The fence and padding follow mdformat's code_inline (MIT). - JSON and Markdown are printed verbatim (cprint_json): Rich wrapped a long line at 80 columns in CI, which put a line break inside a JSON string, so --format json could print JSON that does not parse. - The fluid forge --refine preview prints the same text without markup. * fix(diff): redact each value, then write JSON and Markdown verbatim From the security review of this branch: cprint_json ran the console's redaction regex over the finished document, and its value pattern ([^\s,;]+) took the closing fence of a Markdown code span with it. A column named "[a](https://e.x) token=z" (or ending in token=, token:) closed its path span early, the next fence opened the description, and the description rendered as live Markdown: a link, an image, a mention. The same pass took the closing quote of a JSON string, so --format json could print JSON that does not parse. fluid diff now redacts every value of the report first, with the redactor logging uses (observability.secret_redactor.redact_value, a wider set of secret shapes than the console's), and writes the JSON and Markdown documents verbatim. No regex runs over a finished document, so a dangling "token=" cannot reach a fence either. Text keeps cprint_literal, whose pass cannot break a line's structure. Tests render each name through the CLI and parse the Markdown with markdown-it-py, and check a credential in a name or description is redacted while the JSON still parses. * fix(diff): refuse a contract whose fluidVersion is not a version From the architecture review: detect_version raises on a fluidVersion that is not a version (7, "garbage"), and the error escaped compare_contracts' gate, so fluid diff --baseline exited 1 (diff_failed), which a pipeline reads as "breaking". Any failure to validate a side is now that side's error, raised as ContractValidationError, so the CLI exits 2 (diff_contract_invalid) as for any other invalid contract. * docs(diff): --baseline help names the validation; correct two docstrings From the documentation review: fluid diff --help did not say that both contracts must be schema-valid or that an invalid one exits 2 before any output, which only the CHANGELOG said. The --baseline help says so now. The _code docstring states its padding rule as the code applies it, and the module docstring no longer places the renderers in cli/diff.py. * fix(diff): a duplicate rule id cannot hide a severity escalation From the codebase review: nothing makes a quality rule's id unique, and _rules_by_name kept the last rule of each name, so a new contract with dq.rules [d1 error, d1 warn] against a baseline [d1 warn] reported no escalation, and --fail-on-breaking passed. A baseline [d1 error, d1 warn] against [d1 error, d1 error] was reported as one. Each name now keeps all of its rules, and each version is judged by its most severe rule of that name, in every location the diff reads. * test(diff): pin the code-span padding and the defaults the review found untested From the test-suite review (mutation probes): the space-at-both-ends and backtick-at-both-ends cases embedded the name in a longer path, so they never reached _code's padding branch, and a mutant that dropped it survived. _code is now tested on the raw strings, parsed with markdown-it-py. Also pinned: a quality test with no severity counts as warn; a column rule named by name (no id) is compared; an invalid baseline alone is named in diff_contract_invalid; the pre-0.7 per-column primaryKey flag is still read. * fix(diff): the defect scan's findings in the diff engine From the scoped defect scan of this branch (hunters, skeptic, referee): - A read-only or missing HOME made fluid diff --baseline exit 1, as for a breaking change: the schema manager creates ~/.fluid/schema_cache when it is built. It now falls back to a temporary cache; the diff validates offline against the bundled schemas, so nothing a bundled version needs is lost. - A column the dbt engine builds as a key (engines/dbt/_test_mapping.py column_is_key: semanticType identifier or primary_key, a labels constraint of primary_key or unique, the key flags) is in the primary key, as a primary-key tag already was, so one that stops being required is primary_key_nullable. The predicate is mirrored, because importing the engine loads every engine; a test pins the two together. - A PII change reported at .sensitivity carries the sensitivity values, not the derived booleans, and a pre-0.7 pii: true rewritten as sensitivity: pii is not a change. - The blank Braille pattern (U+2800), a printable symbol that draws as a space, is spelled out like the other characters that draw nothing. * fix(diff): one redacted, JSON-safe envelope for stdout and --out From the scoped defect scan of this branch: - The per-value redaction used the logging redactor alone, which misses shapes the console pattern caught on main: "api key: x" with a space, a separator padded with more than eight spaces, a very long value. A value now goes through both, as the CLI's error renderer does (_redact_str(redact_secret_text(...))). - --out was written from the raw report while stdout was redacted. Both now get the same redacted envelope (_safe_report). - A value JSON cannot hold (a date, bytes or a set that YAML put in a column's quality rule) made json.dump raise after the text report had printed, left --out cut short and exited 1, as for a breaking change; a NaN was written as a bare token. Such values are written as text. * refactor(diff): judge a key with the dbt engine's own predicate, not a copy The diff mirrored engines/dbt/_test_mapping.py::column_is_key (and its is_truthy) because importing the engine loads every engine; ripwire's quality delta flagged the 141-token copy. column_is_key, is_truthy and _PK_KEYS move unchanged to fluid_build/util/contract.py (stdlib-only), _test_mapping re-exports them so every _tm.* caller is unchanged, and _in_primary_key is the ODCS tag check plus that one predicate. The parity test becomes an identity check. * fix(diff): redact a credential held in a mapping key, too From the round-2 review: _safe_report redacted every string value, but a mapping's keys went out as written, and a removed column's free-form labels reach before/after. A key such as "password: x" was printed in --format json and written to --out; on main the whole-text pass of the console had masked it on stdout. Keys now go through both redactors. --- CHANGELOG.md | 60 +- fluid_build/api/changelog.py | 189 +++++- fluid_build/api/changelog_rules.py | 257 +++++-- fluid_build/cli/_preview_panel.py | 14 +- fluid_build/cli/diff.py | 158 ++++- fluid_build/engines/dbt/_test_mapping.py | 31 +- fluid_build/util/contract.py | 34 + tests/test_diff_changelog.py | 54 +- tests/test_diff_changelog_hardening.py | 821 +++++++++++++++++++++++ 9 files changed, 1483 insertions(+), 135 deletions(-) create mode 100644 tests/test_diff_changelog_hardening.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2973b0bc5..415d71f2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -249,9 +249,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 column, a type change that is not a widening, an added `required: true` column), so a pipeline that runs `fluid diff --baseline --fail-on-breaking` can start exiting 1. A finding for a 0.7.x column is reported at `exposes[N].contract.schema.`, and a nullability one at its - `.required`. Still not compared in a 0.7.x contract: quality-rule severity and a column's - `sensitivity`. A 0.7.x primary key is not recognised either, so a key column that becomes - nullable is reported as non-breaking. + `.required`. + +- **`fluid diff --baseline` ignored a 0.7.x contract's quality rules, sensitivity and primary + key.** A 0.7.x contract keeps them under `exposes[].contract`, where the diff did not look. A + rule in `contract.dq.rules`, `contract.quality` or a column's `quality` whose severity becomes + failing is now `quality_severity_escalated` (breaking), as a pre-0.7 `quality.tests` entry was. + `critical` counts as failing, as it does for `fluid test`: warn -> critical is now an + escalation, and critical -> error no longer is. Rules that share an id are judged by the most + severe of them, so a failing rule cannot hide behind a `warn` one of the same id. A column + whose `sensitivity` becomes or stops being `pii` or `phi` is `column_pii_added` / + `column_pii_removed` (info), and any other change to it is the new + `column_sensitivity_changed` (info), each with the sensitivity values as before and after. A + pre-0.7 `pii: true` rewritten as `sensitivity: pii` is not a change. A column the dbt engine + builds as a key (a `primary-key` tag, a `semanticType` of `identifier` or `primary_key`, or a + `labels.constraint` of `primary_key` or `unique`) that stops being `required` is + `primary_key_nullable` (breaking) rather than a non-breaking loosening. + +- **`fluid diff --baseline --format json` could print JSON that does not parse.** Rich wrapped + each long line at the terminal width (80 columns in CI), putting a line break inside a JSON + string, and read `[...]` in a column name as markup. JSON and Markdown are now printed + verbatim, and text without markup or `:emoji:` codes, so a column named `[/]` no longer + crashes the text output. Credential-shaped values are redacted in each value before the + document is written, by both the logging redactor and the console's, so a name ending in + `token=x` no longer cuts its JSON string short. A value JSON cannot hold, such as a date that + YAML put in a column's quality rule, is written as text: `json.dump` raised on it after the + text report had printed, and left `--out` cut short. `--out` now holds the same redacted + envelope as stdout; it held the raw values. + +- **The `fluid forge --refine` preview read no columns from a `schema: {fields: [...]}` + expose.** The diff now unwraps it at `schema` and `contract.schema` alike, as `fluid verify` + does, and reports its columns at `...schema.fields.`. No bundled schema accepts the + wrapper, so `fluid diff --baseline` refuses such a contract as invalid (see Security). - **A failed `fluid apply` on OpenTofu printed raw JSON and no way forward.** The runner uses `-json`, so the error was the last 800 characters of diagnostic lines. It now reads the @@ -529,6 +558,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 and |- or >- drops)`. A `sink.catalogAuth` field gets no hint, since its value is never shown. The schemas themselves are unchanged. +- **`fluid diff --baseline` compared contracts that no schema accepts.** Its rules read only the + shapes a schema allows, so a value of the wrong type hid a change: a column retyped from + `decimal(18,2)` to the number `18` passed `--fail-on-breaking` with exit 0. Both contracts are + now validated first, against the schema their `fluidVersion` names, offline, the gate + `fluid plan` runs. An invalid one exits 2 with `diff_contract_invalid`, which names each + invalid file and logs each error, and nothing is printed or written to `--out`. Exit 2 is + diff(1)'s "trouble", so it never reads as 0 (no changes) or 1 (breaking). A pre-0.7 contract is + refused with the upgrade that `fluid validate` asks for, so a pre-0.7 baseline can no longer be + diffed against its 0.7.x rewrite. In `fluid_build.api.changelog`, `compare_contracts` raises the + new `ContractValidationError`, and `validate=False` diffs without validating; the + `fluid forge --refine` preview uses it, because the contract being refined may be the invalid one. + +- **`fluid diff --baseline` printed contract names raw.** A column or expose name could drive the + terminal (an escape sequence, a hyperlink to another URL, a carriage return or line feed that + forges a line, a bidi control that reorders it), and in `--format markdown` it could add a link, + an image or an HTML comment that hides the rest of a PR comment, or mention someone. Text output + now spells each control, bidi and invisible character as `\x1b` or `\u202e` and doubles a + backslash. Markdown puts the path and the description each in one code span: a backslash escape + is not enough there, because GitHub still links `\@user`, `\#1` and `\:smile\:` after + rendering. The fence is one backtick longer than any run of backticks in the name, and the + document is redacted per value before it is written, so a redaction cannot take the closing + fence with it. A Markdown entry now reads + ``- **column_added** `exposes[0].contract.schema.email` — `column 'email' added to expose 'orders'` ``. + The `fluid forge --refine` preview prints the same text without markup. + ## [0.26.0] — 2026-11-01 ### Added diff --git a/fluid_build/api/changelog.py b/fluid_build/api/changelog.py index e5547907b..767653ed5 100644 --- a/fluid_build/api/changelog.py +++ b/fluid_build/api/changelog.py @@ -15,17 +15,25 @@ """Contract-vs-contract semantic diff (the version-history side of ``fluid diff``). Public entry point: :func:`compare_contracts(baseline, new) -> ChangelogReport`. -The output is the structured envelope from ``changelog_types``; CLI/render -formatting lives in ``cli/diff.py`` (text / json / markdown emitters). +The output is the structured envelope from ``changelog_types``; the text and +Markdown renderers are below, and ``cli/diff.py`` picks one and prints it. The classifier is fluid-schema-native — it understands acquisition engines, ``agentPolicy``, ``sovereignty``, and the SDP/ADP/CDP × Bronze/Silver/Gold cross-check directly, rather than treating contracts as generic schema docs. + +Both contracts are schema-validated before they are compared (see +:class:`ContractValidationError`), and the renderers print every name a +contract supplies as data: spelled out for a terminal, inert in Markdown. """ from __future__ import annotations -from typing import Any, Mapping +import contextlib +import re +import tempfile +from pathlib import Path +from typing import Any, Dict, Iterator, List, Mapping from .changelog_rules import ( diff_agent_policy, @@ -39,11 +47,35 @@ from .changelog_types import Change, ChangelogReport -def compare_contracts(baseline: Mapping[str, Any], new: Mapping[str, Any]) -> ChangelogReport: +class ContractValidationError(ValueError): + """A contract handed to :func:`compare_contracts` is not a valid FLUID contract. + + ``errors`` maps each invalid side, ``"baseline"`` or ``"new"``, to its + errors, worded as ``fluid validate`` words them. A side that validated is + absent. + """ + + def __init__(self, errors: Mapping[str, List[str]]) -> None: + self.errors: Dict[str, List[str]] = {side: list(found) for side, found in errors.items()} + super().__init__( + "; ".join( + f"the {side} contract is not a valid FLUID contract ({len(found)} error(s))" + for side, found in self.errors.items() + ) + ) + + +def compare_contracts( + baseline: Mapping[str, Any], new: Mapping[str, Any], *, validate: bool = True +) -> ChangelogReport: """Return a structured changelog comparing two parsed contracts. ``baseline`` is treated as "old"; ``new`` as the candidate. Idempotent — passing the same dict twice produces an empty report. + + Both are first validated (:func:`_require_valid`), and an invalid one + raises :class:`ContractValidationError`. ``validate=False`` diffs whatever + it is given, for an advisory diff of a contract still being repaired. """ report = ChangelogReport() @@ -53,6 +85,9 @@ def compare_contracts(baseline: Mapping[str, Any], new: Mapping[str, Any]) -> Ch f"got {type(baseline).__name__} / {type(new).__name__}" ) + if validate: + _require_valid({"baseline": baseline, "new": new}) + # Top-level metadata churn — info-level signals. report.extend(diff_metadata(baseline, new)) @@ -99,11 +134,140 @@ def compare_contracts(baseline: Mapping[str, Any], new: Mapping[str, Any]) -> Ch return report +def _require_valid(contracts: Mapping[str, Mapping[str, Any]]) -> None: + """Raise :class:`ContractValidationError` naming every side that is not valid. + + The rules read only the shapes a schema allows, so an invalid contract can + hide a change from them: a column whose ``type`` is a number is never + compared. This is the gate ``fluid plan`` runs before it writes a plan: the + JSON Schema of the contract's own ``fluidVersion``, offline, from the + bundled schemas. A pre-0.7 version has no schema left, and is refused with + the upgrade that ``fluid validate`` asks for instead of "schema not + available". A side the validator cannot even read (a ``fluidVersion`` that + is not a version, such as ``7``) is refused too, never let through. + """ + errors: Dict[str, List[str]] = {} + with _schema_manager() as manager: + for side, contract in contracts.items(): + try: + found = _schema_errors(manager, contract) + except Exception as exc: # noqa: BLE001 — a contract nothing can validate is refused + found = [f"the contract cannot be validated: {exc}"] + if found: + errors[side] = found + if errors: + raise ContractValidationError(errors) + + +@contextlib.contextmanager +def _schema_manager() -> Iterator[Any]: + """The schema manager, with its cache in a temporary directory when HOME is not writable. + + The manager creates ``~/.fluid/schema_cache`` when it is built, which + raised on a read-only or missing HOME (a locked-down CI container) and + made the gate exit 1, as for a breaking change. The diff validates + offline against the bundled schemas, so a temporary cache loses nothing + a bundled version needs. + """ + # jsonschema stays off the import path of every caller that does not validate. + from fluid_build.schema_manager import FluidSchemaManager + + try: + manager = FluidSchemaManager() + except OSError: + manager = None + if manager is not None: + yield manager + return + with tempfile.TemporaryDirectory(prefix="fluid-schema-cache-") as cache: + yield FluidSchemaManager(cache_dir=Path(cache)) + + +def _schema_errors(manager: Any, contract: Mapping[str, Any]) -> List[str]: + version = manager.detect_version(dict(contract)) + if version is not None and (version.major, version.minor) < (0, 7): + return [ + f"fluidVersion {str(contract.get('fluidVersion'))!r} is no longer supported: " + "pre-0.7 schemas have been removed; upgrade the contract to fluidVersion 0.7.x " + "or later" + ] + result = manager.validate_contract(dict(contract), offline_only=True) + return [] if result.is_valid else list(result.errors) + + +# Characters ``str.isprintable`` passes that still draw nothing: the Unicode +# Default_Ignorable_Code_Point letters and marks (the combining grapheme joiner, +# the Hangul fillers, the Khmer inherent vowels, the Mongolian free variation +# selectors and the variation selectors), and the blank Braille pattern, +# which is a symbol but draws as a space. The other default-ignorables are +# format characters, which ``isprintable`` already refuses. +_DRAWS_NOTHING = re.compile( + "[\u034f\u115f\u1160\u17b4\u17b5\u180b-\u180d\u180f\u2800\u3164\ufe00-\ufe0f\uffa0" + "\U000e0100-\U000e01ef]" +) + + +def _spell(ch: str) -> str: + if ch == "\\": + return "\\\\" + if ch.isprintable() and not _DRAWS_NOTHING.match(ch): + return ch + code = ord(ch) + if code < 0x100: + return f"\\x{code:02x}" + if code < 0x10000: + return f"\\u{code:04x}" + return f"\\U{code:08x}" + + +def _text(value: Any) -> str: + """``value`` as one line of text that shows every character it holds. + + A path or a description quotes names from the contract, and printed raw a + name could drive the terminal: ESC opens an escape sequence (a colour, a + retitled window, a link to another URL), a line feed or a carriage return + forges or overwrites a line, and a bidi control reorders what the reader + sees (Trojan Source, CVE-2021-42574). Every character that is not + printable, or prints as nothing, is spelled as ``repr`` spells it + (``\\x1b``, ``\\u202e``), and a backslash is doubled, so a name holding the + four characters ``\\x1b`` still reads differently from one holding ESC. + """ + return "".join(_spell(ch) for ch in str(value)) + + +def _code(value: Any) -> str: + """``value`` as one Markdown code span, the one inline construct nothing is read out of. + + Outside a code span a backslash escape stops CommonMark markup but not + what a host adds after rendering: on GitHub ``\\@name`` still notifies, + ``\\#1`` links an issue, ``\\:smile\\:`` is an emoji and + ``foo\\@example\\.com`` a mail link, and GitLab also reads ``!1``, + ``~label`` and ``%milestone``. Inside a code span every character is + literal on every host. The fence is one backtick longer than the longest + run of backticks in the text, and the text is padded with a space when it + holds a backtick, or when it starts and ends with a space (CommonMark then + strips one space from each end), as mdformat's ``code_inline`` writes it + (MIT, github.com/hukkin/mdformat); empty text becomes one space. The text is spelled by + :func:`_text` first, so it holds no line break, which would let the next + line start a heading before the span is ever parsed. + """ + text = _text(value) + longest = max((len(run) for run in re.findall("`+", text)), default=0) + if longest: + fence = "`" * (longest + 1) + return f"{fence} {text} {fence}" + if text.startswith(" ") and text.endswith(" ") and text.strip(): + return f"` {text} `" + return f"`{text or ' '}`" + + def render_text(report: ChangelogReport) -> str: """Plain-text render: grouped by severity with one-line entries. Used by ``fluid diff --baseline ...`` default output. Kept simple so the - output is grep-friendly in CI logs. + output is grep-friendly in CI logs. Each field is spelled by + :func:`_text`, so a name in it cannot drive the terminal or forge a line; + print the result without Rich markup (``cprint_literal``). """ lines: list[str] = [] @@ -113,7 +277,7 @@ def _section(title: str, changes: list[Change]) -> None: lines.append(title) lines.append("-" * len(title)) for c in changes: - lines.append(f" {c.kind:32s} {c.path}: {c.description}") + lines.append(f" {_text(c.kind):32s} {_text(c.path)}: {_text(c.description)}") lines.append("") _section(f"BREAKING ({len(report.breaking)})", report.breaking) @@ -131,7 +295,16 @@ def _section(title: str, changes: list[Change]) -> None: def render_markdown(report: ChangelogReport) -> str: - """Markdown render — for PR-comment bots that paste this into a discussion.""" + """Markdown render — for PR-comment bots that paste this into a discussion. + + The path and the description quote names from the contract, so each is + one code span (:func:`_code`): a name can add no link, image, comment, + heading or list item, and mentions no one. ``kind`` is this module's own + vocabulary (``[a-z_]``). Redact each value before rendering and print the + result verbatim: Rich would read ``[...]`` in it as markup and wrap its + long lines, and a redaction regex run over the finished text can take a + closing fence with a ``token=x`` value (``fluid diff`` does both right). + """ parts: list[str] = ["# Contract changelog", ""] def _section(title: str, changes: list[Change], emoji: str) -> None: @@ -140,7 +313,7 @@ def _section(title: str, changes: list[Change], emoji: str) -> None: parts.append(f"## {emoji} {title} ({len(changes)})") parts.append("") for c in changes: - parts.append(f"- **{c.kind}** `{c.path}` — {c.description}") + parts.append(f"- **{c.kind}** {_code(c.path)} — {_code(c.description)}") parts.append("") _section("Breaking", report.breaking, "") diff --git a/fluid_build/api/changelog_rules.py b/fluid_build/api/changelog_rules.py index 7cd6e622e..a1733f33b 100644 --- a/fluid_build/api/changelog_rules.py +++ b/fluid_build/api/changelog_rules.py @@ -29,6 +29,8 @@ import re from typing import Any, Iterable, List, Mapping, Optional, Sequence, Tuple +from ..severity import is_error +from ..util.contract import column_is_key from .changelog_types import Change # Type widening table — keys are the "from" type, values are the set of @@ -181,17 +183,22 @@ def _expose_schema(expose: Mapping[str, Any]) -> Tuple[Sequence[Any], Optional[s Pre-0.7 contracts keep their columns at ``schema``; 0.7.x contracts keep them at ``contract.schema``. The flat ``schema`` is read when it is present and ``contract.schema`` only when it is absent, so a contract - that carries both is read as it was before 0.7.x. The location is - ``None`` when the expose carries no schema at all. + that carries both is read as it was before 0.7.x. Either one may wrap + its list as ``{"fields": [...]}``, as ``_verify_reconcile`` and + ``forge_copilot_drift_guard`` accept, and the columns are then found at + ``schema.fields``. The location is ``None`` when the expose carries no + schema at all. """ - columns = expose.get("schema") - if columns is not None: - return columns or [], "schema" - contract = expose.get("contract") - columns = contract.get("schema") if isinstance(contract, Mapping) else None - if columns is not None: - return columns or [], "contract.schema" - return [], None + columns, location = expose.get("schema"), "schema" + if columns is None: + contract = expose.get("contract") + columns = contract.get("schema") if isinstance(contract, Mapping) else None + location = "contract.schema" + if columns is None: + return [], None + if isinstance(columns, Mapping): + columns, location = columns.get("fields"), f"{location}.fields" + return (columns if isinstance(columns, (list, tuple)) else []), location def _is_nullable(column: Mapping[str, Any]) -> bool: @@ -225,7 +232,8 @@ def _primary_key_set(expose: Mapping[str, Any]) -> frozenset[str]: """Extract the set of primary-key column names from an expose dict. Supports both ``primaryKey: [col1, col2]`` at the expose root and the - per-column ``primaryKey: true`` flag — both occur in real contracts. + per-column ``primaryKey: true`` flag — both occur in real contracts — + and a 0.7.x column's primary-key tag (see :func:`_in_primary_key`). """ pk: set[str] = set() expose_pk = expose.get("primaryKey") @@ -234,13 +242,106 @@ def _primary_key_set(expose: Mapping[str, Any]) -> frozenset[str]: if isinstance(n, str): pk.add(n) for col in _expose_schema(expose)[0]: - if isinstance(col, Mapping) and col.get("primaryKey") is True: + if isinstance(col, Mapping) and _in_primary_key(col): name = col.get("name") if isinstance(name, str): pk.add(name) return frozenset(pk) +def _in_primary_key(column: Mapping[str, Any]) -> bool: + """Whether *column* is a key of its expose, the way forge builds it. + + The columns the dbt engine gives a ``primary_key`` constraint + (``util.contract.column_is_key``: a truthy key flag, a ``semanticType`` of + ``identifier`` or ``primary_key``, a ``labels`` ``constraint`` of + ``primary_key`` or ``unique``, a truthy ``labels.unique``), plus the + ``primary-key`` (or ``primaryKey``) tag the ODCS exporter turns into + ``primaryKey: true`` (``providers/odcs/mappers``). + """ + tags = column.get("tags") + if isinstance(tags, (list, tuple)) and ("primary-key" in tags or "primaryKey" in tags): + return True + return column_is_key(column) + + +#: Column sensitivities that mark personal data, as ``cli/docs_build.py`` and +#: the MCP output port's PII redaction read them. +_PERSONAL_DATA = frozenset({"pii", "phi"}) + + +def _is_pii(column: Mapping[str, Any]) -> bool: + """Whether *column* holds personal data. + + Pre-0.7 columns say ``pii: true``. A 0.7.x column has no ``pii`` key and + says ``sensitivity: pii`` (or ``phi``). + """ + sensitivity = column.get("sensitivity") + return bool(column.get("pii")) or ( + isinstance(sensitivity, str) and sensitivity in _PERSONAL_DATA + ) + + +def _sensitivity(column: Mapping[str, Any]) -> Any: + """A column's ``sensitivity``; a pre-0.7 ``pii: true`` with none reads as ``pii``. + + So rewriting ``pii: true`` as ``sensitivity: pii`` is not a change. + """ + sensitivity = column.get("sensitivity") + if sensitivity is None and column.get("pii"): + return "pii" + return sensitivity + + +def _sensitivity_changes( + old: Mapping[str, Any], new: Mapping[str, Any], col_name: str, path_prefix: str +) -> List[Change]: + """PII annotation drift, and any other change to a 0.7.x column's ``sensitivity``. + + A column gaining ``pii: true`` is a governance signal — downstream + consumers may have access controls that key off this flag. Surface as + INFO-level so it shows up in PR-review feedback without breaking the + build. A 0.7.x column says it with ``sensitivity`` (see :func:`_is_pii`), + and the change is reported at the key that carried it. + """ + old_pii, new_pii = _is_pii(old), _is_pii(new) + old_sensitivity, new_sensitivity = _sensitivity(old), _sensitivity(new) + if old_pii != new_pii: + if old.get("sensitivity") != new.get("sensitivity"): + key, before, after = "sensitivity", old_sensitivity, new_sensitivity + else: + key, before, after = "pii", old_pii, new_pii + return [ + Change( + path=f"{path_prefix}.{col_name}.{key}", + kind="column_pii_added" if new_pii else "column_pii_removed", + severity="info", + description=( + f"column '{col_name}' is now PII" + if new_pii + else f"column '{col_name}' PII annotation removed" + ), + before=before, + after=after, + ) + ] + if old_sensitivity != new_sensitivity: + return [ + Change( + path=f"{path_prefix}.{col_name}.sensitivity", + kind="column_sensitivity_changed", + severity="info", + description=( + f"column '{col_name}' sensitivity changed " + f"{old_sensitivity or 'unset'} -> {new_sensitivity or 'unset'}" + ), + before=old_sensitivity, + after=new_sensitivity, + ) + ] + return [] + + def diff_columns( baseline_expose: Mapping[str, Any], new_expose: Mapping[str, Any], @@ -355,27 +456,7 @@ def diff_columns( ) ) - # PII annotation drift. A column gaining ``pii: true`` is a - # governance signal — downstream consumers may have access - # controls that key off this flag. Surface as INFO-level so it - # shows up in PR-review feedback without breaking the build. - old_pii = bool(old.get("pii")) - new_pii = bool(new.get("pii")) - if old_pii != new_pii: - changes.append( - Change( - path=f"{path_prefix}.{col_name}.pii", - kind="column_pii_added" if new_pii else "column_pii_removed", - severity="info", - description=( - f"column '{col_name}' is now PII" - if new_pii - else f"column '{col_name}' PII annotation removed" - ), - before=old_pii, - after=new_pii, - ) - ) + changes.extend(_sensitivity_changes(old, new, col_name, path_prefix)) # Nullability changes. old_null = _is_nullable(old) @@ -582,33 +663,90 @@ def diff_sovereignty(baseline: Mapping[str, Any], new: Mapping[str, Any]) -> Lis return changes +#: Where an expose keeps its quality rules, and the keys that name one there: +#: pre-0.7 ``quality.tests``; in 0.7.x ``contract.dq.rules`` (``$defs.dqRule``) +#: and the inline ``contract.quality``. +_QUALITY_RULES: Tuple[Tuple[Tuple[str, ...], Tuple[str, ...], str], ...] = ( + (("quality", "tests"), ("name", "id"), "quality test"), + (("contract", "dq", "rules"), ("id",), "quality rule"), + (("contract", "quality"), ("rule",), "quality rule"), +) + + def diff_quality_severity( baseline_expose: Mapping[str, Any], new_expose: Mapping[str, Any], expose_id: str, expose_idx: int, ) -> List[Change]: - """A quality rule whose severity escalates warn -> error breaks downstream gating.""" - old_tests = _quality_tests_by_name(baseline_expose) - new_tests = _quality_tests_by_name(new_expose) + """A quality rule whose severity escalates to a failing one breaks downstream gating. + + Reads the rules of every location in ``_QUALITY_RULES`` and, from 0.7.6, + each column's own ``quality`` rules (named by ``id``, ODCS's shape). + ``critical`` fails a run as ``error`` does (``fluid_build.severity``), so + either one is an escalation from ``warn``, and one to the other is not. A + rule is matched across versions by name; one without a name is skipped. + """ + changes: List[Change] = [] + for location, keys, noun in _QUALITY_RULES: + changes.extend( + _severity_escalations( + _rules_by_name(_dig(baseline_expose, location), keys), + _rules_by_name(_dig(new_expose, location), keys), + path_prefix=f"exposes[{expose_idx}].{'.'.join(location)}", + owner=f"expose '{expose_id}'", + noun=noun, + ) + ) + + base_columns = _columns_by_name(_expose_schema(baseline_expose)[0]) + new_cols, new_at = _expose_schema(new_expose) + for col_name, new_col in _columns_by_name(new_cols).items(): + old_col = base_columns.get(col_name) + if old_col is None: + continue + changes.extend( + _severity_escalations( + _rules_by_name(old_col.get("quality"), ("id", "name")), + _rules_by_name(new_col.get("quality"), ("id", "name")), + path_prefix=f"exposes[{expose_idx}].{new_at}.{col_name}.quality", + owner=f"column '{col_name}' of expose '{expose_id}'", + noun="quality rule", + ) + ) + return changes + + +def _severity_escalations( + old_rules: Mapping[str, Sequence[Mapping[str, Any]]], + new_rules: Mapping[str, Sequence[Mapping[str, Any]]], + *, + path_prefix: str, + owner: str, + noun: str, +) -> List[Change]: + """A named rule escalated: failing in the new contract, and not in the old one. + A name can be on several rules (nothing makes an id unique), and each + version is judged by its most severe rule of that name, so a failing rule + cannot hide behind a ``warn`` one of the same id. + """ changes: List[Change] = [] - path_prefix = f"exposes[{expose_idx}].quality.tests" - for name, new_test in new_tests.items(): - old_test = old_tests.get(name) - if old_test is None: + for name, new_group in new_rules.items(): + old_group = old_rules.get(name) + if not old_group: continue - old_sev = str(old_test.get("severity", "warn")).lower() - new_sev = str(new_test.get("severity", "warn")).lower() - if old_sev != "error" and new_sev == "error": + old_sev = _most_severe(old_group) + new_sev = _most_severe(new_group) + if not is_error(old_sev) and is_error(new_sev): changes.append( Change( path=f"{path_prefix}.{name}.severity", kind="quality_severity_escalated", severity="breaking", description=( - f"quality test '{name}' on expose '{expose_id}' " - f"escalated to severity=error (previously {old_sev}); " + f"{noun} '{name}' on {owner} " + f"escalated to severity={new_sev} (previously {old_sev}); " f"downstream gating will now hard-fail" ), before=old_sev, @@ -618,17 +756,28 @@ def diff_quality_severity( return changes -def _quality_tests_by_name(expose: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: - out: dict[str, Mapping[str, Any]] = {} - quality = expose.get("quality") or {} - if not isinstance(quality, Mapping): +def _dig(node: Any, keys: Sequence[str]) -> Any: + for key in keys: + if not isinstance(node, Mapping): + return None + node = node.get(key) + return node + + +def _most_severe(rules: Sequence[Mapping[str, Any]]) -> str: + severities = [str(rule.get("severity", "warn")).lower() for rule in rules] + return next((sev for sev in severities if is_error(sev)), severities[0]) + + +def _rules_by_name(rules: Any, keys: Sequence[str]) -> dict[str, List[Mapping[str, Any]]]: + out: dict[str, List[Mapping[str, Any]]] = {} + if not isinstance(rules, (list, tuple)): return out - tests = quality.get("tests") or [] - for t in tests: - if isinstance(t, Mapping): - name = t.get("name") or t.get("id") + for rule in rules: + if isinstance(rule, Mapping): + name = next((rule.get(k) for k in keys if rule.get(k)), None) if isinstance(name, str): - out[name] = t + out.setdefault(name, []).append(rule) return out diff --git a/fluid_build/cli/_preview_panel.py b/fluid_build/cli/_preview_panel.py index 572ee4780..bd1320b1e 100644 --- a/fluid_build/cli/_preview_panel.py +++ b/fluid_build/cli/_preview_panel.py @@ -714,7 +714,10 @@ def render_refine_diff( try: from fluid_build.api.changelog import compare_contracts, render_text - report = compare_contracts(old_contract, new_contract) + # Not validated: the existing contract is often invalid (that can be why + # it is being refined), and the new one passed the runtime's schema + # gate already. ``fluid diff --baseline`` validates both. + report = compare_contracts(old_contract, new_contract, validate=False) except Exception: # noqa: BLE001 — diff is advisory; never block the write LOG.debug("render_refine_diff: changelog engine unavailable", exc_info=True) return False @@ -743,13 +746,18 @@ def render_refine_diff( try: from rich.panel import Panel as _RichPanel + from fluid_build._console import literal_text + + # The body quotes column names: a ``str`` would be read as markup. border = "red" if report.has_breaking else "yellow" - console.print(_RichPanel(body, title=f"[bold]{title}[/bold]", border_style=border)) + console.print( + _RichPanel(literal_text(body), title=f"[bold]{title}[/bold]", border_style=border) + ) return True except Exception: # noqa: BLE001 — fall through to plain text try: console.print(f"\n=== {title} ===") - console.print(body) + console.print(body, markup=False, emoji=False) return True except Exception: # noqa: BLE001 pass diff --git a/fluid_build/cli/diff.py b/fluid_build/cli/diff.py index 1ed9f5233..f4c7ab92e 100644 --- a/fluid_build/cli/diff.py +++ b/fluid_build/cli/diff.py @@ -17,11 +17,15 @@ import argparse import copy import logging +import math import os +import sys import time from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set +from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set +from .._console import _redact_str +from ..observability.secret_redactor import redact_secret_text from ..observability.tracing import traced_span as _traced_span from ..observability.tracing import traced_stage as _traced_stage from ._common import ( @@ -33,11 +37,13 @@ resolve_provider_from_contract, write_json, ) -from ._logging import info, warn +from ._logging import error, info, warn if TYPE_CHECKING: from fluid_build.iac.drift import StateDriftReport + from ..api.changelog import ContractValidationError + from ..api.changelog_types import ChangelogReport from ._diff_live import LiveDriftReport COMMAND = "diff" @@ -162,7 +168,8 @@ def register(subparsers: argparse._SubParsersAction): help=( "Path to a baseline (old) contract.fluid.yaml. When set, switches " "from drift mode to contract-vs-contract version diff and emits a " - "breaking-change classification." + "breaking-change classification. Both contracts must be schema-valid: " + "an invalid one exits 2 (diff_contract_invalid) before any output." ), ) p.add_argument( @@ -739,11 +746,11 @@ def _run_version_diff(args, logger: logging.Logger) -> int: Loads two contracts, runs the changelog engine, prints in the requested format, optionally writes a JSON envelope to ``--out``, and returns a non-zero exit code when ``--fail-on-breaking`` is set and any breaking - change was detected. + change was detected. A contract that is not schema-valid exits 2 (diff(1)'s + "trouble") before anything is printed or written, so it never reads as 0 + (no changes) or 1 (breaking). """ - from fluid_build.cli.console import cprint - - from ..api.changelog import compare_contracts, render_markdown, render_text + from ..api.changelog import ContractValidationError, compare_contracts if getattr(args, "env", None): # Environment overlays are a drift-mode concept (they shape the @@ -798,29 +805,25 @@ def _run_version_diff(args, logger: logging.Logger) -> int: "fluid.diff.format": getattr(args, "format", "text") or "text", }, ) as span: - report = compare_contracts(baseline, new) + try: + report = compare_contracts(baseline, new) + except ContractValidationError as exc: + raise _invalid_contracts_error( + exc, {"baseline": baseline_path, "new": new_path}, logger + ) from exc span.set_attribute("fluid.diff.breaking_count", len(report.breaking)) span.set_attribute("fluid.diff.non_breaking_count", len(report.non_breaking)) span.set_attribute("fluid.diff.info_count", len(report.info)) - # Render to stdout in the requested format. - fmt = getattr(args, "format", "text") or "text" - if fmt == "json": - # The JSON envelope is also written to --out (below) for CI - # artifact collection. Print to stdout here for piping. - import json as _json + # Render to stdout in the requested format, then always write the same + # envelope to --out so CI runners that don't parse stdout still get a + # machine-readable artifact. Both get the redacted report. + safe = _safe_report(report) + _print_version_diff(safe, getattr(args, "format", "text") or "text") - cprint(_json.dumps(report.to_dict(), indent=2)) - elif fmt == "markdown": - cprint(render_markdown(report)) - else: - cprint(render_text(report)) - - # Always write the structured envelope to --out so CI runners that don't - # parse stdout still get a machine-readable artifact. from ._common import write_json - write_json(args.out, report.to_dict()) + write_json(args.out, safe.to_dict()) info( logger, @@ -834,3 +837,112 @@ def _run_version_diff(args, logger: logging.Logger) -> int: if getattr(args, "fail_on_breaking", False) and report.has_breaking: return 1 return 0 + + +def _safe_report(report: ChangelogReport) -> ChangelogReport: + """``report`` with every value redacted and made JSON-safe, for stdout and ``--out``. + + A string goes through both redactors, as the CLI's error renderer does + (``cli/__init__.py``): ``redact_secret_text`` (JWTs, provider keys, + credentialed URLs, ``key=value``) and the console's pattern, which alone + catches ``api key: x``. A value JSON cannot hold, such as a date, bytes or + a set that YAML put in a column's quality rule, becomes text, so the + envelope always serialises: ``json.dump`` raised on it after the text + report had printed, and left ``--out`` cut short. + """ + import dataclasses + + from fluid_build.observability.secret_redactor import redact_value + + from ..api.changelog_types import ChangelogReport as _Report + + safe = _Report() + for change in (*report.breaking, *report.non_breaking, *report.info): + safe.add( + dataclasses.replace( + change, + **{ + field: _jsonable(redact_value(getattr(change, field))) + for field in ("path", "description", "before", "after") + }, + ) + ) + return safe + + +def _jsonable(value: Any) -> Any: + if isinstance(value, str): + return _redact_str(value) + if value is None or isinstance(value, (bool, int)): + return value + if isinstance(value, float): + return value if math.isfinite(value) else str(value) + if isinstance(value, Mapping): + # A key is contract text too: a free-form ``labels`` key can hold a credential. + return { + _jsonable(redact_secret_text(str(key))): _jsonable(item) for key, item in value.items() + } + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + if isinstance(value, (set, frozenset)): + return sorted((_jsonable(item) for item in value), key=repr) + return _redact_str(str(value)) + + +def _print_version_diff(report: ChangelogReport, fmt: str) -> None: + """Print a version-diff ``report`` (already made safe) to stdout as ``fmt``. + + JSON and Markdown are documents a program reads, so they are written + verbatim: a regex pass over the finished document cut through its + structure (a name ending in ``token=x`` lost the closing quote of its JSON + string, or the closing fence of its Markdown code span, which let the rest + of the line render as Markdown), and Rich would wrap a long line at the + terminal width and read ``[...]`` in a name as markup. Each value was + redacted before (:func:`_safe_report`). Text is printed literally, so a + name cannot style it, swap in an emoji or (``[/]``) crash it, and + soft-wrapped, as the drift report is, so each entry stays one greppable + line. + """ + from fluid_build.cli.console import cprint_literal + + from ..api.changelog import render_markdown, render_text + + if fmt == "json": + import json as _json + + sys.stdout.write(_json.dumps(report.to_dict(), indent=2) + "\n") + elif fmt == "markdown": + sys.stdout.write(render_markdown(report) + "\n") + else: + cprint_literal(render_text(report), soft_wrap=True, highlight=False) + + +def _invalid_contracts_error( + exc: ContractValidationError, paths: Dict[str, str], logger: logging.Logger +) -> CLIError: + """``diff_contract_invalid`` for the contracts ``compare_contracts`` refused. + + Each error is logged as ``fluid plan`` logs its schema errors, and the + message names every invalid file with the command that reports on it. + """ + for side, errors in exc.errors.items(): + for err in errors: + error(logger, "diff_contract_schema_error", side=side, path=paths[side], detail=err) + invalid = list(exc.errors) + named = "; ".join( + f"the {side} contract {paths[side]} has {len(exc.errors[side])} error(s)" + for side in invalid + ) + commands = " and ".join(f"``fluid validate {paths[side]}``" for side in invalid) + return CLIError( + 2, + "diff_contract_invalid", + { + "invalid": invalid, + "errors": {side: exc.errors[side] for side in invalid}, + "message": ( + f"``fluid diff --baseline`` compares valid contracts only: {named}. " + f"Run {commands} for the full report." + ), + }, + ) diff --git a/fluid_build/engines/dbt/_test_mapping.py b/fluid_build/engines/dbt/_test_mapping.py index ade15fb1d..434657f48 100644 --- a/fluid_build/engines/dbt/_test_mapping.py +++ b/fluid_build/engines/dbt/_test_mapping.py @@ -76,6 +76,10 @@ import re from typing import Any, Mapping, Sequence +# Key recognition lives in ``util.contract`` so ``fluid diff`` judges a key the way this +# engine builds one without importing every engine; re-exported for ``_tm.*`` callers. +from ...util.contract import column_is_key, is_truthy # noqa: F401 + # ── Package prefixes / canonical test names ─────────────────────────────── # The single numeric range/between dialect (see module docstring). RANGE_TEST_NAME = "dbt_expectations.expect_column_values_to_be_between" @@ -275,7 +279,6 @@ def sentinel_test(kind: str, column: str | None = None) -> str: # ``maximum`` / FK spellings below are accepted best-effort because contracts # express quality intent inline in practice. -_PK_KEYS = ("primary", "primaryKey", "primary_key", "pk", "isPrimary") _ENUM_KEYS = ("enum", "acceptedValues", "accepted_values") _FK_KEYS = ("foreign_key", "foreignKey", "references", "relationship_to") @@ -298,32 +301,6 @@ def sentinel_test(kind: str, column: str | None = None) -> str: _REFERENCES_RE = re.compile(r"^\s*references\s+(\S+?)\.(\w+)\s*$", re.IGNORECASE) -def is_truthy(value: Any) -> bool: - """Loose truthiness for YAML/JSON booleans-as-strings (``"true"``/``True``).""" - if isinstance(value, bool): - return value - if isinstance(value, str): - return value.strip().lower() in ("true", "1", "yes") - return bool(value) - - -def column_is_key(col: Mapping[str, Any]) -> bool: - """True when the column is declared a uniqueness key / primary key.""" - if is_truthy(col.get("unique")): - return True - if any(is_truthy(col.get(k)) for k in _PK_KEYS): - return True - if str(col.get("semanticType") or "").strip().lower() in ("identifier", "primary_key"): - return True - labels = col.get("labels") - if isinstance(labels, Mapping): - if is_truthy(labels.get("unique")): - return True - if str(labels.get("constraint") or "").strip().lower() in ("primary_key", "unique"): - return True - return False - - def validation_rules(col: Mapping[str, Any]) -> list[Mapping[str, Any]]: """The column's ``validationRules[]`` entries (schema-canonical).""" raw = col.get("validationRules") diff --git a/fluid_build/util/contract.py b/fluid_build/util/contract.py index 02891e6e8..0db30e1f5 100644 --- a/fluid_build/util/contract.py +++ b/fluid_build/util/contract.py @@ -1077,3 +1077,37 @@ def slugify_identifier(value: str, *, fallback: str = "project") -> str: if slug[0].isdigit(): slug = f"x-{slug}" return slug + + +# ── Column keys ──────────────────────────────────────────────────────────── +# Which columns are keys, read the same way wherever it matters: the dbt engine +# gives these a ``primary_key`` constraint (``engines/dbt/schema_yml.py``), and +# ``fluid diff`` reports one that becomes nullable as breaking. + +_PK_KEYS = ("primary", "primaryKey", "primary_key", "pk", "isPrimary") + + +def is_truthy(value: Any) -> bool: + """Loose truthiness for YAML/JSON booleans-as-strings (``"true"``/``True``).""" + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in ("true", "1", "yes") + return bool(value) + + +def column_is_key(col: Mapping[str, Any]) -> bool: + """True when the column is declared a uniqueness key / primary key.""" + if is_truthy(col.get("unique")): + return True + if any(is_truthy(col.get(k)) for k in _PK_KEYS): + return True + if str(col.get("semanticType") or "").strip().lower() in ("identifier", "primary_key"): + return True + labels = col.get("labels") + if isinstance(labels, Mapping): + if is_truthy(labels.get("unique")): + return True + if str(labels.get("constraint") or "").strip().lower() in ("primary_key", "unique"): + return True + return False diff --git a/tests/test_diff_changelog.py b/tests/test_diff_changelog.py index 56ab7b21a..f4e03f6cc 100644 --- a/tests/test_diff_changelog.py +++ b/tests/test_diff_changelog.py @@ -45,7 +45,12 @@ @pytest.fixture def v1_v2(): - """The on-disk fixture pair loaded as parsed contracts.""" + """The on-disk fixture pair loaded as parsed contracts. + + They declare 0.7.3 but keep the pre-0.7 shapes the legacy rules read (a + flat ``schema``, ``quality.tests``, a root ``agentPolicy``), which no + bundled schema accepts, so they are diffed with ``validate=False``. + """ import yaml v1 = yaml.safe_load((FIXTURES / "v1.fluid.yaml").read_text()) @@ -60,14 +65,14 @@ def v1_v2(): def test_same_contract_produces_empty_diff(v1_v2): v1, _ = v1_v2 - report = compare_contracts(v1, v1) + report = compare_contracts(v1, v1, validate=False) assert report.total == 0 assert not report.has_breaking def test_report_total_matches_section_sums(v1_v2): v1, v2 = v1_v2 - report = compare_contracts(v1, v2) + report = compare_contracts(v1, v2, validate=False) assert report.total == (len(report.breaking) + len(report.non_breaking) + len(report.info)) @@ -254,7 +259,7 @@ def test_quality_severity_unchanged_no_change(): def test_fixture_v1_to_v2_classification(v1_v2): """Full pipeline against the on-disk fixture pair.""" v1, v2 = v1_v2 - report = compare_contracts(v1, v2) + report = compare_contracts(v1, v2, validate=False) # v2 vs v1 should detect: # - email column removed (breaking) @@ -296,7 +301,7 @@ def test_fixture_v1_to_v2_classification(v1_v2): def test_render_text_contains_summary_line(v1_v2): v1, v2 = v1_v2 - report = compare_contracts(v1, v2) + report = compare_contracts(v1, v2, validate=False) out = render_text(report) assert "Summary:" in out assert "breaking" in out.lower() @@ -309,7 +314,7 @@ def test_render_text_empty_report_says_no_changes(): def test_render_markdown_includes_section_headings(v1_v2): v1, v2 = v1_v2 - report = compare_contracts(v1, v2) + report = compare_contracts(v1, v2, validate=False) md = render_markdown(report) assert "# Contract changelog" in md if report.breaking: @@ -320,7 +325,7 @@ def test_render_markdown_includes_section_headings(v1_v2): def test_report_to_dict_round_trips_through_json(v1_v2): v1, v2 = v1_v2 - report = compare_contracts(v1, v2) + report = compare_contracts(v1, v2, validate=False) data = report.to_dict() # Must be JSON-serialisable s = json.dumps(data) @@ -334,14 +339,26 @@ def test_report_to_dict_round_trips_through_json(v1_v2): # --------------------------------------------------------------------------- +def _cli_contracts(tmp_path): + """A valid 0.7.6 pair for the CLI, which validates both: ``new`` drops a column.""" + import yaml + + old = tmp_path / "old.fluid.yaml" + new = tmp_path / "new.fluid.yaml" + old.write_text(yaml.safe_dump(_contract_076([_ID_COLUMN, {"name": "note", "type": "string"}]))) + new.write_text(yaml.safe_dump(_contract_076([_ID_COLUMN]))) + return str(old), str(new) + + def test_cli_diff_baseline_mode_smoke(tmp_path, monkeypatch): """End-to-end: ``fluid diff --baseline `` produces JSON and exits 0 by default.""" from fluid_build.cli import diff as diff_mod + old, new = _cli_contracts(tmp_path) out_path = tmp_path / "diff.json" args = argparse.Namespace( - contract=str(FIXTURES / "v2.fluid.yaml"), - baseline=str(FIXTURES / "v1.fluid.yaml"), + contract=new, + baseline=old, env=None, state=None, out=str(out_path), @@ -365,9 +382,10 @@ def test_cli_diff_baseline_fail_on_breaking(tmp_path): """``--fail-on-breaking`` returns 1 when breaking changes exist.""" from fluid_build.cli import diff as diff_mod + old, new = _cli_contracts(tmp_path) args = argparse.Namespace( - contract=str(FIXTURES / "v2.fluid.yaml"), - baseline=str(FIXTURES / "v1.fluid.yaml"), + contract=new, + baseline=old, env=None, state=None, out=str(tmp_path / "diff.json"), @@ -384,9 +402,10 @@ def test_cli_diff_baseline_idempotent(tmp_path): """``--fail-on-breaking`` returns 0 when comparing a contract to itself.""" from fluid_build.cli import diff as diff_mod + old, _ = _cli_contracts(tmp_path) args = argparse.Namespace( - contract=str(FIXTURES / "v1.fluid.yaml"), - baseline=str(FIXTURES / "v1.fluid.yaml"), + contract=old, + baseline=old, env=None, state=None, out=str(tmp_path / "diff.json"), @@ -860,14 +879,14 @@ def test_moving_columns_from_schema_to_contract_schema_is_not_a_change(): old = _contract_flat([{"name": "id", "type": "STRING", "nullable": False}]) new = _contract_076([{"name": "id", "type": "string", "required": True}]) - assert compare_contracts(old, new).total == 0 + assert compare_contracts(old, new, validate=False).total == 0 def test_a_column_dropped_while_moving_to_contract_schema_is_still_found(): old = _contract_flat([{"name": "id", "type": "STRING"}, {"name": "note", "type": "STRING"}]) new = _contract_076([{"name": "id", "type": "string"}]) - report = compare_contracts(old, new) + report = compare_contracts(old, new, validate=False) assert report.total == 1 assert [(c.kind, c.path) for c in report.breaking] == [ @@ -937,9 +956,10 @@ def _fake_traced_span(name, attributes=None): monkeypatch.setattr(diff_mod, "_traced_span", _fake_traced_span) + old, new = _cli_contracts(tmp_path) args = argparse.Namespace( - contract=str(FIXTURES / "v2.fluid.yaml"), - baseline=str(FIXTURES / "v1.fluid.yaml"), + contract=new, + baseline=old, env=None, state=None, out=str(tmp_path / "diff.json"), diff --git a/tests/test_diff_changelog_hardening.py b/tests/test_diff_changelog_hardening.py new file mode 100644 index 000000000..6ed160dce --- /dev/null +++ b/tests/test_diff_changelog_hardening.py @@ -0,0 +1,821 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fluid diff --baseline``: what reaches the gate, how it is printed, what 0.7.x carries. + +Once 0.7.x columns reached ``--fail-on-breaking`` (PR #789), its reviews found +four gaps. The diff ran on contracts no schema accepts, so a value of the wrong +type hid a change. It printed contract names raw to a terminal and to Markdown. +It ignored the quality, sensitivity and primary-key information a 0.7.x contract +keeps under ``exposes[].contract``. And it read no columns from a +``{"fields": [...]}`` schema. +""" + +from __future__ import annotations + +import argparse +import json +import logging + +import pytest +import yaml + +from fluid_build.api.changelog import ( + ContractValidationError, + compare_contracts, + render_markdown, + render_text, +) +from fluid_build.api.changelog_rules import diff_columns, diff_quality_severity +from fluid_build.api.changelog_types import Change, ChangelogReport +from tests.test_diff_changelog import _ID_COLUMN as _ID +from tests.test_diff_changelog import _contract_076 + + +def _contract(columns, **section): + """``_contract_076`` with ``section`` (``dq``, ``quality``) beside the schema too.""" + contract = _contract_076(columns) + contract["exposes"][0]["contract"].update(section) + return contract + + +def _run_cli(tmp_path, old, new, *, fmt="json", fail_on_breaking=False): + """``fluid diff --baseline `` on two contracts written to ``tmp_path``.""" + from fluid_build.cli import diff as diff_mod + + old_path, new_path = tmp_path / "old.fluid.yaml", tmp_path / "new.fluid.yaml" + old_path.write_text(yaml.safe_dump(old), encoding="utf-8") + new_path.write_text(yaml.safe_dump(new), encoding="utf-8") + args = argparse.Namespace( + contract=str(new_path), + baseline=str(old_path), + env=None, + state=None, + out=str(tmp_path / "diff.json"), + exit_on_drift=False, + fail_on_breaking=fail_on_breaking, + format=fmt, + cmd="diff", + ) + return diff_mod.run(args, logging.getLogger("test.diff")) + + +# --------------------------------------------------------------------------- +# Both contracts are schema-validated before they are diffed +# --------------------------------------------------------------------------- + + +def test_every_contract_below_validates_against_the_bundled_schema(): + """The shapes these tests diff: if 0.7.6 stops accepting them they prove nothing.""" + from fluid_build.schema_manager import FluidSchemaManager + + shapes = [ + _contract([_ID]), + _contract([{**_ID, "tags": ["primary-key"]}, {"name": "email", "type": "string"}]), + _contract([{"name": "email", "type": "string", "sensitivity": "pii"}]), + _contract( + [ + { + "name": "email", + "type": "string", + "quality": [{"id": "email-valid", "type": "text", "severity": "warning"}], + } + ], + dq={"rules": [{"id": "complete", "type": "completeness", "severity": "critical"}]}, + quality=[{"rule": "positive", "expression": "amount > 0", "severity": "warning"}], + ), + ] + for shape in shapes: + result = FluidSchemaManager().validate_contract(shape, offline_only=True) + assert result.is_valid, result.errors + + +def test_a_value_of_the_wrong_type_cannot_hide_a_change_from_the_diff(): + """A ``type`` that is a number was never compared, so the retype passed the gate.""" + old = _contract([_ID, {"name": "amount", "type": "decimal(18,2)"}]) + new = _contract([_ID, {"name": "amount", "type": 18}]) + + with pytest.raises(ContractValidationError) as excinfo: + compare_contracts(old, new) + + assert set(excinfo.value.errors) == {"new"} + assert any("exposes[0].contract.schema[1].type" in e for e in excinfo.value.errors["new"]) + + +def test_every_invalid_side_is_reported_at_once(): + old = _contract([_ID]) + del old["metadata"]["owner"] + new = _contract([{"name": "id"}]) + + with pytest.raises(ContractValidationError) as excinfo: + compare_contracts(old, new) + + assert set(excinfo.value.errors) == {"baseline", "new"} + assert "baseline contract" in str(excinfo.value) and "new contract" in str(excinfo.value) + + +@pytest.mark.parametrize("version", ["0.6.0", "0.5.7", "0.4"]) +def test_a_pre_07_contract_is_refused_with_the_upgrade_message(version): + old = _contract([_ID]) + old["fluidVersion"] = version + + with pytest.raises(ContractValidationError) as excinfo: + compare_contracts(old, _contract([_ID])) + + (message,) = excinfo.value.errors["baseline"] + assert "no longer supported" in message and "0.7.x" in message + + +def test_a_contract_without_a_fluid_version_is_refused(): + new = _contract([_ID]) + del new["fluidVersion"] + + with pytest.raises(ContractValidationError) as excinfo: + compare_contracts(_contract([_ID]), new) + + assert set(excinfo.value.errors) == {"new"} + + +@pytest.mark.parametrize("version", ["garbage", 7]) +def test_a_fluid_version_that_is_not_a_version_is_refused_not_raised(tmp_path, version): + """The validator raised on it, and the CLI exited 1 (``diff_failed``), which reads as breaking.""" + from fluid_build.cli._common import CLIError + + new = _contract([_ID]) + new["fluidVersion"] = version + + with pytest.raises(ContractValidationError) as excinfo: + compare_contracts(_contract([_ID]), new) + assert set(excinfo.value.errors) == {"new"} + with pytest.raises(CLIError) as cli_error: + _run_cli(tmp_path, _contract([_ID]), new, fail_on_breaking=True) + assert (cli_error.value.exit_code, cli_error.value.event) == (2, "diff_contract_invalid") + + +def test_the_cli_names_the_baseline_when_only_the_baseline_is_invalid(tmp_path): + from fluid_build.cli._common import CLIError + + old = _contract([_ID, {"name": "amount", "type": 18}]) + + with pytest.raises(CLIError) as excinfo: + _run_cli(tmp_path, old, _contract([_ID])) + + assert excinfo.value.exit_code == 2 + assert excinfo.value.context["invalid"] == ["baseline"] + assert str(tmp_path / "old.fluid.yaml") in excinfo.value.context["message"] + + +def test_validate_false_diffs_what_it_is_given(): + """The refine preview's opt-out: an advisory diff of a contract still being repaired.""" + old = {"exposes": [{"id": "x", "schema": [{"name": "a", "type": "STRING"}]}]} + new = {"exposes": [{"id": "x", "schema": []}]} + + report = compare_contracts(old, new, validate=False) + + assert [c.kind for c in report.breaking] == ["column_removed"] + + +@pytest.mark.parametrize("fail_on_breaking", [True, False]) +def test_the_cli_exits_2_on_an_invalid_contract_whatever_fail_on_breaking_says( + tmp_path, fail_on_breaking +): + """diff(1)'s 2 is "trouble": an invalid contract never reads as 0 (clean) or 1 (breaking).""" + from fluid_build.cli._common import CLIError + + old = _contract([_ID, {"name": "amount", "type": "decimal(18,2)"}]) + new = _contract([_ID, {"name": "amount", "type": 18}]) + + with pytest.raises(CLIError) as excinfo: + _run_cli(tmp_path, old, new, fail_on_breaking=fail_on_breaking) + + assert excinfo.value.exit_code == 2 + assert excinfo.value.event == "diff_contract_invalid" + context = excinfo.value.context + assert context["invalid"] == ["new"] + assert str(tmp_path / "new.fluid.yaml") in context["message"] + assert "fluid validate" in context["message"] + assert not (tmp_path / "diff.json").exists() + + +# --------------------------------------------------------------------------- +# Names are printed as data: escaped for a terminal and inert in Markdown +# --------------------------------------------------------------------------- + + +def _report(*names): + """A report naming each of ``names`` as an added column, as ``diff_columns`` words it.""" + report = ChangelogReport() + for name in names: + report.add( + Change( + path=f"exposes[0].contract.schema.{name}", + kind="column_added", + severity="non_breaking", + description=f"column '{name}' added to expose 'orders'", + ) + ) + return report + + +@pytest.mark.parametrize( + "name, spelled", + [ + pytest.param( + "x\x1b]8;;https://evil.example\x07click\x1b]8;;\x07", + "x\\x1b]8;;https://evil.example\\x07click\\x1b]8;;\\x07", + id="osc8-hyperlink", + ), + pytest.param("x\x1b[2J", "x\\x1b[2J", id="clear-screen"), + pytest.param("x\x9b31m", "x\\x9b31m", id="c1-csi"), + pytest.param("a\rb", "a\\x0db", id="carriage-return"), + pytest.param("a\tb", "a\\x09b", id="tab"), + pytest.param("admin\u202egnp.exe", "admin\\u202egnp.exe", id="bidi-override"), + pytest.param("e\u200bmail", "e\\u200bmail", id="zero-width-space"), + pytest.param("e\ufe0fmail", "e\\ufe0fmail", id="variation-selector"), + pytest.param("\u3164", "\\u3164", id="hangul-filler"), + pytest.param("\u2800", "\\u2800", id="braille-blank"), + pytest.param("x\U000e0041", "x\\U000e0041", id="tag-character"), + pytest.param("a\\x1b", "a\\\\x1b", id="literal-backslash-stays-distinct"), + pytest.param("straße_日本", "straße_日本", id="printable-unicode-is-kept"), + ], +) +def test_render_text_spells_out_what_a_terminal_would_act_on(name, spelled): + out = render_text(_report(name)) + + assert f"column '{spelled}' added" in out + assert f"exposes[0].contract.schema.{spelled}:" in out + assert all(ch == "\n" or ch.isprintable() for ch in out) + + +def test_a_newline_in_a_name_cannot_forge_a_line(): + out = render_text(_report("a\nSummary: 0 breaking, 0 non-breaking, 0 info")) + + assert [line for line in out.splitlines() if line.startswith("Summary:")] == [ + "Summary: 0 breaking, 1 non-breaking, 0 info" + ] + assert "a\\x0aSummary: 0 breaking" in out + + +def _html(markdown): + from markdown_it import MarkdownIt + + return MarkdownIt("commonmark").enable(["table", "strikethrough"]).render(markdown) + + +@pytest.mark.parametrize( + "name", + [ + pytest.param("