diff --git a/.secrets.baseline b/.secrets.baseline index b7a2d390..52b748b2 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -404,14 +404,14 @@ "filename": "fluid_build/providers/_sql_safety.py", "hashed_secret": "fb2620e2ed7e530fcb1ea0a2b2456751057a5178", "is_verified": false, - "line_number": 237 + "line_number": 259 }, { "type": "Secret Keyword", "filename": "fluid_build/providers/_sql_safety.py", "hashed_secret": "6d613a1ee01eec4c0f8ca66df0db71dca0c6e1cf", "is_verified": false, - "line_number": 238 + "line_number": 260 } ], "fluid_build/providers/datamesh_manager/README.md": [ @@ -1710,7 +1710,7 @@ "filename": "tests/test_contract_validation_ext2.py", "hashed_secret": "1a91d62f7ca67399625a4368a6ab5d4a3baa6073", "is_verified": false, - "line_number": 577 + "line_number": 592 } ], "tests/test_credential_adapters.py": [ @@ -2317,5 +2317,5 @@ } ] }, - "generated_at": "2026-11-12T21:55:04Z" + "generated_at": "2026-11-12T22:46:31Z" } diff --git a/AGENTS.md b/AGENTS.md index d2102957..7d8d0ea0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -768,6 +768,33 @@ onprem = "my_pkg.iac:OnPremIacPlugin" - **Tests.** `tests/iac/test_iac_entrypoint_lazy_plugin.py` builds a real throwaway `.dist-info` and is the pattern to copy. +## `fluid test` on Databricks: results table and dashboard (2026-11-12) + +- **Live checks.** `providers/databricks_validation.py` reads columns from the Unity + Catalog tables API and runs the row count and the rules on `DATABRICKS_WAREHOUSE_ID` + through `providers/databricks/statements.py` (Statement Execution API, `:name` + parameters, credential-scrubbed errors). Contract types go through + `engines/dbt/_types.sql_type(..., "databricks")` before they are compared. +- **One reading of the rules.** `providers/quality_engine.py` is the native engine, and + `quality_engine.value_domain` is what a `valid_values` rule allows: the rule's own list, + else every declaration its column makes (`enum`, `regex`, `range` validationRules, and + `labels.references`, resolved by the provider's `resolve_reference`; Databricks finds the + table by its `fluid_product_id` tag). The SodaCL exporter renders the same reading (a + reference or a strict bound stays unmapped); the dbt test mapping + (`engines/dbt/_test_mapping.py`) still has its own. Identifiers are quoted per dialect + (backticks on Databricks and BigQuery, where `"x"` is a string), string literals through + `_sql_safety.quote_string_literal_for`. `schema` rules are decided in + `cli/contract_validation.py::_decide_schema_rules` from the live column comparison. +- **Same cases everywhere.** `cli/_test_cases.py` lists the run's test cases (each rule, then + each structural category). `--output junit` and `--results-table` + (`cli/_test_results_databricks.py`) both read it, so Jenkins and the table agree. +- **Ownership.** The results table is created if missing; the catalog and schema never are, + a product's own schema is refused, and a table without the result columns is never written. + A run is appended in one statement (its rows bound as one JSON parameter), so it is never + half-written. Databricks schemas are never served from the validation cache. + The dashboard (`cli/_test_dashboard_databricks.py`) is found by its workspace path, created, + PATCHed only when its definition differs, and published without embedded credentials. + ## Links - **Documentation**: [https://agenticstiger.github.io/forge_docs/](https://agenticstiger.github.io/forge_docs/) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b3820c2..ca202674 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 script needs a SQL warehouse that runs SQL scripting. A dbt build that `--mode amend-and-build` runs locally is followed by the same re-declaration. +- **`fluid test` checks a Databricks product live, and can keep its results in Unity Catalog.** + `fluid test` used to answer "Live validation against Unity Catalog is not available" for + `platform: databricks`. It now reads the table's columns from the Unity Catalog tables API + (no compute), and runs the row count and every `dq.rules[]` entry as SQL on + `DATABRICKS_WAREHOUSE_ID`, signing in as `fluid apply` and `fluid verify` do (the + `[databricks]` extra). Bronze products count like dbt-built ones. Two new flags: + - `--results-table CATALOG.SCHEMA.TABLE` appends one row per check of the run (each rule, then + each structural check, the same cases `--output junit` reports, which still works beside + it): run id and time, product id, version and layer, expose and table, rule id, type and + severity, passed, observed value, threshold, message, forge version. The table is created if + missing in a schema the operator already has; forge never creates the catalog or the schema, + refuses the product's own schema (before any check runs), and writes nothing into an + existing table without these columns. A run is appended in one statement, so it is never + half-written, and every value is a bound parameter. It refuses `--no-data`, which runs no + live check to record; Databricks schemas are read live, never from the validation cache. + - `--results-dashboard WORKSPACE_PATH` keeps an AI/BI (Lakeview) dashboard over that table: + the pass rate of the latest runs, the pass rate per product (coloured by layer), the failing + checks and the daily trend. It is found by its workspace path (no state file, so ephemeral CI + agents never duplicate it), created on first use, updated only when its definition changed, + and published without embedded credentials. + The row shape follows Databricks Labs DQX's summary-metrics table and Elementary's test + results; the dashboard lookup follows databricks-solutions/ai-dev-kit. No code is taken from + either. + ### Changed - **`fluid apply --mode amend-and-build` runs a Databricks dbt build now.** A dbt build with @@ -78,9 +102,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 path exits 1 where it used to write a template instead. A `fluid ai setup` choice and a single keyring key still come first, and `GEMINI_API_KEY` with `GOOGLE_API_KEY` is one provider. Pick one with `--llm-provider` or `FLUID_LLM_PROVIDER`. +- **The native quality engine runs the rules a contract already declares in full.** A + `valid_values` rule that names no values takes them from its column: its `enum` + validationRule (`"A | B"` or `"A,B"`), its `regex` validationRule, its `range` + validationRule (`">= 0"`, `">= 0 and <= 100"`, `"between 0 and 100"`), or its + `labels.references: .` (on Databricks, the referenced product's table is + found by the `fluid_product_id` tag `fluid apply` puts on it, the tested table's catalog + first, and every non-null value must be in its column). A `completeness`, `accuracy` or + `valid_values` rule may list several comma-separated columns, each of which must hold (a + `valid_values` column against its own declaration); a `uniqueness` or `freshness` rule over + several columns is reported as not run, never guessed. A `schema` rule passes when the live + table has every contract column with its type, and fails, at the rule's severity, when it + does not or could not be read. An `accuracy` rule with no threshold is reported as not run + (it used to pass whenever the column held a value). Every rule's result, passed or not, now + carries what was measured (`--output json` adds `rule_id`, `rule_type` and `expose_id` to + each check). `--engine soda` renders the same column declarations (`valid values`, + `valid regex`, `valid min` / `valid max`, one check per listed column); a reference or a + strict bound stays unmapped there. On the tower-management industry pack `fluid test` ran + 38 of its 168 rules before; it now runs every rule of every deployed product. ### Fixed +- **Quality checks measured a constant on BigQuery (and would have on Databricks).** The + completeness, uniqueness, accuracy and valid_values checks quoted their column the ANSI way, + `"col"`, which BigQuery and Spark SQL read as a string literal, and cast ratios to + `DOUBLE PRECISION`, which neither has. Columns are now backtick-quoted and ratios cast to + `FLOAT64` / `DOUBLE` on those two dialects; ANSI, DuckDB and Snowflake SQL is unchanged. +- **A valid value containing a quote never matched on BigQuery or Databricks.** String literals + doubled the quote (`'it''s'`), which Databricks reads as the adjacent literals `'it'` and `'s'` + (checked on a SQL warehouse: the value becomes `its`) and BigQuery rejects. The new + `_sql_safety.quote_string_literal_for(dialect, value)` escapes it as `\'` there, after + doubling every backslash, so no value can end the literal early. + - **`fluid ai models --provider` works, and takes the names `fluid forge --llm-provider` takes.** The flag stored its value where the global infrastructure `--provider` keeps its own, so the global check read `gemini` as an infrastructure provider: every diff --git a/HONESTLY_TESTED.md b/HONESTLY_TESTED.md index 55ce77ba..d0d5d52f 100644 --- a/HONESTLY_TESTED.md +++ b/HONESTLY_TESTED.md @@ -98,6 +98,8 @@ Markers declared in `pyproject.toml`. | Stage 1 real tofu, stand-in workspace: Genie space | `tests/iac/test_iac_databricks_genie_fake_e2e.py` | ✅ 3. The real `fluid` CLI and `tofu`: validate, dry-run, apply (the space after the tables, shared with the readers), a second apply that changes nothing, the block removed (trashed); a Genie failure fails the apply after the tables with its fix and the next apply converges; an invalid block refused before `tofu` | The stand-in is modelled on the measured API, not the service itself | | Manual live, Free Edition, Genie space, 2026-11-12 | the real `fluid` CLI, no test file: branch `feat/databricks-genie-space`, databricks-sdk 0.162.0, a copy of industry-models tower-management `dp019_site_360` in the scratch catalog `fluid_iactest_genie` (no dbt build, no MENA sovereignty block: the workspace is in us-west-2), 6 synthetic rows | ✅ dry-run (would create, would share), apply `+73` then the space created and shared `CAN_RUN` with two persona service principals, a second apply `+0 ~0 -0` and "no change", the question "Which on-air sites in Amman are colocation-ready with hybrid power and no standing risk?" answered through the Genie Conversation API with the 2 expected sites (its SQL took the latest `snapshot_date` and the inverse of the `at_risk_site_count` filter from the instruction), a metric view created by hand picked up ("updated: metric views", then "no change"), the block removed (trashed). Two defects found and fixed in the run: the metric-view read failed on a schema not created yet, and the service keeps a metric view among the tables. A re-check after the security review (the plugin's own hook over `samples.nyctaxi.trips`, nothing created in Unity Catalog) found the list shows a new space only ~9 s later (a run straight after made a second space) and that a taken title is suffixed on create and 409 on PATCH; fixed, then create, an immediate second run ("no change", same space), and the block removed (trashed, via the folder check) passed live. Swept: the space trashed, `DROP CATALOG fluid_iactest_genie CASCADE`, 0 left | A reader without `SELECT` asking (no persona credential here; the access model is Genie's documented one); a service principal as the deploying identity (its folder is `/Users/`, the one Databricks Asset Bundles build from `current_user.userName`; a create that lands anywhere else is trashed and stops the apply); a space over several products | | Stage 2 emulator (nightly, non-gating) | `tests/iac/test_iac_databricks_minilake.py` (job `nightly-databricks-minilake`) | ⚠️ 1 passed, 4 strict xfails (manual dispatch, run 42267640334, 2026-11-06, minilake v1.9.0). Schema, table and grant creates work, the replace gate holds, adding a column plans no replace, and a grant revoke is classified not gated. The four tests about those resources apply the module without its tag assignments (the `untagged` fixture). An xfail is satisfied only by its own gap's failure (`MinilakeGap`), and a pass fails the run, so an image that closes a gap turns the job red until the mark comes off. Earlier: 1/4 from the first run (2026-10-30) to the 2026-11-04 nightly, then 0/4 after #831, every apply stopping at the first tag. | A second plan is not empty and no in-place table update applies: minilake drops a CREATE TABLE's `TBLPROPERTIES` and `COMMENT`, and DuckDB cannot parse the `ALTER TABLE ... SET TBLPROPERTIES` the provider sends on every update of a table that sets properties. No tag is applied: minilake has no entity-tag-assignments API (it answers 501). v1.9.3 (2026-11-05) has both gaps, so the pin stays; an upstream issue for each is drafted, not filed. Jobs and workspace files are not in this flow yet. | +| Stage 1 unit: `fluid test` on Databricks, results table, dashboard | `tests/providers/test_databricks_validation.py`, `tests/providers/test_databricks_statements.py`, `tests/providers/test_quality_engine_column_rules.py`, `tests/cli/test_contract_validation_schema_rules.py`, `tests/cli/test_fluid_test_results_table.py`, `tests/cli/test_fluid_test_dashboard.py` | ✅ Against a stubbed `api_client.do`: the Unity Catalog columns and the contract's types through the emitter's type table (`VARCHAR` is `string`), a missing table (404 → `missing_resource`, and every rule reported as not run with no statement), the row count, the warehouse check; the statements over `databricks_enforce.SqlRunner` with `:name` parameters, polling, timeout cancel, chunks, type conversion and scrubbed errors; every rule kind in the Databricks dialect (backticked identifiers, `\'` literals, `timestampdiff` freshness, `RLIKE`), comma-listed columns, values from the column's enum, regex or range rule, references resolved by the `fluid_product_id` tag (same catalog first, ambiguity refused, cached); `schema` rules from the column comparison; the result rows (one per JUnit case, `rule_type` `structure` for the structural ones, expose version, UTC run time), the batched parameterised `INSERT`, the refusals (bad name, a product's own schema, a table without the columns), the fix-its; the dashboard definition's structure and the create, unchanged, re-publish, update-with-etag and create-race paths. | What Databricks accepts: the manual live row below. | +| Manual live, Free Edition (founder-approved), 2026-11-12: `fluid test` results | the real `fluid` CLI, no test file: `fluid test --env databricks --output junit --results-table fluid_iactest_dqr.dq.fluid_test_results --results-dashboard '/Workspace/Shared/fluid_iactest_dqr/Data quality'` on the 19 tower-management products in `northwind_towers` (read only: `SELECT` and metadata reads), contracts derived from industry-models main | ✅ 19 runs, 17-34 s each, 18 exit 0; `dp019_site_360` exits 1 because its table is not deployed in the demo workspace (`missing_resource`; its rules are reported as not run, and no statement is sent). 168 rule cases: the 155 on deployed tables all ran (146 pass, 9 freshness rules fail at `warn`: the bronze data is hours old against a 15-minute window), every rule kind the pack uses among them: enum, regex and range values (#4's range rule over five columns included), 23 references (3 of them bronze references outside `consumes[]`, resolved by the `fluid_product_id` tag), `schema` rules and comma-listed completeness rules. 396 rows appended, one per JUnit case; the dashboard was created by the first run, found unchanged by every later one, and updated (PATCH, re-publish) once when its definition changed; in a browser it shows the counters (88.1% of rules passing, 19 products, 20 failing rules at that point), the per-product bars by layer, the trend and the failing-checks table. Afterwards the catalog `fluid_iactest_dqr` was dropped CASCADE and the dashboard deleted. | Lakeview validates widget specs only when the dashboard renders, so a widget shape Databricks changes would show as "Invalid widget definition" without failing the command (the table widget did, until it carried every column key). | | Stage 3 live | — | ❌ not automated: manual maintainer runs only, never CI. Two have run: the manual rows above (2026-11-01 and 2026-11-03). | Everything the rows above leave to the live tier. | ## What's NOT tested (deliberately deferred) diff --git a/fluid_build/_error_catalog.py b/fluid_build/_error_catalog.py index a7f6f068..deefc5a5 100644 --- a/fluid_build/_error_catalog.py +++ b/fluid_build/_error_catalog.py @@ -638,6 +638,82 @@ def slug_for(event: str) -> str: ["Pass the product id to roll back: fluid rollback "], None, ), + # ── fluid test results table / dashboard (Databricks) ────────────────── + "test_results_table_invalid": ( + [ + "Name the table as catalog.schema.table, e.g. --results-table ops.dq.fluid_test_results", + "Each part may hold letters, digits and underscores only", + ], + None, + ), + "test_results_table_in_product_schema": ( + [ + "Put the results table in a schema of its own: every product writes to it, " + + "and fluid apply owns each product's schema", + "Create one with CREATE SCHEMA .dq, then pass " + + "--results-table .dq.fluid_test_results", + ], + None, + ), + "test_results_table_mismatch": ( + [ + "The table exists without the fluid test results columns, so nothing was written", + "Point --results-table at a new table name; forge creates it with the right columns", + ], + None, + ), + "test_results_write_failed": ( + [ + "Check DATABRICKS_HOST, the token or OAuth client, and DATABRICKS_WAREHOUSE_ID", + "The schema must exist and this identity needs USE CATALOG, USE SCHEMA, " + + "CREATE TABLE and MODIFY on it", + "The test report above is unaffected: rerun fluid test once the write can succeed", + ], + None, + ), + "test_results_dashboard_needs_table": ( + [ + "--results-dashboard draws the --results-table table: pass both", + ], + None, + ), + "test_results_table_needs_native_engine": ( + [ + "--results-table records the native engine's per-rule results; drop --engine soda", + "Or keep --engine soda and its JUnit or JSON output, without --results-table", + ], + None, + ), + "test_results_table_needs_live_checks": ( + [ + "--results-table records the live checks of a run, and --no-data runs none", + "Drop --no-data to test and record, or drop --results-table", + ], + None, + ), + "test_results_dashboard_invalid": ( + [ + "Give a workspace path whose last part is the name, e.g. " + + "--results-dashboard '/Workspace/Shared/fluid/Data quality'", + "The path must start with / and have no empty, . or .. parts", + ], + None, + ), + "test_results_dashboard_path_taken": ( + [ + "Something other than a dashboard already sits at that workspace path", + "Choose another --results-dashboard path, or move that object away", + ], + None, + ), + "test_results_dashboard_failed": ( + [ + "The results were written; only the dashboard step failed", + "This identity needs CAN MANAGE on the dashboard (or write access to its folder) " + + "and CAN USE on DATABRICKS_WAREHOUSE_ID", + ], + None, + ), # ── apply safety gates ───────────────────────────────────────────────── "iceberg_catalog_move_blocked": ( [ diff --git a/fluid_build/agent_skills/skills/troubleshooting-fluid-errors/references/error-codes.md b/fluid_build/agent_skills/skills/troubleshooting-fluid-errors/references/error-codes.md index 1bff8355..219bf921 100644 --- a/fluid_build/agent_skills/skills/troubleshooting-fluid-errors/references/error-codes.md +++ b/fluid_build/agent_skills/skills/troubleshooting-fluid-errors/references/error-codes.md @@ -91,4 +91,14 @@ Docs pages are relative to https://agenticstiger.github.io/forge_docs/ | `ERR_SCHEDULE_SYNC_DAGS_DIR_NOT_DIRECTORY` | The --dags-dir value must be an existing directory. Create it (mkdir -p) or point --dags-dir at your Airflow dags/ folder. | advanced/production-troubleshooting.html | | `ERR_SCHEDULE_SYNC_UNHANDLED_SCHEME` | Use a supported destination scheme (file / scp / git+ssh) for --to. | advanced/production-troubleshooting.html | | `ERR_ROLLBACK_PRODUCT_ID_EMPTY` | Pass the product id to roll back: fluid rollback . | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_TABLE_INVALID` | Name the table as catalog.schema.table, e.g. --results-table ops.dq.fluid_test_results. Each part may hold letters, digits and underscores only. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_TABLE_IN_PRODUCT_SCHEMA` | Put the results table in a schema of its own: every product writes to it, and fluid apply owns each product's schema. Create one with CREATE SCHEMA .dq, then pass --results-table .dq.fluid_test_results. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_TABLE_MISMATCH` | The table exists without the fluid test results columns, so nothing was written. Point --results-table at a new table name; forge creates it with the right columns. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_WRITE_FAILED` | Check DATABRICKS_HOST, the token or OAuth client, and DATABRICKS_WAREHOUSE_ID. The schema must exist and this identity needs USE CATALOG, USE SCHEMA, CREATE TABLE and MODIFY on it. The test report above is unaffected: rerun fluid test once the write can succeed. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_DASHBOARD_NEEDS_TABLE` | --results-dashboard draws the --results-table table: pass both. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_TABLE_NEEDS_NATIVE_ENGINE` | --results-table records the native engine's per-rule results; drop --engine soda. Or keep --engine soda and its JUnit or JSON output, without --results-table. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_TABLE_NEEDS_LIVE_CHECKS` | --results-table records the live checks of a run, and --no-data runs none. Drop --no-data to test and record, or drop --results-table. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_DASHBOARD_INVALID` | Give a workspace path whose last part is the name, e.g. --results-dashboard '/Workspace/Shared/fluid/Data quality'. The path must start with / and have no empty, . or .. parts. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_DASHBOARD_PATH_TAKEN` | Something other than a dashboard already sits at that workspace path. Choose another --results-dashboard path, or move that object away. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_DASHBOARD_FAILED` | The results were written; only the dashboard step failed. This identity needs CAN MANAGE on the dashboard (or write access to its folder) and CAN USE on DATABRICKS_WAREHOUSE_ID. | advanced/production-troubleshooting.html | | `ERR_ICEBERG_CATALOG_MOVE_BLOCKED` | Run the printed `tofu state rm` commands: they release the resources from this contract's OpenTofu state and touch nothing in the cloud. Then re-run fluid apply. The released Glue database/table or Snowflake EXTERNAL VOLUME stays in place; delete it by hand only if nothing else uses it. | cli/apply.html#iceberg-catalog-move-guard | diff --git a/fluid_build/cli/_test_cases.py b/fluid_build/cli/_test_cases.py new file mode 100644 index 00000000..f1f78879 --- /dev/null +++ b/fluid_build/cli/_test_cases.py @@ -0,0 +1,75 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""The test cases one ``fluid test`` run reports, in one place. + +``--output junit`` and ``--results-table`` both report a run as the same +cases: one per executed data-quality rule, then one per structural check +category. Reading both from here keeps Jenkins' test list and the results +table from ever disagreeing about what ran. +""" + +from __future__ import annotations + +import re +from typing import Dict, List, Tuple + +#: The structural check categories, in report order. ``quality`` stands for +#: the data-quality rules only when no rule ran on its own (then there is no +#: per-rule case to carry them). +STRUCTURAL_CATEGORIES: Tuple[str, ...] = ( + "schema", + "connection", + "binding", + "missing_resource", + "missing_field", + "type_mismatch", + "mode_mismatch", + "extra_field", + "empty_table", + "row_count_below_threshold", + "quality", + "metadata", + "drift", +) + + +#: The path of an issue raised for one rule: ``exposes[0].contract.dq.rules.``. +_RULE_PATH = re.compile(r"\.rules\.[^.]+$") + + +def rule_checks(report) -> List: + """The data-quality rules that ran, one ``CheckOutcome`` each.""" + return [c for c in getattr(report, "checks", []) if c.category == "quality"] + + +def structural_categories(report) -> Tuple[List[str], List[str], Dict[str, List]]: + """``(built-in, extra, issues by category)`` for the structural cases. + + ``built-in`` is :data:`STRUCTURAL_CATEGORIES` (without ``quality`` when a + rule ran on its own); ``extra`` is every other category the run raised an + issue in, in first-seen order. Each case passes unless it has an + error-severity issue. + """ + by_category: Dict[str, List] = {} + for issue in report.issues: + by_category.setdefault(issue.category, []).append(issue) + builtin = list(STRUCTURAL_CATEGORIES) + if rule_checks(report): + # The per-rule cases already carry every rule outcome; a roll-up + # ``quality`` case on top would double-count them. A quality issue no + # rule case carries (an expose whose rules could not run at all) still + # needs a case, so the roll-up stays, with only those issues. + unattributed = [ + i for i in by_category.get("quality", []) if not _RULE_PATH.search(i.path or "") + ] + if unattributed: + by_category["quality"] = unattributed + else: + builtin.remove("quality") + by_category.pop("quality", None) + extra = [c for c in by_category if c not in builtin] + return builtin, extra, by_category + + +__all__ = ["STRUCTURAL_CATEGORIES", "rule_checks", "structural_categories"] diff --git a/fluid_build/cli/_test_dashboard_databricks.py b/fluid_build/cli/_test_dashboard_databricks.py new file mode 100644 index 00000000..2f697f63 --- /dev/null +++ b/fluid_build/cli/_test_dashboard_databricks.py @@ -0,0 +1,531 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``fluid test --results-dashboard``: an AI/BI (Lakeview) dashboard over the results table. + +The dashboard is found by its workspace path, ``/.lvdash.json`` +(``GET /api/2.0/workspace/get-status``, whose ``resource_id`` is the dashboard +id), so every run, on any machine, finds the same one and no state file is +kept. The lookup is the one databricks-solutions/ai-dev-kit uses; Databricks +Labs DQX records the id in its install state instead, which an ephemeral CI +agent would lose. + +- Missing: the folder is made (``workspace/mkdirs``), the dashboard created + (``POST /api/2.0/lakeview/dashboards``) and published. +- Present with this definition: left alone (published if it never was). +- Present with another definition (another table, or an older forge): replaced + with ``PATCH``, then published again. + +It is published without embedded credentials: each viewer reads the results +table as themself, so they need ``SELECT`` on it and access to the warehouse. + +The widget shapes (counter v2, bar and line v3, table v1, text box) follow the +``.lvdash.json`` files Databricks Labs DQX ships; nothing is copied from them. +""" + +from __future__ import annotations + +import json +import sys +from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple +from urllib.parse import quote + +from ._common import CLIError + +DASHBOARDS_PATH = "/api/2.0/lakeview/dashboards" +GET_STATUS_PATH = "/api/2.0/workspace/get-status" +MKDIRS_PATH = "/api/2.0/workspace/mkdirs" +_SUFFIX = ".lvdash.json" + +#: Rows that are fluid test's own structural checks, not data-quality rules. +_RULES_ONLY = "COALESCE(rule_type, '') <> 'structure'" + + +def _status(message: str) -> None: + sys.stderr.write(message + "\n") + + +def parse_dashboard_path(value: Any) -> Tuple[str, str]: + """``(folder, name)`` of ``--results-dashboard /Folder/Name``.""" + text = str(value or "").strip() + if text.endswith(_SUFFIX): + text = text[: -len(_SUFFIX)] + segments = text.split("/") + problems = ( + not text.startswith("/") + or len(segments) < 3 + or any(not s.strip() or s in (".", "..") for s in segments[1:]) + or any(ord(ch) < 32 or ch in "\\\x7f" for ch in text) + ) + if problems: + raise CLIError( + 2, + "test_results_dashboard_invalid", + { + "results_dashboard": str(value), + "expected": "a workspace path: /Workspace/Shared/fluid/Data quality " + "(the last part is the dashboard's name)", + }, + ) + return "/".join(segments[:-1]), segments[-1] + + +def _sql_lines(sql: str) -> List[str]: + lines = sql.strip().splitlines() + return [line + "\n" for line in lines[:-1]] + lines[-1:] + + +def _table_column(name: str, kind: str, title: str, order: int) -> Dict[str, Any]: + """One column of a table widget (spec version 1), with every key the editor + writes: Lakeview refuses a column without them ("Invalid widget definition").""" + column: Dict[str, Any] = { + "fieldName": name, + "booleanValues": ["false", "true"], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": True, + "type": kind, + "displayAs": kind, + "visible": True, + "order": order, + "title": title, + "allowSearch": kind == "string", + "alignContent": "left" if kind == "string" else "right", + "allowHTML": False, + "highlightLinks": False, + "useMonospaceFont": False, + "preserveWhitespace": False, + } + if kind == "datetime": + column["dateTimeFormat"] = "YYYY-MM-DD HH:mm:ss" + return column + + +def dashboard_definition(table: Tuple[str, str, str]) -> Dict[str, Any]: + """The ``serialized_dashboard`` for a results table (validated name parts).""" + ref = ".".join(f"`{p}`" for p in table) + # Each product's latest run is the latest one that tested rules (a run that + # tested none, say a product that declares none, counts only when no other + # did), so the rule widgets always show a product's last real result. + latest = ( + "WITH last_run AS (\n" + " SELECT product_id, COALESCE(\n" + f" MAX(CASE WHEN {_RULES_ONLY} THEN run_time END), MAX(run_time)\n" + " ) AS run_time\n" + f" FROM {ref} GROUP BY product_id\n" + "),\n" + "latest AS (\n" + f" SELECT t.* FROM {ref} AS t JOIN last_run AS l\n" + " ON t.product_id = l.product_id AND t.run_time = l.run_time\n" + ")\n" + ) + pass_pct = "ROUND(100 * AVG(CASE WHEN passed THEN 1.0 ELSE 0.0 END), 1)" + datasets = [ + { + "name": "fluid_dq_overall", + "displayName": "Latest run, all products", + "queryLines": _sql_lines( + f"{latest}SELECT {pass_pct} AS pass_rate_pct,\n" + " COUNT(DISTINCT product_id) AS products,\n" + " SUM(CASE WHEN passed THEN 0 ELSE 1 END) AS failing_rules\n" + f"FROM latest WHERE {_RULES_ONLY}" + ), + }, + { + "name": "fluid_dq_products", + "displayName": "Latest run per product", + "queryLines": _sql_lines( + f"{latest}SELECT product_id, COALESCE(layer, 'unset') AS layer,\n" + " COUNT(*) AS rules,\n" + " SUM(CASE WHEN passed THEN 0 ELSE 1 END) AS failing_rules,\n" + f" {pass_pct} AS pass_rate_pct,\n" + " MAX(run_time) AS last_run\n" + f"FROM latest WHERE {_RULES_ONLY}\n" + "GROUP BY product_id, COALESCE(layer, 'unset')" + ), + }, + { + "name": "fluid_dq_trend", + "displayName": "Daily pass rate per product", + "queryLines": _sql_lines( + "SELECT DATE(run_time) AS run_date, product_id,\n" + f" {pass_pct} AS pass_rate_pct\n" + f"FROM {ref} WHERE {_RULES_ONLY}\n" + "GROUP BY DATE(run_time), product_id" + ), + }, + { + "name": "fluid_dq_failing", + "displayName": "Failing checks, latest run", + "queryLines": _sql_lines( + f"{latest}SELECT product_id, layer, rule_id, rule_type, severity,\n" + " observed_value, threshold, message, run_time\n" + "FROM latest WHERE NOT passed\n" + "ORDER BY product_id, rule_id" + ), + }, + ] + + def counter(name: str, title: str, field: str, x: int) -> Dict[str, Any]: + agg = f"sum({field})" + return { + "widget": { + "name": name, + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "fluid_dq_overall", + "fields": [{"name": agg, "expression": f"SUM(`{field}`)"}], + "disaggregated": False, + }, + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": {"value": {"fieldName": agg, "displayName": title}}, + "frame": {"title": title, "showTitle": True}, + }, + }, + "position": {"x": x, "y": 1, "width": 2, "height": 2}, + } + + def fields(*names: str) -> List[Dict[str, str]]: + return [{"name": n, "expression": f"`{n}`"} for n in names] + + columns = ( + ("product_id", "string", "Product"), + ("layer", "string", "Layer"), + ("rule_id", "string", "Rule"), + ("rule_type", "string", "Type"), + ("severity", "string", "Severity"), + ("observed_value", "string", "Observed"), + ("threshold", "string", "Threshold"), + ("message", "string", "Message"), + ("run_time", "datetime", "Run time"), + ) + layout = [ + { + "widget": { + "name": "fluid_dq_title", + "multilineTextboxSpec": { + "lines": [ + "## Data quality by product\n", + f"Every `fluid test --results-table {'.'.join(table)}` run: each " + "product's data-quality rules, passed or failed, for Bronze products " + "and dbt-built ones alike.", + ] + }, + }, + "position": {"x": 0, "y": 0, "width": 6, "height": 1}, + }, + counter("fluid_dq_pass_rate", "Rules passed, latest runs (%)", "pass_rate_pct", 0), + counter("fluid_dq_product_count", "Products tested", "products", 2), + counter("fluid_dq_failing_count", "Failing rules, latest runs", "failing_rules", 4), + { + "widget": { + "name": "fluid_dq_pass_rate_by_product", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "fluid_dq_products", + "fields": fields("product_id", "layer", "pass_rate_pct"), + "disaggregated": True, + }, + } + ], + "spec": { + "version": 3, + "widgetType": "bar", + "encodings": { + "x": { + "fieldName": "product_id", + "displayName": "Product", + "scale": {"type": "categorical"}, + }, + "y": { + "fieldName": "pass_rate_pct", + "displayName": "Rules passed (%)", + "scale": {"type": "quantitative"}, + }, + "color": { + "fieldName": "layer", + "displayName": "Layer", + "scale": {"type": "categorical"}, + }, + "label": {"show": True}, + }, + "frame": {"title": "Pass rate per product, latest run", "showTitle": True}, + }, + }, + "position": {"x": 0, "y": 3, "width": 6, "height": 6}, + }, + { + "widget": { + "name": "fluid_dq_trend", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "fluid_dq_trend", + "fields": fields("run_date", "product_id", "pass_rate_pct"), + "disaggregated": True, + }, + } + ], + "spec": { + "version": 3, + "widgetType": "line", + "encodings": { + "x": { + "fieldName": "run_date", + "displayName": "Day", + "scale": {"type": "temporal"}, + }, + "y": { + "fieldName": "pass_rate_pct", + "displayName": "Rules passed (%)", + "scale": {"type": "quantitative"}, + }, + "color": { + "fieldName": "product_id", + "displayName": "Product", + "scale": {"type": "categorical"}, + }, + }, + "frame": {"title": "Pass rate over time", "showTitle": True}, + }, + }, + "position": {"x": 0, "y": 9, "width": 6, "height": 6}, + }, + { + "widget": { + "name": "fluid_dq_failing", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "fluid_dq_failing", + "fields": fields(*(c[0] for c in columns)), + "disaggregated": True, + }, + } + ], + "spec": { + "version": 1, + "widgetType": "table", + "encodings": { + "columns": [ + _table_column(name, kind, title, 100000 + i) + for i, (name, kind, title) in enumerate(columns) + ] + }, + "invisibleColumns": [], + "allowHTMLByDefault": False, + "itemsPerPage": 25, + "paginationSize": "default", + "condensed": True, + "withRowNumber": False, + "frame": {"title": "Failing checks, latest run", "showTitle": True}, + }, + }, + "position": {"x": 0, "y": 15, "width": 6, "height": 8}, + }, + ] + return { + "datasets": datasets, + "pages": [ + { + "name": "fluid_dq", + "displayName": "Data quality", + "pageType": "PAGE_TYPE_CANVAS", + "layout": layout, + } + ], + } + + +def _status_code_is(exc: BaseException, *codes: str) -> bool: + text = f"{getattr(exc, 'error_code', '')} {exc}".upper() + return any(code in text for code in codes) + + +def _is_not_found(exc: BaseException) -> bool: + for attr in ("status_code", "status", "http_status"): + if getattr(exc, attr, None) == 404: + return True + return type(exc).__name__ in {"NotFound", "ResourceDoesNotExist"} or _status_code_is( + exc, "RESOURCE_DOES_NOT_EXIST", "NOT_FOUND" + ) + + +def _later(first: Any, second: Any) -> bool: + """Whether timestamp ``first`` is after ``second`` (``False`` when either is unreadable).""" + import re + from datetime import datetime + + def parse(value: Any) -> datetime: + text = str(value).replace("Z", "+00:00") + # Python 3.10's fromisoformat takes only 3 or 6 fraction digits; the API + # may send 1 to 9. Pad or cut the fraction to 6. + text = re.sub(r"\.(\d+)", lambda m: "." + (m.group(1) + "000000")[:6], text, count=1) + return datetime.fromisoformat(text) + + try: + a, b = parse(first), parse(second) + except (TypeError, ValueError): + return False + try: + return a > b + except TypeError: # one naive, one aware + return False + + +def _same(theirs: Any, ours: Mapping[str, Any]) -> bool: + """Whether a stored ``serialized_dashboard`` holds our datasets and pages.""" + try: + stored = json.loads(theirs) if isinstance(theirs, str) else theirs + except ValueError: + return False + if not isinstance(stored, dict): + return False + return all( + json.dumps(stored.get(key), sort_keys=True) == json.dumps(ours.get(key), sort_keys=True) + for key in ("datasets", "pages") + ) + + +class _Lakeview: + """The few REST calls this needs, through the SDK's ``api_client.do``.""" + + def __init__(self, client: Any) -> None: + self._client = client + + def do( + self, + method: str, + path: str, + *, + query: Optional[Mapping[str, Any]] = None, + body: Optional[Mapping[str, Any]] = None, + ) -> Dict[str, Any]: + out = self._client.api_client.do( + method, path, query=dict(query) if query else None, body=dict(body) if body else None + ) + return out if isinstance(out, dict) else {} + + def find(self, folder: str, name: str) -> Optional[str]: + path = f"{folder}/{name}{_SUFFIX}" + try: + found = self.do("GET", GET_STATUS_PATH, query={"path": path}) + except Exception as exc: # noqa: BLE001 + if _is_not_found(exc): + return None + raise + if str(found.get("object_type") or "").upper() != "DASHBOARD": + raise CLIError( + 1, + "test_results_dashboard_path_taken", + {"results_dashboard": f"{folder}/{name}", "found": found.get("object_type")}, + ) + return str(found.get("resource_id") or "") or None + + +def ensure_dashboard( + dashboard_path: str, + table: Tuple[str, str, str], + *, + client_factory: Optional[Callable[[], Any]] = None, + warehouse_id: Optional[str] = None, +) -> Dict[str, Any]: + """Create, update or leave the dashboard; return what happened. + + Raises :class:`CLIError` with what to do when the dashboard could not be + made or published. + """ + from fluid_build.providers.databricks.statements import ( + default_client, + scrub, + warehouse_id_from_env, + ) + + folder, name = parse_dashboard_path(dashboard_path) + definition = dashboard_definition(table) + serialized = json.dumps(definition, separators=(",", ":")) + try: + client = (client_factory or default_client)() + warehouse = warehouse_id or warehouse_id_from_env() + api = _Lakeview(client) + dashboard_id = api.find(folder, name) + action = "unchanged" + if dashboard_id is None: + api.do("POST", MKDIRS_PATH, body={"path": folder}) + body = { + "display_name": name, + "parent_path": folder, + "serialized_dashboard": serialized, + "warehouse_id": warehouse, + } + try: + dashboard_id = str(api.do("POST", DASHBOARDS_PATH, body=body)["dashboard_id"]) + action = "created" + except Exception as exc: # noqa: BLE001 + # Another run created it between the lookup and here. + if not _status_code_is(exc, "ALREADY_EXISTS", "ALREADY EXISTS"): + raise + dashboard_id = api.find(folder, name) + if dashboard_id is None: + raise + if action != "created": + current = api.do("GET", f"{DASHBOARDS_PATH}/{quote(dashboard_id, safe='')}") + if not _same(current.get("serialized_dashboard"), definition) or ( + current.get("warehouse_id") != warehouse + ): + patch = {"serialized_dashboard": serialized, "warehouse_id": warehouse} + if current.get("etag"): + patch["etag"] = current["etag"] + api.do("PATCH", f"{DASHBOARDS_PATH}/{quote(dashboard_id, safe='')}", body=patch) + action = "updated" + published_path = f"{DASHBOARDS_PATH}/{quote(dashboard_id, safe='')}/published" + publish = action != "unchanged" + if not publish: + # Unchanged here, but a run whose PATCH landed and whose publish failed + # left the published revision behind the draft: publish it then. + try: + published = api.do("GET", published_path) + except Exception as exc: # noqa: BLE001 + if not _is_not_found(exc): + raise + publish = True + else: + publish = _later(current.get("update_time"), published.get("revision_create_time")) + if publish: + api.do( + "POST", published_path, body={"embed_credentials": False, "warehouse_id": warehouse} + ) + except CLIError: + raise + except Exception as exc: # noqa: BLE001 - every failure becomes one safe message + context = { + "results_dashboard": f"{folder}/{name}", + "error": f"{type(exc).__name__}: {scrub(exc)}", + } + if isinstance(exc, ImportError): + from fluid_build.providers.databricks.statements import DATABRICKS_EXTRA_INSTALL + + context["fix"] = f"Install the Databricks extra: {DATABRICKS_EXTRA_INSTALL}" + raise CLIError(1, "test_results_dashboard_failed", context) from None + host = str(getattr(getattr(client, "config", None), "host", "") or "").rstrip("/") + url = f"{host}/dashboardsv3/{dashboard_id}/published" if host else None + _status(f"Dashboard {action}: {folder}/{name}" + (f" ({url})" if url else "")) + return {"action": action, "dashboard_id": dashboard_id, "url": url} + + +__all__ = ["dashboard_definition", "ensure_dashboard", "parse_dashboard_path"] diff --git a/fluid_build/cli/_test_results_databricks.py b/fluid_build/cli/_test_results_databricks.py new file mode 100644 index 00000000..096a1708 --- /dev/null +++ b/fluid_build/cli/_test_results_databricks.py @@ -0,0 +1,370 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``fluid test --results-table``: append one run's results to a Unity Catalog table. + +One row per test case of the run, the same cases ``--output junit`` reports +(``_test_cases.py``): each data-quality rule that ran, then each structural +check category (``rule_type = 'structure'``). The table is shared by every +product, so it lives in a schema the operator names, never in a product's own +schema (``fluid apply`` owns those). + +The shape follows the one-row-per-check-per-run results tables of Databricks +Labs DQX (its summary-metrics table: run id, run time, metric, value) and of +Elementary's ``elementary_test_results``; the pattern is adapted, no code is +taken (DQX is under the Databricks License). + +What is created: the table, with ``CREATE TABLE IF NOT EXISTS``, owned by the +identity that first runs this. The catalog and the schema are the operator's +and are never created. An existing table must have every column below, or +nothing is written: a name that points at some other table writes nothing +into it. + +Every value is bound as a ``:name`` parameter; the three name parts are +validated identifiers, backtick-quoted. +""" + +from __future__ import annotations + +import json +import sys +import uuid +from datetime import datetime, timezone +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple +from urllib.parse import quote + +from fluid_build.providers._sql_safety import validate_ident + +from ._common import CLIError +from ._test_cases import rule_checks, structural_categories + +#: ``(column, type, comment)`` in table order. +RESULT_COLUMNS: Tuple[Tuple[str, str, str], ...] = ( + ("run_id", "STRING NOT NULL", "One fluid test invocation"), + ("run_time", "TIMESTAMP NOT NULL", "When the checks ran (UTC)"), + ("product_id", "STRING NOT NULL", "The contract id"), + ("product_version", "STRING", "The contract version"), + ("layer", "STRING", "metadata.layer: Bronze, Silver or Gold"), + ("expose_id", "STRING", "The expose the check ran against"), + ("table_name", "STRING", "catalog.schema.table of that expose"), + ("rule_id", "STRING NOT NULL", "dq.rules[].id, or the name of the structural check"), + ("rule_type", "STRING", "dq.rules[].type, or structure for the checks of fluid test itself"), + ("severity", "STRING", "The declared severity of the rule"), + ("passed", "BOOLEAN NOT NULL", "Whether the check passed"), + ("observed_value", "STRING", "What was measured"), + ("threshold", "STRING", "What it had to be"), + ("message", "STRING", "What happened"), + ("fluid_version", "STRING", "The data-product-forge version that ran the checks"), +) + +#: ``rule_type`` of the rows for ``fluid test``'s own structural checks. +STRUCTURE_RULE_TYPE = "structure" + +_CASTS = {"run_time": "TIMESTAMP", "passed": "BOOLEAN"} +_TABLE_COMMENT = "fluid test results: one row per check per run (data-product-forge)" +_MAX_TEXT = 2000 +TABLES_PATH = "/api/2.1/unity-catalog/tables" + + +def _status(message: str) -> None: + """One status line on stderr, so ``--output json`` keeps stdout clean.""" + sys.stderr.write(message + "\n") + + +def parse_results_table(value: Any) -> Tuple[str, str, str]: + """``catalog.schema.table`` of ``--results-table``, each part validated.""" + parts = str(value or "").split(".") + try: + if len(parts) != 3: + raise ValueError(value) + return validate_ident(parts[0]), validate_ident(parts[1]), validate_ident(parts[2]) + except ValueError: + raise CLIError( + 2, + "test_results_table_invalid", + { + "results_table": str(value), + "expected": "catalog.schema.table, each part letters, digits and underscores", + }, + ) from None + + +def _expose_tables(contract: Mapping[str, Any]) -> Dict[str, Tuple[str, str, str]]: + """``expose id -> (catalog, schema, table)`` for each expose bound to a table.""" + out: Dict[str, Tuple[str, str, str]] = {} + exposes = contract.get("exposes") + # A malformed ``exposes`` (not a list) is the validator's to report. + for idx, expose in enumerate(exposes if isinstance(exposes, list) else []): + if not isinstance(expose, dict): + continue + binding = expose.get("binding") if isinstance(expose.get("binding"), dict) else {} + location = binding.get("location") if isinstance(binding.get("location"), dict) else {} + catalog = location.get("database") or location.get("catalog") + schema, table = location.get("schema"), location.get("table") + if catalog and schema and table: + key = str(expose.get("id") or expose.get("exposeId") or f"expose_{idx}") + out[key] = (str(catalog), str(schema), str(table)) + return out + + +def _expose_versions(contract: Mapping[str, Any]) -> Dict[str, str]: + """``expose id -> exposes[].version`` (the version ``fluid apply`` tags the table with).""" + out: Dict[str, str] = {} + for idx, expose in enumerate(contract.get("exposes") or []): + if isinstance(expose, dict) and expose.get("version") is not None: + key = str(expose.get("id") or expose.get("exposeId") or f"expose_{idx}") + out[key] = str(expose["version"]) + return out + + +def refuse_product_schema(table: Tuple[str, str, str], contract: Mapping[str, Any]) -> None: + """A results table in one of the tested product's own schemas is refused.""" + for catalog, schema, _ in _expose_tables(contract).values(): + if (catalog.lower(), schema.lower()) == (table[0].lower(), table[1].lower()): + raise CLIError( + 2, + "test_results_table_in_product_schema", + { + "results_table": ".".join(table), + "product_schema": f"{catalog}.{schema}", + "why": "the results table is shared by every product, and fluid apply " + "owns this product's schema", + "fix": f"name a schema of its own, e.g. {table[0]}.dq.{table[2]}", + }, + ) + + +def _text(value: Any) -> Optional[str]: + if value is None: + return None + text = str(value) + return text if len(text) <= _MAX_TEXT else text[: _MAX_TEXT - 3] + "..." + + +def _severity(value: Any) -> Optional[str]: + text = str(value or "").strip().lower() + return text or None + + +def results_rows( + report: Any, + contract: Mapping[str, Any], + *, + run_id: str, + fluid_version: Optional[str], +) -> List[Dict[str, Any]]: + """The rows of one run: each rule that ran, then each structural check.""" + tables = _expose_tables(contract) + versions = _expose_versions(contract) + # The contract's own version when it has one; the report's "0.0.0" is a + # placeholder for "none declared", never a version. + product_version = contract.get("version") + product_version = None if product_version is None else str(product_version) + metadata = contract.get("metadata") if isinstance(contract.get("metadata"), dict) else {} + run_time = report.validation_time.astimezone(timezone.utc) + only_table = next(iter(tables.items())) if len(tables) == 1 else None + common = { + "run_id": run_id, + "run_time": run_time, + "product_id": report.contract_id, + "layer": _text(metadata.get("layer")), + "fluid_version": fluid_version, + } + rows: List[Dict[str, Any]] = [] + for check in rule_checks(report): + expose_id = check.expose_id or (check.name.rsplit(".", 1)[0] if "." in check.name else None) + table = tables.get(expose_id or "") + rows.append( + { + **common, + "product_version": _text(versions.get(expose_id or "", product_version)), + "expose_id": expose_id, + "table_name": ".".join(table) if table else None, + "rule_id": check.rule_id or check.name.rsplit(".", 1)[-1], + "rule_type": check.rule_type, + "severity": _severity(check.severity), + "passed": bool(check.passed), + "observed_value": _text(check.actual), + "threshold": _text(check.expected), + "message": _text(check.message), + } + ) + builtin, extra, by_category = structural_categories(report) + from fluid_build.severity import is_error, is_warning + + for category in builtin + extra: + issues = by_category.get(category, []) + errors = [i for i in issues if is_error(i.severity)] + warnings = [i for i in issues if is_warning(i.severity)] + shown = errors + warnings + single = shown[0] if len(shown) == 1 else None + if single is not None: + observed, threshold = single.actual, single.expected + elif shown: + observed, threshold = f"{len(errors)} error(s), {len(warnings)} warning(s)", None + else: + observed, threshold = None, None + only = only_table[0] if only_table else None + rows.append( + { + **common, + "product_version": _text(versions.get(only or "", product_version)), + "expose_id": only, + "table_name": ".".join(only_table[1]) if only_table else None, + "rule_id": category, + "rule_type": STRUCTURE_RULE_TYPE, + "severity": "error" if errors else ("warning" if warnings else None), + "passed": not errors, + "observed_value": _text(observed), + "threshold": _text(threshold), + "message": _text("; ".join(i.message for i in shown) or "OK"), + } + ) + return rows + + +def _create_table_sql(ref: str) -> str: + from fluid_build.providers._sql_safety import quote_string_literal_for + + columns = ",\n ".join( + f"{name} {kind} COMMENT {quote_string_literal_for('databricks', note)}" + for name, kind, note in RESULT_COLUMNS + ) + comment = quote_string_literal_for("databricks", _TABLE_COMMENT) + return f"CREATE TABLE IF NOT EXISTS {ref} (\n {columns}\n) USING DELTA\nCOMMENT {comment}" + + +def _value(name: str, value: Any) -> Any: + if value is None: + return None + if name == "passed": + return "true" if value else "false" + if name == "run_time" and isinstance(value, datetime): + return value.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + return value + + +def insert_statement(ref: str, rows: Sequence[Mapping[str, Any]]) -> Tuple[str, List]: + """One ``INSERT`` for every row of the run, and its one parameter. + + The rows travel as a single JSON array bound to ``:rows`` and are unpacked in + SQL (``from_json`` + ``explode``), so the run is appended in one statement: + all of it or none of it, whatever its size. A run split over several + statements could be left half-written, and the dashboard would take that + half for the latest run. + """ + from fluid_build.providers.databricks.statements import param + + names = [name for name, _, _ in RESULT_COLUMNS] + struct = ", ".join(f"{name}: STRING" for name in names) + select = ", ".join( + f"CAST(r.{name} AS {_CASTS[name]})" if name in _CASTS else f"r.{name}" for name in names + ) + statement = ( + f"INSERT INTO {ref} ({', '.join(names)})\n" + f"SELECT {select}\n" + f"FROM (SELECT explode(from_json(:rows, 'ARRAY>')) AS r)" + ) + payload = [{name: _value(name, row.get(name)) for name in names} for row in rows] + return statement, [param("rows", json.dumps(payload))] + + +def _fix_for(error: str, table: Tuple[str, str, str]) -> Optional[str]: + """The one action that fixes a failed write, when the error says which.""" + catalog, schema, _ = table + upper = error.upper() + if "NO MODULE NAMED 'DATABRICKS" in upper: + from fluid_build.providers.databricks.statements import DATABRICKS_EXTRA_INSTALL + + return f"Install the Databricks extra: {DATABRICKS_EXTRA_INSTALL}" + if "SCHEMA_NOT_FOUND" in upper or "NO_SUCH_SCHEMA" in upper: + return f"Create the schema first (forge never creates it): CREATE SCHEMA `{catalog}`.`{schema}`" + if "CATALOG_NOT_FOUND" in upper or "NO_SUCH_CATALOG" in upper: + return f"Catalog `{catalog}` does not exist; name an existing catalog" + if "PERMISSION" in upper or "INSUFFICIENT_PRIVILEGES" in upper or "NOT AUTHORIZED" in upper: + return ( + f"Grant this identity USE CATALOG on `{catalog}`, USE SCHEMA and CREATE TABLE on " + f"`{catalog}`.`{schema}`, and MODIFY on the table once it exists" + ) + return None + + +def _check_columns(client: Any, full_name: str) -> None: + """Refuse a table that lacks one of :data:`RESULT_COLUMNS`.""" + table = client.api_client.do("GET", f"{TABLES_PATH}/{quote(full_name, safe='')}") + have = { + str(c.get("name", "")).lower() + for c in (table.get("columns") or [] if isinstance(table, dict) else []) + if isinstance(c, dict) + } + missing = [name for name, _, _ in RESULT_COLUMNS if name not in have] + if missing: + raise CLIError( + 1, + "test_results_table_mismatch", + { + "results_table": full_name, + "missing_columns": ", ".join(missing), + "why": "the table exists and is not a fluid test results table, so nothing " + "was written to it", + }, + ) + + +def write_results( + report: Any, + contract: Mapping[str, Any], + results_table: str, + *, + client_factory: Optional[Callable[[], Any]] = None, + warehouse_id: Optional[str] = None, + run_id: Optional[str] = None, +) -> Tuple[str, int]: + """Append the run's rows; return ``(table, rows written)``. + + Raises :class:`CLIError` with what to do when nothing could be written. + """ + from fluid_build import __version__ + from fluid_build.providers.databricks.statements import ( + StatementRunner, + default_client, + scrub, + warehouse_id_from_env, + ) + + table = parse_results_table(results_table) + refuse_product_schema(table, contract) + full_name = ".".join(table) + ref = ".".join(f"`{p}`" for p in table) + rows = results_rows( + report, contract, run_id=run_id or uuid.uuid4().hex, fluid_version=__version__ + ) + try: + client = (client_factory or default_client)() + runner = StatementRunner(client, warehouse_id or warehouse_id_from_env()) + runner.run(_create_table_sql(ref)) + _check_columns(client, full_name) + statement, params = insert_statement(ref, rows) + runner.run(statement, params) + except CLIError: + raise + except Exception as exc: # noqa: BLE001 - every failure becomes one safe message + message = f"{type(exc).__name__}: {scrub(exc)}" + context: Dict[str, Any] = {"results_table": full_name, "error": message} + fix = _fix_for(message, table) + if fix: + context["fix"] = fix + raise CLIError(1, "test_results_write_failed", context) from None + _status(f"Results: {len(rows)} row(s) appended to {full_name}") + return full_name, len(rows) + + +__all__ = [ + "RESULT_COLUMNS", + "STRUCTURE_RULE_TYPE", + "insert_statement", + "parse_results_table", + "refuse_product_schema", + "results_rows", + "write_results", +] diff --git a/fluid_build/cli/contract_validation.py b/fluid_build/cli/contract_validation.py index b7053517..57dd4b90 100644 --- a/fluid_build/cli/contract_validation.py +++ b/fluid_build/cli/contract_validation.py @@ -87,17 +87,10 @@ def _dist_installed(dist_name: str) -> bool: SNOWFLAKE_VALIDATION_AVAILABLE = _dist_installed("snowflake-connector-python") AWS_VALIDATION_AVAILABLE = _dist_installed("boto3") LOCAL_VALIDATION_AVAILABLE = _dist_installed("duckdb") +DATABRICKS_VALIDATION_AVAILABLE = _dist_installed("databricks-sdk") #: Known platforms with no live validation provider: ``(warning, suggestion)``. -#: Databricks has no Unity Catalog introspection on this path yet (no SDK); its -#: offline checks are `fluid validate`'s, and `fluid apply --mode dry-run` is a -#: real `tofu plan` against the workspace. _NO_LIVE_VALIDATION = { - "databricks": ( - "Live validation against Unity Catalog is not available for provider 'databricks' yet", - "Run 'fluid validate' for the offline binding checks and " - "'fluid apply --mode dry-run' to plan against the workspace.", - ), # `fluid validate` accepts `azure` (the schema enum lists it) although there is # no azure provider: say so as a warning, so the two commands agree. "azure": ( @@ -110,6 +103,12 @@ def _dist_installed(dist_name: str) -> bool: COMMAND = "contract-validation" +def _rule_kind(rule: Any) -> str: + """A ``dq.rules[]`` entry's ``type``, lower-cased ("" when it has none).""" + kind = rule.get("type") if isinstance(rule, dict) else None + return kind.strip().lower() if isinstance(kind, str) else "" + + @dataclass class ValidationIssue: """Represents a validation issue found during contract validation.""" @@ -157,6 +156,11 @@ class CheckOutcome: message: str = "" expected: Optional[Any] = None actual: Optional[Any] = None + #: The data-quality rule this outcome is for (``dq.rules[].id`` / ``.type``) + #: and the expose it ran against, so a results table can key on them. + rule_id: Optional[str] = None + rule_type: Optional[str] = None + expose_id: Optional[str] = None @dataclass @@ -573,6 +577,22 @@ def _detect_and_validate_provider(self) -> None: from ..providers.local_validation import LocalValidationProvider self.validation_provider = LocalValidationProvider({"base_dir": base_dir}) + elif self.provider_name == "databricks": + if not DATABRICKS_VALIDATION_AVAILABLE: + self.report.add_issue( + "error", + "binding", + "databricks-sdk package is required for Databricks validation", + "binding.platform", + suggestion="Install it with: pip install 'data-product-forge[databricks]'", + ) + return + from ..providers.databricks_validation import DatabricksValidationProvider + + # Host, credentials and the SQL warehouse come from the + # environment (DATABRICKS_HOST, DATABRICKS_WAREHOUSE_ID), as for + # `fluid apply`; the contract names neither. + self.validation_provider = DatabricksValidationProvider({}) elif self.provider_name in _NO_LIVE_VALIDATION: message, suggestion = _NO_LIVE_VALIDATION[self.provider_name] self.report.add_issue( @@ -734,12 +754,23 @@ def _validate_single_expose(self, expose: Dict[str, Any], idx: int) -> None: # Also check nested contract.dq.rules pattern if not dq_rules: dq_rules = expose.get("contract", {}).get("dq", {}).get("rules", []) + # A ``schema`` rule ("the table conforms to the contract schema") is + # decided by the live column comparison below, not by a query. + schema_rules: List[Dict[str, Any]] = [] + if isinstance(dq_rules, list): + schema_rules = [r for r in dq_rules if _rule_kind(r) == "schema"] + dq_rules = [r for r in dq_rules if _rule_kind(r) != "schema"] if dq_rules and self.validation_provider and self.check_data: self._run_expose_quality_checks(expose, dq_rules, path_prefix) # Validate against actual resource (if provider available) if self.check_data and self.provider_name: - self._validate_against_actual_resource(expose, path_prefix) + first = len(self.report.issues) + compared = self._validate_against_actual_resource(expose, path_prefix) + if schema_rules and self.validation_provider: + self._decide_schema_rules( + expose_id, schema_rules, self.report.issues[first:], compared, path_prefix + ) def _validate_binding(self, binding: Dict[str, Any], path: str, expose_id: str) -> None: """Validate binding configuration.""" @@ -799,8 +830,22 @@ def _run_expose_quality_checks( """Execute data-quality rules against the live resource via the provider.""" expose_id = expose.get("id", "unknown") LOG.debug("Running %d quality checks for %s", len(rules), expose_id) + # A provider that hands back every rule's result (passes included) + # lets each outcome carry what was measured, not only the failures. + # ``ValidationProvider.run_quality_results`` answers ``None`` for a + # provider that only reports failures. + run_results = getattr(self.validation_provider, "run_quality_results", None) + results = None try: - issues = self.validation_provider.run_quality_checks(expose, rules) + if callable(run_results): + results = run_results(expose, rules) + if isinstance(results, list): + from fluid_build.providers.quality_engine import quality_results_to_issues + + issues = quality_results_to_issues(results) + else: + results = None + issues = self.validation_provider.run_quality_checks(expose, rules) except Exception as exc: LOG.warning("Quality check execution failed for %s: %s", expose_id, exc) self.report.add_issue( @@ -830,13 +875,82 @@ def _run_expose_quality_checks( documentation_url=issue.documentation_url, ) - self._record_quality_outcomes(expose_id, rules, issues) + self._record_quality_outcomes(expose_id, rules, issues, results) + + def _decide_schema_rules( + self, + expose_id: str, + rules: List[Dict[str, Any]], + resource_issues: List[ValidationIssue], + compared: bool, + path_prefix: str, + ) -> None: + """Decide each ``schema`` rule from the live column comparison. + + It passes when every contract column is in the table with its type + (the comparison's errors are a missing column or a changed type; an + extra column or a nullability difference does not break the rule). A + comparison that could not run is a failed rule, not a pass. + """ + missing_table = any(i.category == "missing_resource" for i in resource_issues) + broken = [ + i + for i in resource_issues + if i.category in ("missing_field", "type_mismatch") and _is_error_severity(i.severity) + ] + expected = "every contract column, with its type" + for rule in rules: + if not isinstance(rule, dict) or not rule.get("id"): + continue + rule_id = str(rule["id"]) + severity = rule.get("severity", "error") + if missing_table: + passed, actual = False, "table does not exist" + message = f"Rule '{rule_id}': the table does not exist, so no column could conform" + elif not compared: + passed, actual = False, "not executed" + message = ( + f"Rule '{rule_id}' is a schema rule, but the live table's columns could " + "not be read, so it was not checked" + ) + elif broken: + gone = sum(1 for i in broken if i.category == "missing_field") + actual = f"{gone} column(s) missing, {len(broken) - gone} type change(s)" + passed = False + message = f"Rule '{rule_id}': the table does not conform to the contract ({actual})" + else: + passed, actual = True, "all contract columns match" + message = "Schema conforms to the contract" + if not passed: + self.report.add_issue( + severity, + "quality", + message, + f"{path_prefix}.contract.dq.rules.{rule_id}", + expected=expected, + actual=actual, + ) + self.report.record_check( + CheckOutcome( + name=f"{expose_id}.{rule_id}", + category="quality", + passed=passed, + severity=severity, + message=message, + expected=expected, + actual=actual, + rule_id=rule_id, + rule_type="schema", + expose_id=expose_id, + ) + ) def _record_quality_outcomes( self, expose_id: str, rules: List[Dict[str, Any]], issues: List[Any], + results: Optional[List[Any]] = None, ) -> None: """Record one :class:`CheckOutcome` per DQ rule that actually ran. @@ -855,6 +969,10 @@ def _record_quality_outcomes( if not declared: return + if results is not None: + self._record_result_outcomes(expose_id, rules, results) + return + by_rule: Dict[str, Any] = {} for issue in issues: rule_id = (getattr(issue, "path", "") or "").rsplit(".", 1)[-1] @@ -881,6 +999,9 @@ def _record_quality_outcomes( f"{rule.get('type', 'rule')} check passed for " f"'{rule.get('selector', '')}'" ), + rule_id=str(rule_id), + rule_type=_rule_kind(rule) or None, + expose_id=expose_id, ) else: outcome = CheckOutcome( @@ -891,9 +1012,45 @@ def _record_quality_outcomes( message=issue.message, expected=issue.expected, actual=issue.actual, + rule_id=str(rule_id), + rule_type=_rule_kind(rule) or None, + expose_id=expose_id, ) self.report.record_check(outcome) + def _record_result_outcomes( + self, expose_id: str, rules: List[Dict[str, Any]], results: List[Any] + ) -> None: + """One :class:`CheckOutcome` per rule from the provider's full results, so a + passing rule carries what was measured too.""" + # Results come back in rule order; a queue per id pairs the n-th rule + # with an id with the n-th result for it, so two rules sharing an id + # cannot hide one's failure behind the other's pass. + queues: Dict[Any, List[Any]] = {} + for result in results: + queues.setdefault(getattr(result, "rule_id", None), []).append(result) + for rule in rules: + if not isinstance(rule, dict) or not rule.get("id"): + continue + queue = queues.get(rule["id"]) + if not queue: + continue + result = queue.pop(0) + self.report.record_check( + CheckOutcome( + name=f"{expose_id}.{rule['id']}", + category="quality", + passed=bool(result.passed), + severity=result.severity or rule.get("severity", "error"), + message=result.message, + expected=result.expected, + actual=result.actual, + rule_id=str(rule["id"]), + rule_type=result.rule_type or _rule_kind(rule) or None, + expose_id=expose_id, + ) + ) + def _validate_schema_definition( self, schema: List[Dict[str, Any]], path: str, expose_id: str ) -> None: @@ -952,8 +1109,12 @@ def _validate_schema_definition( f"{col_path}.type", ) - def _validate_against_actual_resource(self, expose: Dict[str, Any], path: str) -> None: - """Validate contract schema against actual deployed resource.""" + def _validate_against_actual_resource(self, expose: Dict[str, Any], path: str) -> bool: + """Validate contract schema against actual deployed resource. + + Returns whether the live columns were compared with the contract (a + ``schema`` data-quality rule is decided by that comparison). + """ expose_id = expose.get("id") or expose.get("exposeId") or "unknown" LOG.debug("Validating actual resource for %s", expose_id) @@ -963,18 +1124,22 @@ def _validate_against_actual_resource(self, expose: Dict[str, Any], path: str) - binding = expose.get("binding", {}) location = binding.get("location", {}) props = location.get("properties", {}) - self._validate_bigquery_resource(expose, path, props) + return self._validate_bigquery_resource(expose, path, props) elif self.provider_name == "aws" and self._iceberg_outside_glue(expose, path): - return - elif self.provider_name in ("snowflake", "aws", "local"): - self._validate_generic_resource(expose, path) + return False + elif self.provider_name in ("snowflake", "aws", "local", "databricks"): + return self._validate_generic_resource(expose, path) else: LOG.debug("Provider '%s' resource validation not yet implemented", self.provider_name) + return False def _validate_bigquery_resource( self, expose: Dict[str, Any], path: str, props: Dict[str, Any] - ) -> None: - """Validate against actual BigQuery table using provider abstraction.""" + ) -> bool: + """Validate against actual BigQuery table using provider abstraction. + + Returns whether the live columns were compared with the contract. + """ if not self.validation_provider: self.report.add_issue( "warning", @@ -982,7 +1147,7 @@ def _validate_bigquery_resource( "No validation provider initialized, skipping resource validation", path, ) - return + return False project = props.get("project", self.project) dataset = props.get("dataset") @@ -1005,7 +1170,7 @@ def _validate_bigquery_resource( suggestion=f"Add the following properties to binding.location.properties: {', '.join(missing)}", documentation_url="https://agenticstiger.github.io/forge_docs/providers/gcp.html", ) - return + return False table_fqn = f"{project}.{dataset}.{table}" @@ -1091,6 +1256,7 @@ def _validate_bigquery_resource( LOG.info(f"✅ Resource '{table_fqn}' validated successfully") else: LOG.warning(f"⚠️ Resource '{table_fqn}' validation found issues") + return True except Exception as e: self.report.add_issue( @@ -1101,6 +1267,7 @@ def _validate_bigquery_resource( suggestion="Check provider credentials and network connectivity", documentation_url="https://agenticstiger.github.io/forge_docs/advanced/production-troubleshooting.html", ) + return False def _iceberg_outside_glue(self, expose: Dict[str, Any], path: str) -> bool: """Report an Iceberg table cataloged outside Glue as not checked. @@ -1128,8 +1295,9 @@ def _iceberg_outside_glue(self, expose: Dict[str, Any], path: str) -> bool: ) return True - def _validate_generic_resource(self, expose: Dict[str, Any], path: str) -> None: - """Validate a resource using the active validation provider (Snowflake, AWS, local).""" + def _validate_generic_resource(self, expose: Dict[str, Any], path: str) -> bool: + """Validate a resource with the active validation provider (Snowflake, AWS, + local, Databricks). Returns whether the comparison ran.""" if not self.validation_provider: self.report.add_issue( "warning", @@ -1137,24 +1305,27 @@ def _validate_generic_resource(self, expose: Dict[str, Any], path: str) -> None: "No validation provider initialized, skipping resource validation", path, ) - return + return False expose_id = expose.get("id") or expose.get("exposeId") or "unknown" try: - # Cache lookup + # Cache lookup. Databricks is always read live: its columns are a free + # metadata read, and a cached row count would report an hour-old table + # (and write it to a results table) after a build changed it. + cache = None if self.provider_name == "databricks" else self.cache actual_schema = None cache_key = self._resource_cache_key(expose, expose_id) - if self.cache: - actual_schema = self.cache.get_schema(cache_key, self.provider_name) + if cache: + actual_schema = cache.get_schema(cache_key, self.provider_name) if actual_schema: LOG.info("Using cached schema for %s", cache_key) # Fetch from provider if actual_schema is None: actual_schema = self.validation_provider.get_resource_schema(expose) - if self.cache and actual_schema: - self.cache.set_schema(cache_key, self.provider_name, actual_schema) + if cache and actual_schema: + cache.set_schema(cache_key, self.provider_name, actual_schema) # Validate result = self.validation_provider.validate_resource(expose, actual_schema) @@ -1206,6 +1377,7 @@ def _validate_generic_resource(self, expose: Dict[str, Any], path: str) -> None: LOG.info("Resource '%s' validated successfully", expose_id) else: LOG.warning("Resource '%s' validation found issues", expose_id) + return True except Exception as e: self.report.add_issue( @@ -1215,6 +1387,7 @@ def _validate_generic_resource(self, expose: Dict[str, Any], path: str) -> None: path, suggestion="Check provider credentials and connectivity", ) + return False def _resource_cache_key(self, expose: Dict[str, Any], expose_id: str) -> str: """Cache identity for an expose's bound resource. diff --git a/fluid_build/cli/test.py b/fluid_build/cli/test.py index 14cefc7a..479ac14c 100644 --- a/fluid_build/cli/test.py +++ b/fluid_build/cli/test.py @@ -33,7 +33,7 @@ import sys import xml.etree.ElementTree as ET from pathlib import Path -from typing import Dict, List, Optional +from typing import List, Optional from fluid_build.cli.console import cprint, cprint_literal, literal_text, success, warning from fluid_build.cli.console import error as console_error @@ -116,6 +116,12 @@ def register(subparsers: argparse._SubParsersAction) -> None: # Enable drift detection fluid test contract.fluid.yaml --check-drift + + # Databricks: JUnit for Jenkins, plus one row per check in a Unity Catalog + # table and an AI/BI dashboard over it + fluid test contract.fluid.yaml --env databricks --output junit \ + --output-file results.xml --results-table ops.dq.fluid_test_results \ + --results-dashboard "/Workspace/Shared/fluid/Data quality" """, formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -220,9 +226,98 @@ def register(subparsers: argparse._SubParsersAction) -> None: ), ) + # --- results table + dashboard (Databricks Unity Catalog) --- + p.add_argument( + "--results-table", + metavar="CATALOG.SCHEMA.TABLE", + help=( + "Append this run's results to a Unity Catalog table, one row per check: each " + "data-quality rule plus the schema, row-count and metadata checks (the same " + "cases as --output junit, which still works beside it). The table is created " + "if missing, in a schema you already have: forge never creates the catalog or " + "the schema, and refuses the product's own schema. Signs in like `fluid apply` " + "(DATABRICKS_HOST and a token or an OAuth client) and writes on " + "DATABRICKS_WAREHOUSE_ID." + ), + ) + p.add_argument( + "--results-dashboard", + metavar="WORKSPACE_PATH", + help=( + "With --results-table: keep an AI/BI dashboard over that table at this " + "workspace path, e.g. '/Workspace/Shared/fluid/Data quality' (the last part is " + "its name). It shows each product's pass rate, the failing rules and the trend. " + "Created on first use, found by its path afterwards and updated only when its " + "definition changed (an edit made in the dashboard editor is replaced then); " + "published without embedded credentials, so viewers need SELECT on the table." + ), + ) + p.set_defaults(cmd=COMMAND, func=run) +def _write_results(report, contract, results_table: str, results_dashboard) -> None: + """Append the run to the results table, then keep the dashboard over it.""" + from ._test_results_databricks import parse_results_table, write_results + + write_results(report, contract, results_table) + if results_dashboard: + from ._test_dashboard_databricks import ensure_dashboard + + ensure_dashboard(results_dashboard, parse_results_table(results_table)) + + +def _results_options(args: argparse.Namespace, contract_path: Path): + """``(table, dashboard path)`` of the results flags, checked before any test runs.""" + table = getattr(args, "results_table", None) + dashboard = getattr(args, "results_dashboard", None) + if dashboard and not table: + raise CLIError( + 2, + "test_results_dashboard_needs_table", + {"results_dashboard": dashboard, "fix": "add --results-table CATALOG.SCHEMA.TABLE"}, + ) + if table and getattr(args, "engine", "native") == "soda": + raise CLIError( + 2, + "test_results_table_needs_native_engine", + { + "results_table": table, + "why": "--results-table records the native engine's per-rule results", + "fix": "drop --engine soda, or drop --results-table", + }, + ) + if table and getattr(args, "no_data", False): + raise CLIError( + 2, + "test_results_table_needs_live_checks", + { + "results_table": table, + "why": "--no-data runs no live check, so there would be no result to record", + "fix": "drop --no-data, or drop --results-table", + }, + ) + if table: + from ._test_results_databricks import parse_results_table, refuse_product_schema + + parsed = parse_results_table(table) + # The product's own schema is refused here, before any statement runs, from + # the contract as written; write_results checks again with the overlay applied. + try: + from fluid_build.loader import load_contract + + written = load_contract(contract_path) + except Exception: # noqa: BLE001 - an unreadable contract is the run's to report + written = None + if isinstance(written, dict): + refuse_product_schema(parsed, written) + if dashboard: + from ._test_dashboard_databricks import parse_dashboard_path + + parse_dashboard_path(dashboard) + return table, dashboard + + # ====================================================================== # Entry point # ====================================================================== @@ -237,6 +332,9 @@ def run(args: argparse.Namespace, logger: logging.Logger) -> int: # file isn't there" rather than one slug per command. raise CLIError(1, "contract_file_not_found", {"path": str(contract_path)}) + # A malformed results flag fails before any check runs (or any compute starts). + results_table, results_dashboard = _results_options(args, contract_path) + # Soda engine dispatch — generates SodaCL from the contract's quality # block and shells out to the user's installed `soda` binary. Skipped # when --engine native (the default). @@ -298,6 +396,11 @@ def run(args: argparse.Namespace, logger: logging.Logger) -> int: if publish_url: _publish_results(report, publish_url, logger) + # --- results table + dashboard (Unity Catalog). After the report is out, + # so a JUnit file exists even when the write fails. --- + if results_table: + _write_results(report, validator.contract or {}, results_table, results_dashboard) + # --- exit code --- if not report.is_valid(): return 1 @@ -647,6 +750,9 @@ def _output_json(report, output_file: Optional[str] = None) -> None: "message": c.message, "expected": c.expected, "actual": c.actual, + "rule_id": getattr(c, "rule_id", None), + "rule_type": getattr(c, "rule_type", None), + "expose_id": getattr(c, "expose_id", None), } for c in getattr(report, "checks", []) ], @@ -681,12 +787,15 @@ def _output_junit(report, output_file: Optional[str] = None) -> None: ts.set("time", f"{report.duration:.3f}") ts.set("timestamp", report.validation_time.isoformat()) + from ._test_cases import rule_checks as _rule_checks + from ._test_cases import structural_categories as _structural_categories + classname = f"fluid.test.{report.contract_id}" total = 0 failures = 0 # ── one testcase per executed data-quality rule ── - rule_checks = [c for c in getattr(report, "checks", []) if c.category == "quality"] + rule_checks = _rule_checks(report) for check in rule_checks: total += 1 tc = ET.SubElement(ts, "testcase") @@ -707,33 +816,7 @@ def _output_junit(report, output_file: Optional[str] = None) -> None: fail.text = "\n".join(body) # ── one testcase per structural check category ── - categories_seen: Dict[str, List] = {} - for issue in report.issues: - categories_seen.setdefault(issue.category, []).append(issue) - - structural_categories = [ - "schema", - "connection", - "binding", - "missing_resource", - "missing_field", - "type_mismatch", - "mode_mismatch", - "extra_field", - "empty_table", - "row_count_below_threshold", - "quality", - "metadata", - "drift", - ] - if rule_checks: - # The per-rule cases above already carry every rule outcome; a - # roll-up ``quality`` case on top would double-count them. - structural_categories.remove("quality") - - extra_categories = [c for c in categories_seen if c not in structural_categories] - if rule_checks and "quality" in extra_categories: - extra_categories.remove("quality") + structural_categories, extra_categories, categories_seen = _structural_categories(report) for cat in structural_categories + extra_categories: total += 1 diff --git a/fluid_build/exporters/sodacl.py b/fluid_build/exporters/sodacl.py index fa615be0..c12c7bbd 100644 --- a/fluid_build/exporters/sodacl.py +++ b/fluid_build/exporters/sodacl.py @@ -289,11 +289,14 @@ def _dq_rules_to_checks( unmapped: list[UnmappedRule], ) -> list[Any]: """Convert ``contract.dq.rules[]`` to SodaCL checks, accounting for each.""" + from fluid_build.providers.quality_engine import expose_columns + checks: list[Any] = [] + columns = expose_columns(dict(expose)) for rule in _dq_rules(expose): rule_id = str(rule.get("id") or "unnamed") rule_type = str(rule.get("type") or "").strip().lower() - emitted, reason = _convert_dq_rule(rule, rule_type) + emitted, reason = _convert_dq_rule(rule, rule_type, columns) if emitted: checks.extend(emitted) mapped.append(f"{expose_id}.{rule_id}") @@ -309,11 +312,15 @@ def _dq_rules_to_checks( return checks -def _convert_dq_rule(rule: Mapping[str, Any], rule_type: str) -> tuple[list[Any], str]: +def _convert_dq_rule( + rule: Mapping[str, Any], rule_type: str, columns: Any = None +) -> tuple[list[Any], str]: """Map one ``$defs.dqRule`` to SodaCL checks, or explain why we can't. Returns ``(checks, reason)``. ``checks`` empty ⇒ unmapped, and ``reason`` - says why in terms the contract author can act on. + says why in terms the contract author can act on. ``columns`` is the + expose's ``contract.schema``: a valid_values rule reads its column's + declarations from it, as the native engine does. """ selector = rule.get("selector") selector = selector.strip() if isinstance(selector, str) else "" @@ -337,6 +344,20 @@ def _convert_dq_rule(rule: Mapping[str, Any], rule_type: str) -> tuple[list[Any] if not selector: return [], "rule declares no 'selector' (column name)" + from fluid_build.providers.quality_engine import rule_columns + + listed = rule_columns(selector) + if len(listed) > 1: + # As the native engine reads it: every listed column must hold. + if rule_type not in ("completeness", "accuracy", "anomaly_detection") + _VALIDITY: + return [], f"a {rule_type} rule takes one column, not {len(listed)}" + per_column: list[Any] = [] + for column in listed: + emitted, reason = _convert_dq_rule({**rule, "selector": column}, rule_type, columns) + if not emitted: + return [], f"column {column!r}: {reason}" + per_column.extend(emitted) + return per_column, "" if not _SAFE_COLUMN.match(selector): # SodaCL check expressions are parsed by an ANTLR grammar; an # exotic identifier would produce a confusing parse error deep @@ -351,8 +372,8 @@ def _convert_dq_rule(rule: Mapping[str, Any], rule_type: str) -> tuple[list[Any] # Column-scoped anomaly_detection delegates to the same bound logic # the native engine uses (``_check_anomaly_detection`` → ``_check_accuracy``). return _bound_checks(selector, threshold, operator) - if rule_type in ("validity", "valid_values"): - return _valid_values_checks(selector, rule) + if rule_type in _VALIDITY: + return _valid_values_checks(selector, rule, columns) if rule_type == "freshness": return _freshness_checks(selector, rule) @@ -363,6 +384,8 @@ def _convert_dq_rule(rule: Mapping[str, Any], rule_type: str) -> tuple[list[Any] ) +_VALIDITY = ("validity", "valid_values") + # SodaCL threshold comparators (soda-core 3.x supports =, !=, <, >, <=, >=). _SODACL_OPERATORS = { ">=": ">=", @@ -445,7 +468,7 @@ def _bound_checks(selector: str, threshold, operator: str) -> tuple[list[Any], s if threshold is None: return [], ( "bound rule declares no 'threshold', so there is nothing to " - "assert (the native engine treats this as a no-op pass)" + "assert (the native engine reports it as not run, too)" ) if operator in (">=", ">"): return [f"min({selector}) {operator} {_num(threshold)}"], "" @@ -472,18 +495,45 @@ def _bound_checks(selector: str, threshold, operator: str) -> tuple[list[Any], s ) -def _valid_values_checks(selector: str, rule: Mapping[str, Any]) -> tuple[list[Any], str]: - """validity / valid_values ⇒ ``invalid_count(col) = 0`` + ``valid values``.""" - from fluid_build.providers.quality_engine import extract_valid_values +def _valid_values_checks( + selector: str, rule: Mapping[str, Any], columns: Any = None +) -> tuple[list[Any], str]: + """validity / valid_values ⇒ ``invalid_count(col) = 0`` with the column's + ``valid values`` / ``valid regex`` / ``valid min`` / ``valid max``. - values = extract_valid_values(rule) - if not values: + The allowed values come from the native engine's own reading + (``quality_engine.value_domain``): the rule's list, else every declaration + the column makes. One SodaCL cannot express (a strict bound, a reference to + another product's table) leaves the rule unmapped rather than half-checked. + """ + from fluid_build.providers.quality_engine import value_domain + + domain = value_domain(dict(rule), columns, selector) + config: dict[str, Any] = {} + if domain.values: + config["valid values"] = list(domain.values) + if domain.pattern: + config["valid regex"] = domain.pattern + for op, bound in domain.bounds: + if op not in (">=", "<="): + return [], ( + f"range bound {op} {bound:g} has no SodaCL equivalent (valid min and " + "valid max are inclusive)" + ) + config["valid min" if op == ">=" else "valid max"] = _num_value(bound) + if domain.reference is not None: + product, column = domain.reference + return [], ( + f"the column references {product}.{column}, another product's table, " + "which SodaCL cannot look up" + ) + if not config: return [], ( "valid_values rule declares no allowed values — declare them in " - 'the description ("COLUMN valid values: A, B, C.") or as a ' - "'validValues' list" + 'the description ("COLUMN valid values: A, B, C."), as an enum, regex or ' + "range validationRule on the column, or as a 'validValues' list" ) - return [{f"invalid_count({selector}) = 0": {"valid values": list(values)}}], "" + return [{f"invalid_count({selector}) = 0": config}], "" def _freshness_checks(selector: str, rule: Mapping[str, Any]) -> tuple[list[Any], str]: diff --git a/fluid_build/iac/providers/databricks_enforce.py b/fluid_build/iac/providers/databricks_enforce.py index d78d8c11..e8cbd38b 100644 --- a/fluid_build/iac/providers/databricks_enforce.py +++ b/fluid_build/iac/providers/databricks_enforce.py @@ -281,21 +281,26 @@ def run(self, statement: str) -> None: """Run one statement to ``SUCCEEDED``; :class:`StatementFailed` otherwise.""" self.execute(statement) - def execute(self, statement: str) -> Mapping[str, Any]: - """:meth:`run`, and the finished statement's document (its ``result`` and ``manifest``).""" + def execute( + self, statement: str, parameters: Optional[Sequence[Mapping[str, Any]]] = None + ) -> Mapping[str, Any]: + """:meth:`run`, and the finished statement's document (its ``result`` and ``manifest``). + + ``parameters`` binds the statement's ``:name`` markers (``{"name", "value", "type"}`` + each), so a value never becomes statement text. + """ do = self._client.api_client.do doc: Any = None + body: Dict[str, Any] = { + "warehouse_id": self._warehouse, + "statement": statement, + "wait_timeout": "30s", + "on_wait_timeout": "CONTINUE", + } + if parameters: + body["parameters"] = [dict(p) for p in parameters] try: - doc = do( - "POST", - "/api/2.0/sql/statements/", - body={ - "warehouse_id": self._warehouse, - "statement": statement, - "wait_timeout": "30s", - "on_wait_timeout": "CONTINUE", - }, - ) + doc = do("POST", "/api/2.0/sql/statements/", body=body) deadline = time.monotonic() + self._timeout while self._state(doc) in ("PENDING", "RUNNING"): if time.monotonic() > deadline: diff --git a/fluid_build/providers/_sql_safety.py b/fluid_build/providers/_sql_safety.py index b282325f..470f5bdd 100644 --- a/fluid_build/providers/_sql_safety.py +++ b/fluid_build/providers/_sql_safety.py @@ -67,6 +67,28 @@ def quote_string_literal(value: str, *, backslash_escapes: bool = True) -> str: return "'" + escaped.replace("'", "''") + "'" +# Dialects that take a quote inside ``'...'`` only as ``\'``. Spark SQL +# (Databricks) reads ``'it''s'`` as the adjacent literals ``'it'`` and ``'s'`` +# and joins them into ``its`` (checked on a SQL warehouse); BigQuery rejects +# it. Both honour backslash escapes, so ``\\`` is doubled first. +_BACKSLASH_QUOTE_DIALECTS = frozenset({"bigquery", "databricks", "spark"}) + + +def quote_string_literal_for(dialect: str | None, value: str) -> str: + """Quote ``value`` as a string literal of ``dialect`` that reads back as ``value``. + + :func:`quote_string_literal` for every dialect that doubles a quote, and + ``'...\\'...'`` for the ones in :data:`_BACKSLASH_QUOTE_DIALECTS`: with + every backslash doubled before the quotes are escaped, no value can end + the literal early on either. + """ + if not isinstance(value, str): + raise ValueError(f"Invalid SQL string literal: {value!r}") + if (dialect or "").strip().lower() in _BACKSLASH_QUOTE_DIALECTS: + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" + return quote_string_literal(value, backslash_escapes=dialect_uses_backslash_escapes(dialect)) + + def quote_ansi_string_literal(value: str) -> str: """Quote a SQL string literal for a *standard-conforming* dialect. diff --git a/fluid_build/providers/bigquery_validation.py b/fluid_build/providers/bigquery_validation.py index 3c660dfa..a352b4b7 100644 --- a/fluid_build/providers/bigquery_validation.py +++ b/fluid_build/providers/bigquery_validation.py @@ -17,6 +17,7 @@ from fluid_build.providers._sql_safety import validate_ident from fluid_build.providers.quality_engine import ( execute_quality_checks, + expose_columns, quality_results_to_issues, ) from fluid_build.providers.validation_provider import ( @@ -416,5 +417,6 @@ def _exec(sql): table_ref=table_ref, execute_fn=_exec, dialect="bigquery", + columns=expose_columns(resource_spec), ) return quality_results_to_issues(results) diff --git a/fluid_build/providers/databricks/statements.py b/fluid_build/providers/databricks/statements.py new file mode 100644 index 00000000..dc260075 --- /dev/null +++ b/fluid_build/providers/databricks/statements.py @@ -0,0 +1,154 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""Rows from a Databricks SQL warehouse, for ``fluid test``. + +Statements run through ``iac/providers/databricks_enforce.SqlRunner``, the Statement +Execution API runner ``fluid apply`` already uses for masks and row filters: it polls a +long statement, cancels one that times out or is interrupted, and scrubs every error. +This module adds what reading results needs: ``:name`` parameters, and the rows of an +``INLINE`` ``JSON_ARRAY`` result (the API's defaults), converted by each column's +``type_name``, since every value arrives as a string, across every chunk of the result. + +The client is ``databricks_enforce.default_client()``, ``WorkspaceClient`` built from +``databricks_auth.sdk_config`` (the policy ``fluid apply`` and ``fluid verify`` share); +the SDK is the ``[databricks]`` extra, imported only when a client is made. +""" + +from __future__ import annotations + +import os +import time +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence +from urllib.parse import quote + +from fluid_build.iac.providers.databricks_enforce import ( + DATABRICKS_EXTRA_INSTALL, + WAREHOUSE_ENV, + SqlRunner, + default_client, + scrub, +) +from fluid_build.iac.providers.databricks_enforce import StatementFailed as StatementError + +STATEMENTS_PATH = "/api/2.0/sql/statements" + +#: Converters for the ``type_name`` of a ``JSON_ARRAY`` result column. +_INTEGER_TYPES = frozenset({"BYTE", "SHORT", "INT", "LONG"}) +_FLOAT_TYPES = frozenset({"FLOAT", "DOUBLE", "DECIMAL"}) + +#: A guard against a result that never stops naming a next chunk. +_MAX_CHUNKS = 10_000 + + +def warehouse_id_from_env(env: Optional[Mapping[str, str]] = None) -> str: + """The SQL warehouse id from ``DATABRICKS_WAREHOUSE_ID``, or a fix-it error.""" + value = str((env if env is not None else os.environ).get(WAREHOUSE_ENV) or "").strip() + if not value: + raise StatementError( + f"{WAREHOUSE_ENV} is not set, so there is no SQL warehouse to run the checks on. " + "Set it to the id of a SQL warehouse you can use (SQL Warehouses > the " + "warehouse > Connection details)." + ) + return value + + +def param(name: str, value: Any, type_name: str = "STRING") -> Dict[str, Any]: + """A ``parameters`` entry binding ``value`` to ``:name`` as ``type_name``. + + ``None`` binds NULL (the API reads an omitted value as NULL). + """ + entry: Dict[str, Any] = {"name": name, "type": type_name} + if value is not None: + entry["value"] = str(value) + return entry + + +def _convert(value: Any, type_name: str) -> Any: + if value is None: + return None + try: + if type_name in _INTEGER_TYPES: + return int(value) + if type_name in _FLOAT_TYPES: + return float(value) + if type_name == "BOOLEAN": + return str(value).lower() == "true" + except (TypeError, ValueError): + return value + return value + + +class StatementRunner: + """Run statements on one warehouse and return their rows as tuples.""" + + def __init__( + self, + client: Any, + warehouse_id: str, + *, + timeout_seconds: float = 300.0, + poll_seconds: float = 1.0, + sleep: Callable[[float], None] = time.sleep, + ) -> None: + self._client = client + self._sql = SqlRunner( + client, warehouse_id, timeout=timeout_seconds, poll=poll_seconds, sleep=sleep + ) + self._unreachable: Optional[str] = None + + def run( + self, sql: str, parameters: Optional[Sequence[Mapping[str, Any]]] = None + ) -> List[tuple]: + """Run ``sql`` (``:name`` markers bound from ``parameters``) and return its rows. + + A statement that failed because the warehouse did not answer (a timeout, a + transport error, a cancelled statement), rather than because of its SQL, stops + the ones after it: each would wait out the same timeout. + """ + if self._unreachable: + raise StatementError(f"not run: {self._unreachable}") + try: + doc = self._sql.execute(sql, parameters) + except StatementError as exc: + if not str(exc).startswith("FAILED"): + self._unreachable = f"an earlier statement did not complete ({exc})" + raise + columns = ((doc.get("manifest") or {}).get("schema") or {}).get("columns") or [] + types = [str(c.get("type_name") or "").upper() for c in columns] + statement_id = str(doc.get("statement_id") or "") + result = doc.get("result") or {} + rows: List[tuple] = [] + for _ in range(_MAX_CHUNKS): + for raw in result.get("data_array") or []: + rows.append( + tuple( + _convert(v, types[i] if i < len(types) else "") for i, v in enumerate(raw) + ) + ) + index = result.get("next_chunk_index") + if index is None or not statement_id: + break + result = self._chunk(statement_id, int(index)) + return rows + + def _chunk(self, statement_id: str, index: int) -> Mapping[str, Any]: + path = f"{STATEMENTS_PATH}/{quote(statement_id, safe='')}/result/chunks/{index}" + try: + out = self._client.api_client.do("GET", path) + except Exception as exc: # noqa: BLE001 - every transport failure is the statement's + raise StatementError(f"{type(exc).__name__}: {scrub(exc)}") from None + return out if isinstance(out, dict) else {} + + +__all__ = [ + "DATABRICKS_EXTRA_INSTALL", + "STATEMENTS_PATH", + "StatementError", + "StatementRunner", + "WAREHOUSE_ENV", + "default_client", + "param", + "scrub", + "warehouse_id_from_env", +] diff --git a/fluid_build/providers/databricks_validation.py b/fluid_build/providers/databricks_validation.py new file mode 100644 index 00000000..6ee83a8f --- /dev/null +++ b/fluid_build/providers/databricks_validation.py @@ -0,0 +1,421 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``fluid test`` against a Unity Catalog table (``binding.platform: databricks``). + +What is read, and what it costs: + +- **Columns**: ``GET /api/2.1/unity-catalog/tables/{catalog.schema.table}``, a + metadata call that starts no compute. A 404 means the table does not exist. + The contract's types go through the emitter's own FLUID → Databricks type + table (``engines/dbt/_types.sql_type``) and both sides through the + provider's normaliser, so ``VARCHAR`` matches the ``string`` that + ``fluid apply`` created. +- **Row count and data-quality rules**: SQL on ``DATABRICKS_WAREHOUSE_ID`` + through the Statement Execution API (``databricks/statements.py``), one + ``SELECT`` per rule; a rule over several columns is still one statement. + Nothing is written. + +The table is ``binding.location`` ``database`` (the Unity Catalog catalog, the +convention ``fluid apply`` uses), ``schema`` and ``table``; every part is +validated before it reaches a URL or a statement, and backtick-quoted in SQL. + +Sign-in is ``databricks_auth.sdk_config`` (``DATABRICKS_HOST`` plus a token, +an OAuth client or a named profile), the policy ``fluid apply`` and +``fluid verify`` share. +""" + +from __future__ import annotations + +import os +from typing import Any, Callable, Dict, List, Optional, Tuple +from urllib.parse import quote + +from fluid_build.providers._sql_safety import validate_ident +from fluid_build.providers.quality_engine import ( + QualityCheckResult, + execute_quality_checks, + expose_columns, + quality_results_to_issues, +) +from fluid_build.providers.validation_provider import ( + FieldSchema, + ResourceSchema, + ResourceType, + ValidationIssue, + ValidationProvider, + ValidationResult, +) + +TABLES_PATH = "/api/2.1/unity-catalog/tables" + +#: The table tag ``fluid apply`` puts a product's id in (``iac/providers/databricks.py`` +#: ``_PRODUCT_TAGS``); a reference to another product finds its table by it. +PRODUCT_TAG = "fluid_product_id" +WAREHOUSES_PATH = "/api/2.0/sql/warehouses" + + +def _is_not_found(exc: BaseException) -> bool: + for attr in ("status_code", "status", "http_status"): + if getattr(exc, attr, None) == 404: + return True + if type(exc).__name__ in {"NotFound", "ResourceDoesNotExist"}: + return True + code = str(getattr(exc, "error_code", "") or "") + return code in {"TABLE_DOES_NOT_EXIST", "RESOURCE_DOES_NOT_EXIST", "NOT_FOUND"} + + +def table_name(resource_spec: Dict[str, Any]) -> Optional[Tuple[str, str, str]]: + """``(catalog, schema, table)`` of an expose, each validated, or ``None``. + + Raises ``ValueError`` for a part that is not a plain identifier. + """ + binding = resource_spec.get("binding") or {} + location = binding.get("location") if isinstance(binding, dict) else None + if not isinstance(location, dict): + return None + catalog = location.get("database") or location.get("catalog") + schema = location.get("schema") + table = location.get("table") + if not (catalog and schema and table): + return None + return validate_ident(str(catalog)), validate_ident(str(schema)), validate_ident(str(table)) + + +def sql_table_ref(parts: Tuple[str, str, str]) -> str: + """```catalog`.`schema`.`table``` for validated parts.""" + return ".".join(f"`{validate_ident(p)}`" for p in parts) + + +def _not_run(rule: Dict[str, Any], full_name: str) -> QualityCheckResult: + """A rule on a table that does not exist: failed at its severity, never run.""" + rule_id = str(rule.get("id") or "unnamed") + return QualityCheckResult( + rule_id=rule_id, + rule_type=str(rule.get("type") or "").lower(), + selector=str(rule.get("selector") or ""), + passed=False, + severity=rule.get("severity", "warning"), + message=f"Rule '{rule_id}' was not checked: table {full_name} does not exist", + expected="the table to exist", + actual="not executed", + ) + + +class DatabricksValidationProvider(ValidationProvider): + """Validate an expose against its live Unity Catalog table.""" + + def __init__(self, config: Dict[str, Any]): + super().__init__(config) + self._client_factory: Optional[Callable[[], Any]] = config.get("client_factory") + self._warehouse_id: Optional[str] = config.get("warehouse_id") + self._client: Any = None + self._runner: Any = None + self._references: Dict[Tuple[str, str, str], Any] = {} + self._tables: Dict[str, Optional[Dict[str, Any]]] = {} + + @property + def provider_name(self) -> str: + return "databricks" + + # ── plumbing ──────────────────────────────────────────────────────── + + def _get_client(self) -> Any: + if self._client is None: + from fluid_build.providers.databricks.statements import default_client + + self._client = (self._client_factory or default_client)() + return self._client + + def _sql(self): + if self._runner is None: + from fluid_build.providers.databricks.statements import ( + StatementRunner, + warehouse_id_from_env, + ) + + warehouse = self._warehouse_id or warehouse_id_from_env(os.environ) + self._runner = StatementRunner(self._get_client(), warehouse) + return self._runner + + def _table(self, parts: Tuple[str, str, str]) -> Optional[Dict[str, Any]]: + """The table's Unity Catalog metadata, read once per run; ``None`` when it + does not exist. A metadata read, so no compute starts.""" + full_name = ".".join(parts) + if full_name not in self._tables: + try: + self._tables[full_name] = self._get(f"{TABLES_PATH}/{quote(full_name, safe='')}") + except Exception as exc: # noqa: BLE001 + if not _is_not_found(exc): + from fluid_build.providers.databricks.statements import StatementError, scrub + + raise StatementError(f"{type(exc).__name__}: {scrub(exc)}") from None + self._tables[full_name] = None + return self._tables[full_name] + + def _get(self, path: str) -> Dict[str, Any]: + out = self._get_client().api_client.do("GET", path) + return out if isinstance(out, dict) else {} + + # ── ValidationProvider ────────────────────────────────────────────── + + def validate_connection(self) -> bool: + """Sign in and find the warehouse; neither starts compute. + + Raises with a message that says what to fix: no host or credential, + no ``DATABRICKS_WAREHOUSE_ID``, or a warehouse this identity cannot see. + """ + from fluid_build.providers.databricks.statements import ( + StatementError, + scrub, + warehouse_id_from_env, + ) + + warehouse = self._warehouse_id or warehouse_id_from_env(os.environ) + try: + self._get(f"{WAREHOUSES_PATH}/{quote(warehouse, safe='')}") + except Exception as exc: # noqa: BLE001 - one safe message per failure + if _is_not_found(exc): + raise StatementError( + f"SQL warehouse {warehouse!r} was not found in the workspace " + "DATABRICKS_HOST names, or this identity cannot use it" + ) from None + raise StatementError(f"{type(exc).__name__}: {scrub(exc)}") from None + return True + + def get_resource_schema(self, resource_spec: Dict[str, Any]) -> Optional[ResourceSchema]: + parts = table_name(resource_spec) + if parts is None: + raise ValueError( + "binding.location needs database (the Unity Catalog catalog), schema and table" + ) + full_name = ".".join(parts) + table = self._table(parts) + if table is None: + return None + from fluid_build.iac.providers.databricks import normalize_column_type + + fields: List[FieldSchema] = [] + for col in table.get("columns") or []: + if not isinstance(col, dict) or not col.get("name"): + continue + raw = col.get("type_text") or col.get("type_name") or "" + fields.append( + FieldSchema( + name=str(col["name"]), + type=normalize_column_type(raw), + mode="NULLABLE" if col.get("nullable", True) is not False else "REQUIRED", + description=col.get("comment"), + ) + ) + rows = self._sql().run(f"SELECT COUNT(*) FROM {sql_table_ref(parts)}") + row_count = int(rows[0][0]) if rows and rows[0] and rows[0][0] is not None else None + return ResourceSchema( + resource_type=ResourceType.TABLE, + fully_qualified_name=full_name, + fields=fields, + row_count=row_count, + metadata={"table_type": table.get("table_type")}, + ) + + def validate_resource( + self, + contract_spec: Dict[str, Any], + actual_schema: Optional[ResourceSchema], + ) -> ValidationResult: + issues: List[ValidationIssue] = [] + resource_name = contract_spec.get("id") or contract_spec.get("exposeId") or "unknown" + if actual_schema is None: + parts = table_name(contract_spec) + name = ".".join(parts) if parts else "unknown" + issues.append( + ValidationIssue( + severity="error", + category="missing_resource", + message=f"Table '{name}' does not exist in Unity Catalog", + path="exposes[].binding.location", + expected=name, + actual=None, + suggestion="Run 'fluid apply' for this contract, or correct " + "binding.location (database is the catalog)", + ) + ) + return ValidationResult(resource_name, False, issues, None) + + expected = self._expected_fields(contract_spec) + if expected: + issues.extend(self.compare_schemas(expected, actual_schema.fields)) + + if actual_schema.row_count == 0: + issues.append( + ValidationIssue( + severity="warning", + category="empty_table", + message=f"Table '{actual_schema.fully_qualified_name}' has no rows", + path="exposes[].binding.location", + expected="> 0 rows", + actual="0 rows", + suggestion="Run the product's build, then test again", + ) + ) + sla = contract_spec.get("sla") or {} + min_rows = sla.get("row_count_min") if isinstance(sla, dict) else None + if min_rows is not None and actual_schema.row_count is not None: + if actual_schema.row_count < min_rows: + issues.append( + ValidationIssue( + severity="error", + category="row_count_below_threshold", + message=( + f"Table has {actual_schema.row_count} rows, below minimum of " + f"{min_rows}" + ), + path="exposes[].sla.row_count_min", + expected=f">= {min_rows} rows", + actual=f"{actual_schema.row_count} rows", + suggestion="Check data pipeline for issues", + ) + ) + return ValidationResult( + resource_name=resource_name, + success=not any(i.severity == "error" for i in issues), + issues=issues, + schema=actual_schema, + ) + + def normalize_type(self, provider_type: str) -> str: + from fluid_build.iac.providers.databricks import normalize_column_type + + return normalize_column_type(provider_type) + + @staticmethod + def _expected_fields(resource_spec: Dict[str, Any]) -> List[FieldSchema]: + """The contract's columns, typed as ``fluid apply`` creates them.""" + from fluid_build.engines.dbt._types import sql_type + + fields: List[FieldSchema] = [] + for col in expose_columns(resource_spec): + if "required" in col: + mode = "REQUIRED" if col["required"] else "NULLABLE" + elif "nullable" in col: + mode = "NULLABLE" if col["nullable"] else "REQUIRED" + else: + mode = str(col.get("mode") or "NULLABLE").upper() + fields.append( + FieldSchema( + name=str(col.get("name", "")), + type=sql_type(str(col.get("type") or "string"), "databricks"), + mode=mode, + description=col.get("description"), + ) + ) + return fields + + # ── data-quality rules ────────────────────────────────────────────── + + def run_quality_results( + self, resource_spec: Dict[str, Any], rules: List[Dict[str, Any]] + ) -> List[QualityCheckResult]: + """Every rule's result, passing ones included (with what was measured). + + A table that does not exist runs no statement: each rule is reported as not + executed, and the missing table itself is the resource check's error. + """ + parts = table_name(resource_spec) + if parts is None: + raise ValueError( + "binding.location needs database (the Unity Catalog catalog), schema and table" + ) + if self._table(parts) is None: + return [_not_run(rule, ".".join(parts)) for rule in rules if isinstance(rule, dict)] + return execute_quality_checks( + rules=rules, + table_ref=sql_table_ref(parts), + execute_fn=self._sql().run, + dialect="databricks", + columns=expose_columns(resource_spec), + resolve_reference=lambda product, column: self._reference_table( + product, column, prefer_catalog=parts[0] + ), + ) + + def _reference_table( + self, product_id: str, column: str, prefer_catalog: Optional[str] = None + ) -> str: + """The live table of ``product_id`` that has ``column``, as a SQL reference. + + Found by the ``fluid_product_id`` tag ``fluid apply`` puts on every table it + makes (one ``system.information_schema.table_tags`` query, which lists only + what this identity can see), then by the column, from each candidate's + Unity Catalog metadata. Tables in ``prefer_catalog`` (the tested table's + catalog) win over tables elsewhere, so a look-alike tagged table in another + catalog cannot stand in for the product's own. Raises ``LookupError`` saying + why when there is no such table, or more than one. + """ + key = (product_id, column.lower(), (prefer_catalog or "").lower()) + if key not in self._references: + try: + self._references[key] = self._find_reference(product_id, column, prefer_catalog) + except Exception as exc: # noqa: BLE001 - one failed lookup, said once per rule + from fluid_build.providers.databricks.statements import scrub + + self._references[key] = LookupError( + f"the lookup of the table tagged {PRODUCT_TAG}={product_id} failed " + f"({type(exc).__name__}: {scrub(exc)})" + ) + found = self._references[key] + if isinstance(found, LookupError): + raise found + return found + + def _find_reference( + self, product_id: str, column: str, prefer_catalog: Optional[str] = None + ) -> Any: + from fluid_build.providers.databricks.statements import param + + rows = self._sql().run( + "SELECT catalog_name, schema_name, table_name " + "FROM system.information_schema.table_tags " + "WHERE tag_name = :tag AND tag_value = :product", + [param("tag", PRODUCT_TAG), param("product", product_id)], + ) + candidates = sorted({tuple(str(v) for v in row[:3]) for row in rows if len(row) >= 3}) + having = [] + for parts in candidates: + try: + table = self._get(f"{TABLES_PATH}/{quote('.'.join(parts), safe='')}") + except Exception: # noqa: BLE001 - a table this identity cannot read is no match + continue + names = {str(c.get("name", "")).lower() for c in table.get("columns") or []} + if column.lower() in names: + having.append(parts) + local = [p for p in having if prefer_catalog and p[0].lower() == prefer_catalog.lower()] + if local: + having = local + if not having: + return LookupError( + f"no table tagged {PRODUCT_TAG}={product_id} with a column {column!r} is " + "visible to this identity (has that product been applied?)" + ) + if len(having) > 1: + return LookupError( + # The tables are not named: this message is written to the shared + # results table, whose readers may not see them. + f"{len(having)} tables tagged {PRODUCT_TAG}={product_id} have a column " + f"{column!r}, so which one it references is ambiguous; keep the tag on the " + "product's own table only" + ) + try: + return sql_table_ref(having[0]) + except ValueError: + return LookupError( + f"the table of {product_id} has a name that is not a plain identifier" + ) + + def run_quality_checks( + self, resource_spec: Dict[str, Any], rules: List[Dict[str, Any]] + ) -> List[ValidationIssue]: + return quality_results_to_issues(self.run_quality_results(resource_spec, rules)) + + +__all__ = ["DatabricksValidationProvider", "sql_table_ref", "table_name"] diff --git a/fluid_build/providers/local_validation.py b/fluid_build/providers/local_validation.py index 4ab17792..00be176c 100644 --- a/fluid_build/providers/local_validation.py +++ b/fluid_build/providers/local_validation.py @@ -20,6 +20,7 @@ from fluid_build.providers._sql_safety import quote_ansi_string_literal, validate_ident from fluid_build.providers.quality_engine import ( execute_quality_checks, + expose_columns, quality_results_to_issues, ) from fluid_build.providers.validation_provider import ( @@ -432,6 +433,7 @@ def _exec(sql): table_ref=table_ref, execute_fn=_exec, dialect="ansi", + columns=expose_columns(resource_spec), ) return quality_results_to_issues(results) finally: @@ -465,6 +467,7 @@ def _exec(sql): table_ref=table_ref, execute_fn=_exec, dialect="ansi", + columns=expose_columns(resource_spec), ) return quality_results_to_issues(results) finally: diff --git a/fluid_build/providers/quality_engine.py b/fluid_build/providers/quality_engine.py index 13d96719..175ae20e 100644 --- a/fluid_build/providers/quality_engine.py +++ b/fluid_build/providers/quality_engine.py @@ -17,14 +17,12 @@ from __future__ import annotations import logging +import math import re from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Tuple -from fluid_build.providers._sql_safety import ( - dialect_uses_backslash_escapes, - quote_string_literal, -) +from fluid_build.providers._sql_safety import quote_string_literal_for from fluid_build.providers.validation_provider import ValidationIssue LOG = logging.getLogger("fluid.providers.quality_engine") @@ -39,6 +37,166 @@ def _validate_ident(name: str) -> str: return name +# Dialects that read ``"x"`` as a string literal, not as a column: an +# identifier there needs backticks. ``COUNT(DISTINCT "col")`` on Databricks +# or BigQuery counts one constant, so a uniqueness or completeness rule +# quoted the ANSI way passes whatever the data holds. +_BACKTICK_DIALECTS = frozenset({"bigquery", "databricks"}) + +# The floating-point type the ratio casts use. ``DOUBLE PRECISION`` is ANSI, +# DuckDB and Snowflake; Spark SQL and BigQuery do not accept it. +_DOUBLE_TYPES = {"bigquery": "FLOAT64", "databricks": "DOUBLE"} + + +def _dialect(dialect: Optional[str]) -> str: + return (dialect or "ansi").strip().lower() + + +def _qi(column: str, dialect: Optional[str] = "ansi") -> str: + """``column`` validated and quoted as an identifier of ``dialect``.""" + col = _validate_ident(column) + return f"`{col}`" if _dialect(dialect) in _BACKTICK_DIALECTS else f'"{col}"' + + +def _double(dialect: Optional[str]) -> str: + return _DOUBLE_TYPES.get(_dialect(dialect), "DOUBLE PRECISION") + + +def rule_columns(selector: Any) -> List[str]: + """The columns a rule's ``selector`` names, in order. + + ``dqRule.selector`` is one column, ``"*"``, or a comma-separated list of + columns (the industry packs write ``site_id,meter_id,reading_ts`` for + "all mandatory attributes"). Completeness and accuracy rules hold for + every listed column; the other column rules take exactly one. + """ + if not isinstance(selector, str): + return [] + return [part.strip() for part in selector.split(",") if part.strip()] + + +def _column_constraint(columns: Any, name: str, kind: str) -> Optional[str]: + """The ``constraint`` of the first ``validationRules[]`` entry of ``kind`` on + the contract column called ``name`` (case-insensitive), if any.""" + if not isinstance(columns, (list, tuple)) or not isinstance(name, str): + return None + for col in columns: + if not isinstance(col, dict) or str(col.get("name", "")).lower() != name.lower(): + continue + for entry in col.get("validationRules") or []: + if not isinstance(entry, dict): + continue + if str(entry.get("type") or "").strip().lower() != kind: + continue + constraint = entry.get("constraint") + if isinstance(constraint, str) and constraint.strip(): + return constraint.strip() + return None + + +def expose_columns(expose: Any) -> List[Dict[str, Any]]: + """The expose's declared columns (``contract.schema``, else ``schema``).""" + if not isinstance(expose, dict): + return [] + contract = expose.get("contract") + columns = contract.get("schema") if isinstance(contract, dict) else None + if not columns: + columns = expose.get("schema") + return [c for c in columns if isinstance(c, dict)] if isinstance(columns, list) else [] + + +def extract_reference(rule: Dict[str, Any], columns: Any = None) -> Optional[Tuple[str, str]]: + """``(product id, column)`` that a ``valid_values`` rule's column references. + + Used when the rule names no values and its column declares no ``enum`` or + ``regex``: the column's ``labels.references: .`` (the + foreign-key convention the Unity Catalog keys read too) makes a valid value + one that the referenced product's column holds. + """ + name = rule.get("selector") or "" + if not isinstance(columns, (list, tuple)) or not isinstance(name, str): + return None + for col in columns: + if not isinstance(col, dict) or str(col.get("name", "")).lower() != name.lower(): + continue + labels = col.get("labels") + target = labels.get("references") if isinstance(labels, dict) else None + if isinstance(target, str): + product, _, column = target.strip().rpartition(".") + if product and column: + return product, column + return None + + +_NUMBER = r"-?\d+(?:\.\d+)?" +_RANGE_BETWEEN = re.compile(rf"^\s*between\s+({_NUMBER})\s+and\s+({_NUMBER})\s*$", re.IGNORECASE) +_RANGE_BOUND = re.compile(rf"^\s*(>=|<=|>|<)\s*({_NUMBER})\s*$") +_RANGE_AND = re.compile(r"\s+and\s+", re.IGNORECASE) + + +def parse_range(constraint: Any) -> List[Tuple[str, float]]: + """The bounds of a ``range`` validationRule: ``">= 0"``, ``">= 0 and <= 100"`` + or ``"between 0 and 100"`` (``[]`` for anything else).""" + if not isinstance(constraint, str): + return [] + m = _RANGE_BETWEEN.match(constraint) + if m: + bounds = [(">=", float(m.group(1))), ("<=", float(m.group(2)))] + else: + bounds = [] + for part in _RANGE_AND.split(constraint.strip()): + m = _RANGE_BOUND.match(part) + if not m: + return [] + bounds.append((m.group(1), float(m.group(2)))) + # A bound too large for a float would reach SQL as ``inf``: not a number. + return bounds if all(math.isfinite(n) for _, n in bounds) else [] + + +@dataclass(frozen=True) +class ValueDomain: + """What a valid value of one column is: the rule's own value list, or else + every declaration the column makes (its ``enum``, ``regex`` and ``range`` + validationRules and its ``labels.references``), all of which must hold.""" + + values: Tuple[str, ...] = () + pattern: Optional[str] = None + bounds: Tuple[Tuple[str, float], ...] = () + reference: Optional[Tuple[str, str]] = None + + +def value_domain(rule: Dict[str, Any], columns: Any = None, column: Optional[str] = None): + """The :class:`ValueDomain` of ``column`` (default: the rule's selector).""" + sub = rule if column is None else {**rule, "selector": column} + listed = extract_valid_values(sub) + if listed: + return ValueDomain(values=tuple(listed)) + name = sub.get("selector") or "" + enum = _column_constraint(columns, name, "enum") + return ValueDomain( + values=tuple(v.strip() for v in re.split(r"[|,]", enum) if v.strip()) if enum else (), + pattern=extract_value_pattern(sub, columns), + bounds=tuple(parse_range(_column_constraint(columns, name, "range"))), + reference=extract_reference(sub, columns), + ) + + +#: ``resolve_reference(product id, column) -> table reference for SQL``; raises +#: ``LookupError`` (with a message that says why) when the table cannot be found. +ReferenceResolver = Callable[[str, str], str] + + +def extract_value_pattern(rule: Dict[str, Any], columns: Any = None) -> Optional[str]: + """The regular expression a ``valid_values`` rule's column must match. + + Used only when the rule names no allowed values (see + :func:`extract_valid_values`): the selected column's own ``regex`` + validationRule is then what "a valid value" means + (``country_code: ^[A-Z]{2}$``). + """ + return _column_constraint(columns, rule.get("selector") or "", "regex") + + @dataclass class QualityCheckResult: """Result of a single data quality check.""" @@ -58,61 +216,144 @@ class QualityCheckResult: # ------------------------------------------------------------------ -def _completeness_sql(table_ref: str, column: str) -> str: - """Generate SQL to compute non-null fraction for a column.""" - col = _validate_ident(column) +def _completeness_expr(column: str, dialect: str) -> str: return ( - "SELECT " - f'CAST(SUM(CASE WHEN "{col}" IS NOT NULL THEN 1 ELSE 0 END) AS DOUBLE PRECISION) ' - "/ NULLIF(COUNT(*), 0) AS completeness_ratio " - f"FROM {table_ref}" + f"CAST(SUM(CASE WHEN {_qi(column, dialect)} IS NOT NULL THEN 1 ELSE 0 END) " + f"AS {_double(dialect)}) / NULLIF(COUNT(*), 0)" ) -def _uniqueness_sql(table_ref: str, column: str) -> str: +def _completeness_sql(table_ref: str, column: str, dialect: str = "ansi") -> str: + """Generate SQL to compute non-null fraction for a column.""" + return f"SELECT {_completeness_expr(column, dialect)} AS completeness_ratio FROM {table_ref}" + + +def _completeness_columns_sql(table_ref: str, columns: List[str], dialect: str) -> str: + """One statement: the non-null fraction of each of ``columns``, in order.""" + ratios = ", ".join(_completeness_expr(c, dialect) for c in columns) + return f"SELECT {ratios} FROM {table_ref}" + + +def _uniqueness_sql(table_ref: str, column: str, dialect: str = "ansi") -> str: """Generate SQL to compute distinct-value fraction for a column.""" - col = _validate_ident(column) + col = _qi(column, dialect) return ( "SELECT " - f'CAST(COUNT(DISTINCT "{col}") AS DOUBLE PRECISION) ' - f'/ NULLIF(COUNT("{col}"), 0) AS uniqueness_ratio ' + f"CAST(COUNT(DISTINCT {col}) AS {_double(dialect)}) " + f"/ NULLIF(COUNT({col}), 0) AS uniqueness_ratio " f"FROM {table_ref}" ) -def _accuracy_min_sql(table_ref: str, column: str) -> str: +def _accuracy_min_sql(table_ref: str, column: str, dialect: str = "ansi") -> str: """Generate SQL to get the minimum value of a numeric column.""" - col = _validate_ident(column) - return f'SELECT MIN("{col}") AS min_val FROM {table_ref}' + return f"SELECT MIN({_qi(column, dialect)}) AS min_val FROM {table_ref}" -def _accuracy_max_sql(table_ref: str, column: str) -> str: +def _accuracy_max_sql(table_ref: str, column: str, dialect: str = "ansi") -> str: """Generate SQL to get the maximum value of a numeric column.""" - col = _validate_ident(column) - return f'SELECT MAX("{col}") AS max_val FROM {table_ref}' + return f"SELECT MAX({_qi(column, dialect)}) AS max_val FROM {table_ref}" + +def _accuracy_bound_columns_sql( + table_ref: str, columns: List[str], aggregate: str, dialect: str +) -> str: + """One statement: ``MIN`` (or ``MAX``) of each of ``columns``, in order.""" + func = "MAX" if aggregate == "max" else "MIN" + bounds = ", ".join(f"{func}({_qi(c, dialect)})" for c in columns) + return f"SELECT {bounds} FROM {table_ref}" -def _accuracy_violations_sql(table_ref: str, column: str, operator: str, threshold) -> str: + +def _violation_predicate(column: str, operator: str, threshold, dialect: str) -> str: + col = _qi(column, dialect) + negated = _NEGATED_OPERATORS[operator] + # ``threshold`` is schema-typed ``number``; coerce so nothing but a + # numeric literal can reach the SQL text. + literal = float(threshold) + return f"{col} IS NOT NULL AND {col} {negated} {literal}" + + +def _accuracy_violations_sql( + table_ref: str, column: str, operator: str, threshold, dialect: str = "ansi" +) -> str: """Count rows whose value does NOT satisfy ``value threshold``. Used for equality/inequality bounds, where no single aggregate can decide the rule: ``MIN(col) == t`` says nothing about the rest of the column. """ - col = _validate_ident(column) - negated = _NEGATED_OPERATORS[operator] - # ``threshold`` is schema-typed ``number``; coerce so nothing but a - # numeric literal can reach the SQL text. - literal = float(threshold) return ( f"SELECT COUNT(*) AS violation_count FROM {table_ref} " - f'WHERE "{col}" IS NOT NULL AND "{col}" {negated} {literal}' + f"WHERE {_violation_predicate(column, operator, threshold, dialect)}" + ) + + +def _accuracy_violations_columns_sql( + table_ref: str, columns: List[str], operator: str, threshold, dialect: str +) -> str: + """One statement: the violating-row count of each of ``columns``, in order.""" + counts = ", ".join( + f"SUM(CASE WHEN {_violation_predicate(c, operator, threshold, dialect)} " + "THEN 1 ELSE 0 END)" + for c in columns + ) + return f"SELECT {counts} FROM {table_ref}" + + +def _validity_pattern_sql(table_ref: str, column: str, pattern: str, dialect: str) -> str: + """Count non-null values of ``column`` that do not match ``pattern`` anywhere. + + Each dialect's search-anywhere regex test (Snowflake's ``REGEXP_LIKE`` + matches the whole value, so ``REGEXP_INSTR`` stands in for it); an + anchored pattern (``^[A-Z]{2}$``) means the same on all of them. + """ + col = _qi(column, dialect) + return ( + f"SELECT COUNT(*) AS invalid_count FROM {table_ref} " + f"WHERE {col} IS NOT NULL AND {_pattern_mismatch(col, pattern, dialect)}" + ) + + +def _pattern_mismatch(col: str, pattern: str, dialect: str) -> str: + """The predicate "the (quoted) column ``col`` does not match ``pattern``".""" + literal = quote_string_literal_for(dialect, pattern) + name = _dialect(dialect) + if name == "databricks": + return f"NOT ({col} RLIKE {literal})" + if name == "bigquery": + return f"NOT REGEXP_CONTAINS({col}, {literal})" + if name == "snowflake": + return f"REGEXP_INSTR({col}, {literal}) = 0" + if name in ("ansi", "duckdb"): + return f"NOT regexp_matches({col}, {literal})" + raise ValueError(f"no regular-expression test for SQL dialect {dialect!r}") + + +def _range_violation(col: str, bounds: Tuple[Tuple[str, float], ...]) -> str: + """The predicate "the (quoted) column ``col`` is outside ``bounds``".""" + inside = " AND ".join(f"{col} {op} {float(n)!r}" for op, n in bounds if op in _OPERATORS) + return f"NOT ({inside})" + + +def _validity_range_sql( + table_ref: str, column: str, bounds: Tuple[Tuple[str, float], ...], dialect: str +) -> str: + """Count non-null values of ``column`` outside the declared bounds.""" + col = _qi(column, dialect) + return ( + f"SELECT COUNT(*) AS invalid_count FROM {table_ref} " + f"WHERE {col} IS NOT NULL AND {_range_violation(col, bounds)}" ) def _freshness_sql(table_ref: str, column: str, dialect: str = "ansi") -> str: """Generate SQL to get age of most-recent timestamp value (in seconds).""" col = _validate_ident(column) + if dialect == "databricks": + return ( + f"SELECT timestampdiff(SECOND, MAX(`{col}`), current_timestamp()) AS age_seconds " + f"FROM {table_ref}" + ) if dialect == "snowflake": return ( f"SELECT DATEDIFF('second', MAX(\"{col}\"), CURRENT_TIMESTAMP()) AS age_seconds " @@ -222,7 +463,7 @@ def _parse_duration_seconds(value: str) -> Optional[int]: parse_duration_seconds = _parse_duration_seconds -def extract_valid_values(rule: Dict[str, Any]) -> List[str]: +def extract_valid_values(rule: Dict[str, Any], columns: Any = None) -> List[str]: """Allowed values for a ``validity`` / ``valid_values`` rule. ``$defs.dqRule`` is ``additionalProperties: false`` and has no @@ -230,6 +471,12 @@ def extract_valid_values(rule: Dict[str, Any]) -> List[str]: the description: ``"COLUMN valid values: A, B, C."``. An explicit ``validValues`` list still wins when present (hand-written files). + With ``columns`` (the expose's ``contract.schema``), a rule that names no + values takes them from its column's own ``enum`` validationRule + (``constraint: "GOOD | ESTIMATED | SUSPECT"``, ``|`` or ``,`` separated): + that is where a contract already declares them, and where the industry + packs do. + Shared by the native engine and the SodaCL exporter so the two cannot disagree about which values a rule allows. """ @@ -237,12 +484,16 @@ def extract_valid_values(rule: Dict[str, Any]) -> List[str]: if explicit: return [str(v) for v in explicit] description = rule.get("description") or "" - if not isinstance(description, str) or " valid values:" not in description.lower(): - return [] - m = re.search(r"valid values:\s*([^.]+)", description, re.IGNORECASE) - if not m: - return [] - return [v.strip() for v in m.group(1).split(",") if v.strip()] + if isinstance(description, str) and " valid values:" in description.lower(): + m = re.search(r"valid values:\s*([^.]+)", description, re.IGNORECASE) + if m: + values = [v.strip() for v in m.group(1).split(",") if v.strip()] + if values: + return values + enum = _column_constraint(columns, rule.get("selector") or "", "enum") + if enum: + return [v.strip() for v in re.split(r"[|,]", enum) if v.strip()] + return [] # ------------------------------------------------------------------ @@ -255,6 +506,8 @@ def execute_quality_checks( table_ref: str, execute_fn, dialect: str = "ansi", + columns: Any = None, + resolve_reference: Optional[ReferenceResolver] = None, ) -> List[QualityCheckResult]: """ Execute a list of DQ rules against a live table. @@ -269,7 +522,15 @@ def execute_quality_checks( ``execute_fn(sql) -> list[tuple]`` — runs a SQL statement and returns the result rows. dialect : str - SQL dialect hint: ``"ansi"``, ``"snowflake"``, ``"bigquery"``. + SQL dialect hint: ``"ansi"``, ``"snowflake"``, ``"bigquery"``, + ``"databricks"``. + columns : list[dict], optional + The expose's ``contract.schema``. A ``valid_values`` rule that names + no values reads its column's ``enum`` (or ``regex``) validationRule, or + its ``labels.references``. + resolve_reference : callable, optional + ``(product id, column) -> table reference`` of a referenced product; + without it a reference rule is reported as not executed. Returns ------- @@ -303,22 +564,26 @@ def execute_quality_checks( # Support validValues from explicit key OR parsed from description # Description format: "column valid values: val1, val2, val3." # Shared with the SodaCL exporter so both engines read the same list. - explicit_vv = extract_valid_values(rule) - - result = _execute_single_rule( - rule_id=rule_id, - rule_type=rule_type, - selector=selector, - severity=severity, - description=description, - threshold=threshold, - operator=operator, - valid_values=explicit_vv, - window=rule.get("window", rule.get("freshness")), - table_ref=table_ref, - execute_fn=execute_fn, - dialect=dialect, - ) + if rule_type in _VALIDITY_TYPES and len(rule_columns(selector)) > 1: + result = _check_validity_columns( + rule, columns, table_ref, execute_fn, dialect, resolve_reference + ) + else: + result = _execute_single_rule( + rule_id=rule_id, + rule_type=rule_type, + selector=selector, + severity=severity, + description=description, + threshold=threshold, + operator=operator, + domain=value_domain(rule, columns), + resolve_reference=resolve_reference, + window=rule.get("window", rule.get("freshness")), + table_ref=table_ref, + execute_fn=execute_fn, + dialect=dialect, + ) results.append(result) except Exception as e: results.append( @@ -373,6 +638,13 @@ def quality_results_to_issues( } ) +# Rule types whose comma-separated selector means "every listed column" (a +# valid_values rule checks each listed column against its own declaration). +_MULTI_COLUMN_RULE_TYPES = frozenset( + {"completeness", "accuracy", "anomaly_detection", "validity", "valid_values"} +) +_VALIDITY_TYPES = frozenset({"validity", "valid_values"}) + # Rule types ``$defs.dqRule.type`` accepts but this engine cannot run. # They are reported as failures at the rule's declared severity so a gate # nobody is enforcing can never read as green. @@ -388,13 +660,34 @@ def _execute_single_rule( description: str, threshold, operator: str, - valid_values: List[str], + domain: ValueDomain, window: Optional[str], table_ref: str, execute_fn, dialect: str, + resolve_reference: Optional[ReferenceResolver] = None, ) -> QualityCheckResult: + columns = rule_columns(selector) + if len(columns) > 1 and rule_type not in _MULTI_COLUMN_RULE_TYPES: + # ``uniqueness`` over a list would mean a composite key and + # ``valid_values`` / ``freshness`` one column's domain or clock; + # guessing either would check something the author did not write. + return QualityCheckResult( + rule_id=rule_id, + rule_type=rule_type, + selector=selector, + passed=False, + severity=severity, + message=( + f"Rule '{rule_id}' is a {rule_type} rule over {len(columns)} columns; " + f"a {rule_type} rule takes one column, so nothing was checked. " + "Write one rule per column." + ), + expected="one column", + actual="not executed", + ) + if rule_type == "completeness": return _check_completeness( rule_id, @@ -405,6 +698,7 @@ def _execute_single_rule( operator, table_ref, execute_fn, + dialect, ) elif rule_type == "uniqueness": return _check_uniqueness( @@ -416,6 +710,7 @@ def _execute_single_rule( operator, table_ref, execute_fn, + dialect, ) elif rule_type == "accuracy": return _check_accuracy( @@ -427,17 +722,21 @@ def _execute_single_rule( operator, table_ref, execute_fn, + dialect, ) - elif rule_type == "validity": + elif rule_type in _VALIDITY_TYPES: + # ``valid_values`` is the schema's name; ``validity`` the older alias. return _check_validity( rule_id, selector, severity, description, - valid_values, + domain, table_ref, execute_fn, dialect, + rule_type=rule_type, + resolve_reference=resolve_reference, ) elif rule_type == "freshness": return _check_freshness( @@ -461,17 +760,6 @@ def _execute_single_rule( operator, table_ref, execute_fn, - ) - elif rule_type == "valid_values": - # Alias for validity — checks that column values are in the validValues list - return _check_validity( - rule_id, - selector, - severity, - description, - valid_values, - table_ref, - execute_fn, dialect, ) elif rule_type in _UNIMPLEMENTED_RULE_TYPES: @@ -528,8 +816,23 @@ def _check_completeness( operator, table_ref, execute_fn, + dialect: str = "ansi", ) -> QualityCheckResult: - sql = _completeness_sql(table_ref, selector) + columns = rule_columns(selector) + if len(columns) > 1: + return _check_completeness_columns( + rule_id, + selector, + columns, + severity, + description, + threshold, + operator, + table_ref, + execute_fn, + dialect, + ) + sql = _completeness_sql(table_ref, selector, dialect) rows = execute_fn(sql) ratio = rows[0][0] if rows and rows[0][0] is not None else 0.0 passed = _compare(ratio, threshold, operator) if threshold is not None else ratio == 1.0 @@ -549,6 +852,59 @@ def _check_completeness( ) +def _row_values(rows, width: int) -> List[Any]: + """The first result row padded to ``width`` (an empty result reads as NULLs).""" + first = list(rows[0]) if rows else [] + return [first[i] if i < len(first) else None for i in range(width)] + + +def _listed(pairs: List[tuple], fmt) -> str: + """``'a' (x), 'b' (y)`` for at most five pairs, then how many more.""" + shown = ", ".join(f"'{name}' ({fmt(value)})" for name, value in pairs[:5]) + return shown + (f" and {len(pairs) - 5} more" if len(pairs) > 5 else "") + + +def _check_completeness_columns( + rule_id, + selector, + columns: List[str], + severity, + description, + threshold, + operator, + table_ref, + execute_fn, + dialect: str, +) -> QualityCheckResult: + """A completeness rule over several columns: each must meet the bound. + + One statement measures every column. The observed value is the lowest + ratio, and the message names each column below the bound. + """ + rows = execute_fn(_completeness_columns_sql(table_ref, columns, dialect)) + ratios = [float(v) if v is not None else 0.0 for v in _row_values(rows, len(columns))] + + def meets(ratio: float) -> bool: + return _compare(ratio, threshold, operator) if threshold is not None else ratio == 1.0 + + failing = [(c, r) for c, r in zip(columns, ratios, strict=True) if not meets(r)] + return QualityCheckResult( + rule_id=rule_id, + rule_type="completeness", + selector=selector, + passed=not failing, + severity=severity, + message=( + f"{description or rule_id} — completeness is below the bound for " + f"{_listed(failing, lambda r: f'{r:.2%}')}" + if failing + else f"Completeness OK for all {len(columns)} columns" + ), + expected=f"{operator} {threshold}" if threshold is not None else "1.0", + actual=f"{min(ratios):.4f}", + ) + + def _check_uniqueness( rule_id, selector, @@ -558,8 +914,9 @@ def _check_uniqueness( operator, table_ref, execute_fn, + dialect: str = "ansi", ) -> QualityCheckResult: - sql = _uniqueness_sql(table_ref, selector) + sql = _uniqueness_sql(table_ref, selector, dialect) rows = execute_fn(sql) ratio = rows[0][0] if rows and rows[0][0] is not None else 0.0 passed = _compare(ratio, threshold, operator) if threshold is not None else ratio == 1.0 @@ -588,13 +945,44 @@ def _check_accuracy( operator, table_ref, execute_fn, + dialect: str = "ansi", ) -> QualityCheckResult: + if threshold is None: + # A bound with nothing to compare against checks nothing; reading it as + # passed whenever the column holds a value made a gate read green. + return QualityCheckResult( + rule_id=rule_id, + rule_type="accuracy", + selector=selector, + passed=False, + severity=severity, + message=( + f"Rule '{rule_id}' is an accuracy rule with no threshold, so nothing was " + "checked. Declare a threshold and an operator (>=, >, <=, <, == or !=)." + ), + expected="a threshold to compare with", + actual="not executed", + ) + columns = rule_columns(selector) + if len(columns) > 1: + return _check_accuracy_columns( + rule_id, + selector, + columns, + severity, + description, + threshold, + operator, + table_ref, + execute_fn, + dialect, + ) # A bound rule asserts something about EVERY row, so the aggregate # that decides it depends on the direction of the bound. Always # asking for MIN() made every upper bound ('<=', '<') trivially true # and every equality bound undecidable. if threshold is not None and operator in _NEGATED_OPERATORS: - sql = _accuracy_violations_sql(table_ref, selector, operator, threshold) + sql = _accuracy_violations_sql(table_ref, selector, operator, threshold, dialect) rows = execute_fn(sql) violations = rows[0][0] if rows and rows[0][0] is not None else 0 passed = int(violations) == 0 @@ -617,9 +1005,9 @@ def _check_accuracy( use_max = operator in _UPPER_BOUND_OPERATORS aggregate = "max" if use_max else "min" sql = ( - _accuracy_max_sql(table_ref, selector) + _accuracy_max_sql(table_ref, selector, dialect) if use_max - else _accuracy_min_sql(table_ref, selector) + else _accuracy_min_sql(table_ref, selector, dialect) ) rows = execute_fn(sql) bound_val = rows[0][0] if rows and rows[0][0] is not None else None @@ -649,55 +1037,159 @@ def _check_accuracy( ) +def _check_accuracy_columns( + rule_id, + selector, + columns: List[str], + severity, + description, + threshold, + operator, + table_ref, + execute_fn, + dialect: str, +) -> QualityCheckResult: + """An accuracy rule over several columns: every column must hold the bound. + + One statement measures every column, with the same aggregate the + single-column rule uses (``MIN`` for a lower bound, ``MAX`` for an upper + one, a violating-row count for ``==`` / ``!=``). A column with no values + fails as "no data", as it does on its own. + """ + if threshold is not None and operator in _NEGATED_OPERATORS: + rows = execute_fn( + _accuracy_violations_columns_sql(table_ref, columns, operator, threshold, dialect) + ) + counts = [int(v) if v is not None else 0 for v in _row_values(rows, len(columns))] + failing = [(c, n) for c, n in zip(columns, counts, strict=True) if n] + return QualityCheckResult( + rule_id=rule_id, + rule_type="accuracy", + selector=selector, + passed=not failing, + severity=severity, + message=( + f"{description or rule_id} — rows that do not satisfy {operator} {threshold} " + f"in {_listed(failing, str)}" + if failing + else f"Accuracy OK for all {len(columns)} columns" + ), + expected=f"0 rows violating {operator} {threshold}", + actual=f"{sum(counts)} violating row(s)", + ) + + aggregate = "max" if operator in _UPPER_BOUND_OPERATORS else "min" + rows = execute_fn(_accuracy_bound_columns_sql(table_ref, columns, aggregate, dialect)) + bounds = _row_values(rows, len(columns)) + empty = [c for c, v in zip(columns, bounds, strict=True) if v is None] + failing = [ + (c, v) + for c, v in zip(columns, bounds, strict=True) + if v is not None and threshold is not None and not _compare(v, threshold, operator) + ] + problems = [] + if failing: + problems.append(f"{aggregate} value is outside the bound for {_listed(failing, str)}") + if empty: + problems.append("no data in " + ", ".join(f"'{c}'" for c in empty[:5])) + present = [v for v in bounds if v is not None] + worst = None + if present: + try: + worst = (max if aggregate == "max" else min)(float(v) for v in present) + except (TypeError, ValueError): + worst = None + return QualityCheckResult( + rule_id=rule_id, + rule_type="accuracy", + selector=selector, + passed=not problems, + severity=severity, + message=( + f"{description or rule_id} — " + "; ".join(problems) + if problems + else f"Accuracy OK for all {len(columns)} columns" + ), + expected=f"{operator} {threshold}" if threshold is not None else "pass", + actual=None if worst is None else f"{worst:g}", + ) + + def _check_validity( rule_id, selector, severity, description, - valid_values, + domain: ValueDomain, table_ref, execute_fn, dialect: str = "ansi", + *, + rule_type: str = "validity", + resolve_reference: Optional[ReferenceResolver] = None, ) -> QualityCheckResult: - if not valid_values: + col = _qi(selector, dialect) + violations: List[str] = [] + kinds: List[str] = [] + if domain.values: + # The literals are contract-supplied, so the quoting must hold on the + # *target* dialect (a value ending in a backslash must not end the + # literal early and append live SQL to this predicate). + escaped = ", ".join(quote_string_literal_for(dialect, str(v)) for v in domain.values) + violations.append(f"{col} NOT IN ({escaped})") + kinds.append("invalid values") + if domain.pattern: + violations.append(_pattern_mismatch(col, domain.pattern, dialect)) + kinds.append(f"values not matching {domain.pattern}") + if domain.bounds: + violations.append(_range_violation(col, domain.bounds)) + kinds.append("values outside " + " and ".join(f"{op} {n:g}" for op, n in domain.bounds)) + if not violations and domain.reference is not None: + return _check_reference( + rule_id, + selector, + severity, + description, + domain.reference, + resolve_reference, + table_ref, + execute_fn, + dialect, + rule_type, + ) + if violations: + invalid = violations[0] if len(violations) == 1 else f"({' OR '.join(violations)})" + sql = ( + f"SELECT COUNT(*) AS invalid_count FROM {table_ref} " + f"WHERE {col} IS NOT NULL AND {invalid}" + ) + expected = "0 " + kinds[0] if len(kinds) == 1 else "0 " + "; 0 ".join(kinds) + else: # A misconfigured gate is a failed gate. Forcing 'warning' here # overrode the author's declared severity, so a rule declared # `severity: error` that checks nothing at all exited 0. return QualityCheckResult( rule_id=rule_id, - rule_type="validity", + rule_type=rule_type, selector=selector, passed=False, severity=severity, message=( f"Rule '{rule_id}' is a valid_values rule but declares no allowed " "values, so nothing was checked. Declare them in the description " - f"(\"{selector or 'COLUMN'} valid values: A, B, C.\") or as a " - "'validValues' list." + f"(\"{selector or 'COLUMN'} valid values: A, B, C.\"), as an enum, regex " + "or range validationRule on the column, as the column's " + "labels.references, or as a 'validValues' list." ), expected="a non-empty list of allowed values", actual="no allowed values declared", ) - col = _validate_ident(selector) - # Build SQL with quoted string literals for valid values. The literals are - # contract-supplied, so the quoting must hold on the *target* dialect: - # Snowflake/BigQuery honour ``\`` escapes inside ``'...'``, and a value - # ending in a backslash would otherwise terminate the literal early and - # append live SQL to this predicate (silently flipping the check to pass). - backslash_escapes = dialect_uses_backslash_escapes(dialect) - escaped = ", ".join( - quote_string_literal(str(v), backslash_escapes=backslash_escapes) for v in valid_values - ) - sql = ( - f"SELECT COUNT(*) AS invalid_count FROM {table_ref} " - f'WHERE "{col}" IS NOT NULL AND "{col}" NOT IN ({escaped})' - ) rows = execute_fn(sql) invalid_count = rows[0][0] if rows and rows[0][0] is not None else 0 - passed = invalid_count == 0 - return QualityCheckResult( + passed = int(invalid_count) == 0 + result = QualityCheckResult( rule_id=rule_id, - rule_type="validity", + rule_type=rule_type, selector=selector, passed=passed, severity=severity, @@ -706,9 +1198,164 @@ def _check_validity( if not passed else f"Validity OK for '{selector}'" ), - expected="0 invalid values", + expected=expected, actual=f"{invalid_count} invalid value(s)", ) + if domain.reference is None: + return result + # The column also references another product: that must hold too. + reference = _check_reference( + rule_id, + selector, + severity, + description, + domain.reference, + resolve_reference, + table_ref, + execute_fn, + dialect, + rule_type, + ) + failed = [r.message for r in (result, reference) if not r.passed] + return QualityCheckResult( + rule_id=rule_id, + rule_type=rule_type, + selector=selector, + passed=not failed, + severity=severity, + message="; ".join(failed) if failed else result.message, + expected=f"{result.expected}; {reference.expected}", + actual=( + "not executed" + if reference.actual == "not executed" + else f"{result.actual}; {reference.actual}" + ), + ) + + +_NOT_RUN = ("not executed", "no allowed values declared") + + +def _check_validity_columns( + rule: Dict[str, Any], + columns: Any, + table_ref, + execute_fn, + dialect: str, + resolve_reference: Optional[ReferenceResolver], +) -> QualityCheckResult: + """A valid_values rule over several columns: each column against its own + declared values (the rule's list, else each column's validationRule or + reference). Passes when every column was checked and holds no invalid value.""" + rule_id = rule.get("id", "unnamed") + rule_type = rule.get("type", "").lower() + selector = rule.get("selector", "") + severity = rule.get("severity", "warning") + names = rule_columns(selector) + parts = [ + _check_validity( + rule_id, + name, + severity, + rule.get("description", ""), + value_domain(rule, columns, name), + table_ref, + execute_fn, + dialect, + rule_type=rule_type, + resolve_reference=resolve_reference, + ) + for name in names + ] + skipped = [n for n, r in zip(names, parts, strict=True) if r.actual in _NOT_RUN] + failing = [ + (n, r.actual) + for n, r in zip(names, parts, strict=True) + if not r.passed and n not in skipped + ] + problems = [] + if failing: + problems.append("invalid values in " + _listed(failing, str)) + if skipped: + problems.append("nothing checked for " + ", ".join(f"'{n}'" for n in skipped[:5])) + total = 0 + for result in parts: + head = str(result.actual or "").split(" ", 1)[0] + total += int(head) if head.isdigit() else 0 + return QualityCheckResult( + rule_id=rule_id, + rule_type=rule_type, + selector=selector, + passed=not problems, + severity=severity, + message=( + f"{rule.get('description') or rule_id} — " + "; ".join(problems) + if problems + else f"Validity OK for all {len(names)} columns" + ), + expected="0 invalid values in every column", + actual="not executed" if skipped else f"{total} invalid value(s)", + ) + + +def _check_reference( + rule_id, + selector, + severity, + description, + reference: Tuple[str, str], + resolve_reference: Optional[ReferenceResolver], + table_ref, + execute_fn, + dialect: str, + rule_type: str, +) -> QualityCheckResult: + """Every non-null value of ``selector`` exists in the referenced column.""" + product, column = reference + target = f"{product}.{column}" + expected = f"every value in {target}" + + def not_run(why: str) -> QualityCheckResult: + return QualityCheckResult( + rule_id=rule_id, + rule_type=rule_type, + selector=selector, + passed=False, + severity=severity, + message=f"Rule '{rule_id}' checks '{selector}' against {target}, but {why}, " + "so nothing was checked", + expected=expected, + actual="not executed", + ) + + if resolve_reference is None: + return not_run("this platform cannot look up another product's table") + try: + parent = resolve_reference(product, column) + except LookupError as exc: + return not_run(str(exc)) + col, pcol = _qi(selector, dialect), _qi(column, dialect) + sql = ( + f"SELECT COUNT(*) AS missing_count FROM {table_ref} AS c " + f"WHERE c.{col} IS NOT NULL AND NOT EXISTS " + f"(SELECT 1 FROM {parent} AS p WHERE p.{pcol} = c.{col})" + ) + rows = execute_fn(sql) + missing = int(rows[0][0]) if rows and rows[0][0] is not None else 0 + return QualityCheckResult( + rule_id=rule_id, + rule_type=rule_type, + selector=selector, + passed=missing == 0, + severity=severity, + message=( + f"{description or rule_id} — {missing} value(s) of '{selector}' are not in {target}" + if missing + else f"Every '{selector}' value is in {target}" + ), + expected=f"0 values missing from {target}", + actual=f"{missing} value(s) missing", + ) def _check_freshness( @@ -790,6 +1437,7 @@ def _check_anomaly_detection( operator, table_ref, execute_fn, + dialect: str = "ansi", ) -> QualityCheckResult: """Anomaly detection via row-count check (selector '*') or MIN/MAX check for a column.""" if selector == "*" or selector == "": @@ -817,5 +1465,13 @@ def _check_anomaly_detection( # Column-level anomaly: delegate to the bound check, which picks # MIN or MAX (or a violating-row count) from the operator. return _check_accuracy( - rule_id, selector, severity, description, threshold, operator, table_ref, execute_fn + rule_id, + selector, + severity, + description, + threshold, + operator, + table_ref, + execute_fn, + dialect, ) diff --git a/fluid_build/providers/snowflake_validation.py b/fluid_build/providers/snowflake_validation.py index 5a4c331f..f69fd914 100644 --- a/fluid_build/providers/snowflake_validation.py +++ b/fluid_build/providers/snowflake_validation.py @@ -17,6 +17,7 @@ from fluid_build.providers._sql_safety import validate_ident from fluid_build.providers.quality_engine import ( execute_quality_checks, + expose_columns, quality_results_to_issues, ) from fluid_build.providers.validation_provider import ( @@ -454,5 +455,6 @@ def run_quality_checks( table_ref=table_ref, execute_fn=conn.execute, dialect="snowflake", + columns=expose_columns(resource_spec), ) return quality_results_to_issues(results) diff --git a/fluid_build/providers/validation_provider.py b/fluid_build/providers/validation_provider.py index 1bbe1892..7a909e5b 100644 --- a/fluid_build/providers/validation_provider.py +++ b/fluid_build/providers/validation_provider.py @@ -322,3 +322,16 @@ def run_quality_checks( path="contract.dq.rules", ) ] + + def run_quality_results( + self, + resource_spec: Dict[str, Any], + rules: List[Dict[str, Any]], + ) -> Optional[List[Any]]: + """Every rule's ``QualityCheckResult``, the passing ones included. + + ``None`` (the default) means this provider reports failures only, + through :meth:`run_quality_checks`. A provider that returns the full + list lets a report record what each passing rule measured, too. + """ + return None diff --git a/tests/cli/test_contract_validation_schema_rules.py b/tests/cli/test_contract_validation_schema_rules.py new file mode 100644 index 00000000..629acd2c --- /dev/null +++ b/tests/cli/test_contract_validation_schema_rules.py @@ -0,0 +1,410 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""ContractValidator: Databricks provider wiring, ``schema`` dq rules decided from the +live column comparison, and CheckOutcomes that carry rule id/type/expose id.""" + +from __future__ import annotations + +from datetime import datetime + +import pytest + +from fluid_build.cli import contract_validation as cv +from fluid_build.providers.quality_engine import QualityCheckResult +from fluid_build.providers.validation_provider import ( + FieldSchema, + ResourceSchema, + ResourceType, + ValidationProvider, + ValidationResult, +) + +CONTRACT_COLUMNS = [ + {"name": "a", "type": "string"}, + {"name": "b", "type": "int"}, +] + + +class FakeProvider(ValidationProvider): + """Deterministic provider: canned live schema, canned quality results.""" + + def __init__(self, live=None, results="unset", issues=None, schema_error=None): + super().__init__({}) + self.live = live + self.results = results + self.issues = issues or [] + self.schema_error = schema_error + self.quality_calls = [] + + @property + def provider_name(self): + return "databricks" + + def validate_connection(self): + return True + + def get_resource_schema(self, resource_spec): + if self.schema_error: + raise self.schema_error + return self.live + + def validate_resource(self, contract_spec, actual_schema): + if actual_schema is None: + from fluid_build.providers.validation_provider import ValidationIssue + + return ValidationResult( + "x", + False, + [ + ValidationIssue( + severity="error", + category="missing_resource", + message="Table does not exist", + path="p", + ) + ], + None, + ) + expected = [FieldSchema(c["name"], c["type"], "NULLABLE") for c in contract_spec["schema"]] + issues = self.compare_schemas(expected, actual_schema.fields) + return ValidationResult("x", not issues, issues, actual_schema) + + def normalize_type(self, provider_type): + return provider_type.lower() + + def run_quality_checks(self, resource_spec, rules): + self.quality_calls.append(("checks", rules)) + return self.issues + + def run_quality_results(self, resource_spec, rules): + self.quality_calls.append(("results", rules)) + return None if self.results == "unset" else self.results + + +def live(*cols): + return ResourceSchema( + ResourceType.TABLE, "c.s.t", [FieldSchema(n, t, "NULLABLE") for n, t in cols], row_count=1 + ) + + +def make_validator(provider, tmp_path): + v = cv.ContractValidator( + tmp_path / "contract.fluid.yaml", + provider_name="databricks", + use_cache=False, + track_history=False, + ) + v.report = cv.ValidationReport( + contract_path="x", + contract_id="id", + contract_version="1", + validation_time=datetime.now(), + duration=0.0, + ) + v.validation_provider = provider + return v + + +def expose(rules): + return { + "id": "e1", + "type": "table", + "binding": { + "platform": "databricks", + "location": {"database": "c", "schema": "s", "table": "t"}, + }, + "schema": CONTRACT_COLUMNS, + "contract": {"dq": {"rules": rules}}, + } + + +SCHEMA_RULE = {"id": "conforms", "type": "schema", "selector": "*", "severity": "error"} + + +def quality_issues(v): + return [i for i in v.report.issues if i.category == "quality"] + + +def check(v, name): + (c,) = [c for c in v.report.checks if c.name == name] + return c + + +# ------------------------------------------------------------- provider wiring + + +def detect(tmp_path, monkeypatch, available): + v = cv.ContractValidator(tmp_path / "c.yaml", use_cache=False, track_history=False) + v.contract = {"exposes": [{"binding": {"platform": "databricks"}}]} + v.report = cv.ValidationReport("x", "id", "1", datetime.now(), 0.0) + monkeypatch.setattr(cv, "DATABRICKS_VALIDATION_AVAILABLE", available) + v._detect_and_validate_provider() + return v + + +def test_databricks_provider_is_built_when_sdk_available(tmp_path, monkeypatch): + from fluid_build.providers.databricks_validation import DatabricksValidationProvider + + v = detect(tmp_path, monkeypatch, True) + assert isinstance(v.validation_provider, DatabricksValidationProvider) + assert [i for i in v.report.issues if i.category == "binding"] == [] + + +def test_databricks_without_sdk_is_binding_error_naming_sdk(tmp_path, monkeypatch): + v = detect(tmp_path, monkeypatch, False) + assert v.validation_provider is None + (issue,) = v.report.issues + assert issue.severity == "error" and issue.category == "binding" + assert "databricks-sdk" in issue.message + + +# --------------------------------------------------- schema rules are not sent + + +def test_schema_rules_are_not_sent_to_the_provider(tmp_path): + other = {"id": "c1", "type": "completeness", "selector": "a"} + p = FakeProvider(live=live(("a", "string"), ("b", "int")), results=[]) + v = make_validator(p, tmp_path) + v._validate_single_expose(expose([SCHEMA_RULE, other]), 0) + assert p.quality_calls and all( + [r["id"] for r in rules] == ["c1"] for _, rules in p.quality_calls + ) + + +def test_only_schema_rules_means_no_provider_quality_call(tmp_path): + p = FakeProvider(live=live(("a", "string"), ("b", "int"))) + v = make_validator(p, tmp_path) + v._validate_single_expose(expose([SCHEMA_RULE]), 0) + assert p.quality_calls == [] + + +def test_schema_rule_passes_when_columns_match(tmp_path): + v = make_validator(FakeProvider(live=live(("a", "string"), ("b", "int"))), tmp_path) + v._validate_single_expose(expose([SCHEMA_RULE]), 0) + c = check(v, "e1.conforms") + assert c.passed and c.rule_type == "schema" and c.rule_id == "conforms" and c.expose_id == "e1" + assert c.actual == "all contract columns match" and quality_issues(v) == [] + + +def test_schema_rule_ignores_extra_live_column(tmp_path): + v = make_validator( + FakeProvider(live=live(("a", "string"), ("b", "int"), ("extra", "string"))), tmp_path + ) + v._validate_single_expose(expose([SCHEMA_RULE]), 0) + assert check(v, "e1.conforms").passed + + +def test_schema_rule_fails_on_missing_column_at_rule_severity(tmp_path): + rule = {**SCHEMA_RULE, "severity": "warning"} + v = make_validator(FakeProvider(live=live(("a", "string"))), tmp_path) + v._validate_single_expose(expose([rule]), 0) + c = check(v, "e1.conforms") + assert not c.passed and c.severity == "warning" + assert c.actual == "1 column(s) missing, 0 type change(s)" + (issue,) = quality_issues(v) + assert issue.severity == "warning" and issue.path.endswith("dq.rules.conforms") + + +def test_schema_rule_fails_on_type_change(tmp_path): + v = make_validator(FakeProvider(live=live(("a", "string"), ("b", "bigint"))), tmp_path) + v._validate_single_expose(expose([SCHEMA_RULE]), 0) + c = check(v, "e1.conforms") + assert not c.passed and c.actual == "0 column(s) missing, 1 type change(s)" + assert quality_issues(v)[0].severity == "error" + + +def test_schema_rule_fails_when_table_missing(tmp_path): + v = make_validator(FakeProvider(live=None), tmp_path) + v._validate_single_expose(expose([SCHEMA_RULE]), 0) + c = check(v, "e1.conforms") + assert not c.passed and c.actual == "table does not exist" + + +def test_schema_rule_not_executed_when_comparison_did_not_run(tmp_path): + p = FakeProvider(schema_error=RuntimeError("no network")) + v = make_validator(p, tmp_path) + v._validate_single_expose(expose([SCHEMA_RULE]), 0) + c = check(v, "e1.conforms") + assert not c.passed and c.actual == "not executed" + + +def test_schema_rule_not_decided_without_check_data(tmp_path): + v = make_validator(FakeProvider(live=live(("a", "string"), ("b", "int"))), tmp_path) + v.check_data = False + v._validate_single_expose(expose([SCHEMA_RULE]), 0) + assert v.report.checks == [] + + +# ------------------------------------------------- outcomes for passing rules + + +def test_provider_results_list_gives_passing_outcomes_expected_and_actual(tmp_path): + results = [ + QualityCheckResult("c1", "completeness", "a", True, "error", "ok", "1.0", "1.0000"), + QualityCheckResult("u1", "uniqueness", "a", False, "warning", "dupes", "1.0", "0.5000"), + ] + rules = [ + {"id": "c1", "type": "completeness", "selector": "a"}, + {"id": "u1", "type": "uniqueness", "selector": "a", "severity": "warning"}, + ] + p = FakeProvider(live=live(("a", "string"), ("b", "int")), results=results) + v = make_validator(p, tmp_path) + v._validate_single_expose(expose(rules), 0) + ok = check(v, "e1.c1") + assert ok.passed and (ok.expected, ok.actual) == ("1.0", "1.0000") + assert (ok.rule_id, ok.rule_type, ok.expose_id) == ("c1", "completeness", "e1") + bad = check(v, "e1.u1") + assert not bad.passed and bad.actual == "0.5000" and bad.severity == "warning" + assert [i.path for i in quality_issues(v)] == ["exposes[0].contract.dq.rules.u1"] + + +def test_results_none_falls_back_to_run_quality_checks_and_still_tags_outcomes(tmp_path): + rules = [{"id": "c1", "type": "completeness", "selector": "a"}] + p = FakeProvider(live=live(("a", "string"), ("b", "int")), results="unset") + v = make_validator(p, tmp_path) + v._validate_single_expose(expose(rules), 0) + assert [k for k, _ in p.quality_calls] == ["results", "checks"] + c = check(v, "e1.c1") + assert c.passed and (c.rule_id, c.rule_type, c.expose_id) == ("c1", "completeness", "e1") + + +def test_provider_exception_becomes_issue_not_crash(tmp_path): + class Boom(FakeProvider): + def run_quality_results(self, resource_spec, rules): + raise RuntimeError("warehouse down") + + v = make_validator(Boom(live=live(("a", "string"), ("b", "int"))), tmp_path) + v._validate_single_expose(expose([{"id": "c1", "type": "completeness", "selector": "a"}]), 0) + assert any("warehouse down" in i.message for i in v.report.issues) + + +@pytest.mark.parametrize("value", ["Schema", " schema "]) +def test_rule_kind_normalises_case_and_space(value): + assert cv._rule_kind({"type": value}) == "schema" + assert cv._rule_kind("nope") == "" and cv._rule_kind({}) == "" + + +# ------------------------------------------------ gcp: a failed read is not a pass + + +def gcp_expose(rules, **props): + location = {"properties": {"project": "p", "dataset": "d", "table": "t", **props}} + return {**expose(rules), "binding": {"platform": "gcp", "location": location}} + + +def gcp_validator(provider, tmp_path): + v = make_validator(provider, tmp_path) + v.provider_name = "gcp" + return v + + +def test_bigquery_read_failure_leaves_the_schema_rule_not_executed(tmp_path): + p = FakeProvider(schema_error=RuntimeError("bigquery down")) + v = gcp_validator(p, tmp_path) + v._validate_single_expose(gcp_expose([SCHEMA_RULE]), 0) + c = check(v, "e1.conforms") + assert not c.passed and c.actual == "not executed" + assert "Schema conforms" not in (c.message or "") + + +def test_bigquery_resource_returns_false_on_each_early_exit(tmp_path): + v = gcp_validator(FakeProvider(live=live(("a", "string"))), tmp_path) + assert v._validate_bigquery_resource(gcp_expose([]), "p", {"project": "p"}) is False + v.validation_provider = None + full = {"project": "p", "dataset": "d", "table": "t"} + assert v._validate_bigquery_resource(gcp_expose([]), "p", full) is False + assert v._validate_against_actual_resource(gcp_expose([]), "p") is False + + +def test_bigquery_resource_returns_true_when_columns_were_compared(tmp_path): + v = gcp_validator(FakeProvider(live=live(("a", "string"), ("b", "int"))), tmp_path) + assert v._validate_against_actual_resource(gcp_expose([]), "p") is True + + +# ------------------------------------------------------- duplicate rule ids + + +def test_two_rules_sharing_an_id_are_paired_with_their_own_results_in_order(tmp_path): + results = [ + QualityCheckResult("d", "completeness", "a", False, "error", "bad", "1.0", "0.5"), + QualityCheckResult("d", "completeness", "b", True, "error", "ok", "1.0", "1.0"), + ] + rules = [ + {"id": "d", "type": "completeness", "selector": "a"}, + {"id": "d", "type": "completeness", "selector": "b"}, + ] + p = FakeProvider(live=live(("a", "string"), ("b", "int")), results=results) + v = make_validator(p, tmp_path) + v._validate_single_expose(expose(rules), 0) + outcomes = [c for c in v.report.checks if c.name == "e1.d"] + assert [c.passed for c in outcomes] == [False, True] + assert [c.actual for c in outcomes] == ["0.5", "1.0"] + + +# --------------------------------------------- the schema cache and Databricks + + +class CountingProvider(FakeProvider): + """Row count 0 on the first read, 5 on the second; empty tables are an issue.""" + + def __init__(self): + super().__init__() + self.reads = 0 + + def get_resource_schema(self, resource_spec): + self.reads += 1 + count = 0 if self.reads == 1 else 5 + return ResourceSchema( + ResourceType.TABLE, + "c.s.t", + [FieldSchema("a", "string", "NULLABLE"), FieldSchema("b", "int", "NULLABLE")], + row_count=count, + ) + + def validate_resource(self, contract_spec, actual_schema): + from fluid_build.providers.validation_provider import ValidationIssue + + issues = [] + if actual_schema.row_count == 0: + issues.append( + ValidationIssue( + severity="warning", category="empty_table", message="empty", path="p" + ) + ) + return ValidationResult("x", True, issues, actual_schema) + + +def cached_validator(provider_name, tmp_path, monkeypatch): + from fluid_build.providers.validation_cache import ValidationCache + + monkeypatch.setenv("HOME", str(tmp_path / "home")) + provider = CountingProvider() + v = make_validator(provider, tmp_path) + v.provider_name = provider_name + v.use_cache = True + v.cache = ValidationCache(cache_dir=tmp_path / "cache", ttl=3600) + return v, provider + + +def empty_table_issues(v): + return [i for i in v.report.issues if i.category == "empty_table"] + + +def test_databricks_never_reads_the_schema_cache(tmp_path, monkeypatch): + v, provider = cached_validator("databricks", tmp_path, monkeypatch) + e = expose([]) + v._validate_generic_resource(e, "p") + assert len(empty_table_issues(v)) == 1 + v._validate_generic_resource(e, "p") + assert provider.reads == 2 + assert len(empty_table_issues(v)) == 1 # the second read saw 5 rows + assert list((tmp_path / "cache").glob("*.json")) == [] + + +def test_snowflake_still_uses_the_schema_cache(tmp_path, monkeypatch): + v, provider = cached_validator("snowflake", tmp_path, monkeypatch) + e = expose([]) + v._validate_generic_resource(e, "p") + v._validate_generic_resource(e, "p") + assert provider.reads == 1 diff --git a/tests/cli/test_fluid_test_dashboard.py b/tests/cli/test_fluid_test_dashboard.py new file mode 100644 index 00000000..5dc7a6a5 --- /dev/null +++ b/tests/cli/test_fluid_test_dashboard.py @@ -0,0 +1,438 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``fluid test --results-dashboard``: path parsing, the dashboard definition's structural +invariants, and ensure_dashboard's create/update/unchanged flows against a fake client.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest + +from fluid_build.cli import _test_dashboard_databricks as dash +from fluid_build.cli._common import CLIError + +TABLE = ("main", "dq", "results") +REF = "`main`.`dq`.`results`" +PATH = "/Workspace/Shared/fluid/Data quality" +FAKE_TOKEN = "fake-test-token-0000" # pragma: allowlist secret + + +class NotFound(Exception): + status_code = 404 + + +# ------------------------------------------------------------ path parsing + + +def test_parse_dashboard_path_splits_folder_and_name(): + assert dash.parse_dashboard_path(PATH) == ("/Workspace/Shared/fluid", "Data quality") + + +def test_parse_dashboard_path_strips_lvdash_suffix(): + assert dash.parse_dashboard_path(PATH + ".lvdash.json") == ( + "/Workspace/Shared/fluid", + "Data quality", + ) + + +def test_parse_dashboard_path_accepts_a_name_directly_under_workspace(): + # Pins current behaviour: the leading empty segment counts toward the minimum of 3. + assert dash.parse_dashboard_path("/Workspace/x") == ("/Workspace", "x") + + +@pytest.mark.parametrize( + "bad", + [ + "Workspace/Shared/x", + "/Workspace/../x", + "/Workspace//x", + "/Workspace/Shared/", + "/Workspace/./x", + "/Workspace/Shared/a\x00b", + "/Workspace/Shared/a\nb", + "/x", + "", + None, + ], +) +def test_parse_dashboard_path_rejects(bad): + with pytest.raises(CLIError) as ei: + dash.parse_dashboard_path(bad) + assert ei.value.exit_code == 2 and ei.value.event == "test_results_dashboard_invalid" + + +# ------------------------------------------------------ definition shape + + +@pytest.fixture(scope="module") +def definition(): + return dash.dashboard_definition(TABLE) + + +def widgets(defn): + return [item for page in defn["pages"] for item in page["layout"]] + + +def field_names(widget): + return {f["name"] for q in widget.get("queries", []) for f in q["query"]["fields"]} + + +def encoded_fields(node): + found = [] + if isinstance(node, dict): + if "fieldName" in node: + found.append(node["fieldName"]) + for v in node.values(): + found.extend(encoded_fields(v)) + elif isinstance(node, list): + for v in node: + found.extend(encoded_fields(v)) + return found + + +def test_definition_is_json_serialisable_and_deterministic(definition): + assert json.dumps(definition) == json.dumps(dash.dashboard_definition(TABLE)) + + +def test_every_query_points_at_a_dataset(definition): + names = {d["name"] for d in definition["datasets"]} + used = { + q["query"]["datasetName"] + for w in widgets(definition) + for q in w["widget"].get("queries", []) + } + assert used and used <= names + + +def test_every_encoding_field_is_a_query_field(definition): + for item in widgets(definition): + w = item["widget"] + if "queries" not in w: + continue + assert set(encoded_fields(w["spec"]["encodings"])) <= field_names(w), w["name"] + + +def test_positions_fit_six_column_grid_without_overlap(definition): + boxes = [i["position"] for i in widgets(definition)] + for p in boxes: + assert p["x"] >= 0 and p["x"] + p["width"] <= 6 + + def overlap(a, b): + return ( + a["x"] < b["x"] + b["width"] + and b["x"] < a["x"] + a["width"] + and a["y"] < b["y"] + b["height"] + and b["y"] < a["y"] + a["height"] + ) + + for i, a in enumerate(boxes): + for b in boxes[i + 1 :]: + assert not overlap(a, b), (a, b) + + +def test_spec_versions_by_widget_type(definition): + seen = {} + for item in widgets(definition): + spec = item["widget"].get("spec") + if spec: + seen.setdefault(spec["widgetType"], set()).add(spec["version"]) + assert seen == {"counter": {2}, "bar": {3}, "line": {3}, "table": {1}} + + +def test_datasets_read_the_backticked_table(definition): + for d in definition["datasets"]: + assert REF in "".join(d["queryLines"]), d["name"] + + +def test_structure_filter_on_rule_datasets_but_not_on_failing_list(definition): + sql = {d["name"]: "".join(d["queryLines"]) for d in definition["datasets"]} + for name in ("fluid_dq_overall", "fluid_dq_products", "fluid_dq_trend"): + assert "COALESCE(rule_type, '') <> 'structure'" in sql[name] + # last_run's CTE names 'structure'; the rows listed (after FROM latest) are not filtered. + assert "structure" not in sql["fluid_dq_failing"].split("FROM latest", 1)[1] + assert "WHERE NOT passed" in sql["fluid_dq_failing"] + + +def test_definition_names_the_table_in_the_text_box(definition): + box = widgets(definition)[0]["widget"]["multilineTextboxSpec"]["lines"] + assert "main.dq.results" in "".join(box) + + +# ---------------------------------------------------------- ensure_dashboard + + +class FakeApi: + def __init__(self, status=None, dashboard=None, published=None, create=None): + self.status = status if status is not None else NotFound("nope") + self.dashboard = dashboard + self.published = published if published is not None else {"revision_create_time": "t"} + self.create = create if create is not None else {"dashboard_id": "dash-1"} + self.calls = [] + + def do(self, method, path, query=None, body=None, **kw): + self.calls.append((method, path, query, body)) + if (method, path) == ("GET", dash.GET_STATUS_PATH): + result = self.status + elif (method, path) == ("POST", dash.MKDIRS_PATH): + result = {} + elif (method, path) == ("POST", dash.DASHBOARDS_PATH): + result = self.create + elif method == "GET" and path.endswith("/published"): + result = self.published + elif method == "GET" and path.startswith(dash.DASHBOARDS_PATH + "/"): + result = self.dashboard + elif method in ("PATCH", "POST"): + result = {} + else: + raise AssertionError(f"unexpected {method} {path}") + if isinstance(result, Exception): + raise result + return result + + def verbs(self): + return [(c[0], c[1]) for c in self.calls] + + +class FakeClient: + def __init__(self, api, host="https://adb-1.azuredatabricks.net/"): + self.api_client = api + self.config = SimpleNamespace(host=host) + + +def ensure(api, **kw): + return dash.ensure_dashboard( + PATH, TABLE, client_factory=lambda: FakeClient(api), warehouse_id="wh", **kw + ) + + +def existing_status(object_type="DASHBOARD"): + return { + "object_type": object_type, + "path": "/Workspace/Shared/fluid/Data quality.lvdash.json", + "resource_id": "dash-1", + } + + +def live_dashboard(serialized=None, warehouse="wh"): + return { + "create_time": "t", + "dashboard_id": "dash-1", + "display_name": "Data quality", + "etag": "etag-7", + "lifecycle_state": "ACTIVE", + "parent_path": "/Workspace/Shared/fluid", + "path": "/Workspace/Shared/fluid/Data quality.lvdash.json", + "serialized_dashboard": ( + serialized if serialized is not None else json.dumps(dash.dashboard_definition(TABLE)) + ), + "update_time": "t", + "warehouse_id": warehouse, + } + + +def test_missing_dashboard_is_created_and_published(capsys): + api = FakeApi() + result = ensure(api) + assert result["action"] == "created" and result["dashboard_id"] == "dash-1" + assert api.verbs() == [ + ("GET", dash.GET_STATUS_PATH), + ("POST", dash.MKDIRS_PATH), + ("POST", dash.DASHBOARDS_PATH), + ("POST", f"{dash.DASHBOARDS_PATH}/dash-1/published"), + ] + assert api.calls[0][2] == {"path": "/Workspace/Shared/fluid/Data quality.lvdash.json"} + assert api.calls[1][3] == {"path": "/Workspace/Shared/fluid"} + create = api.calls[2][3] + assert create["display_name"] == "Data quality" + assert create["parent_path"] == "/Workspace/Shared/fluid" and create["warehouse_id"] == "wh" + assert json.loads(create["serialized_dashboard"]) == dash.dashboard_definition(TABLE) + assert api.calls[3][3] == {"embed_credentials": False, "warehouse_id": "wh"} + err = capsys.readouterr() + assert err.out == "" and "Dashboard created" in err.err + assert result["url"] == "https://adb-1.azuredatabricks.net/dashboardsv3/dash-1/published" + + +def test_same_definition_and_published_is_left_alone(): + api = FakeApi(status=existing_status(), dashboard=live_dashboard()) + result = ensure(api) + assert result["action"] == "unchanged" + assert [v[0] for v in api.verbs()] == ["GET", "GET", "GET"] + assert not any(v[0] in ("PATCH", "POST") for v in api.verbs()) + + +def test_unchanged_but_never_published_publishes_only(): + api = FakeApi(status=existing_status(), dashboard=live_dashboard(), published=NotFound("x")) + result = ensure(api) + assert result["action"] == "unchanged" + assert api.verbs()[-1] == ("POST", f"{dash.DASHBOARDS_PATH}/dash-1/published") + assert "PATCH" not in [v[0] for v in api.verbs()] + + +def test_different_definition_is_patched_with_etag_then_published(): + api = FakeApi( + status=existing_status(), + dashboard=live_dashboard(serialized='{"datasets": [], "pages": []}'), + ) + result = ensure(api) + assert result["action"] == "updated" + verbs = api.verbs() + assert verbs[-2:] == [ + ("PATCH", f"{dash.DASHBOARDS_PATH}/dash-1"), + ("POST", f"{dash.DASHBOARDS_PATH}/dash-1/published"), + ] + patch = [c for c in api.calls if c[0] == "PATCH"][0][3] + assert patch["etag"] == "etag-7" and patch["warehouse_id"] == "wh" + assert json.loads(patch["serialized_dashboard"]) == dash.dashboard_definition(TABLE) + + +def test_different_warehouse_counts_as_changed(): + api = FakeApi(status=existing_status(), dashboard=live_dashboard(warehouse="other")) + assert ensure(api)["action"] == "updated" + + +def test_unparseable_stored_definition_is_replaced(): + api = FakeApi(status=existing_status(), dashboard=live_dashboard(serialized="not json")) + assert ensure(api)["action"] == "updated" + + +def test_create_race_already_exists_is_treated_as_existing(): + class Api(FakeApi): + def __init__(self): + super().__init__(dashboard=live_dashboard()) + self.lookups = 0 + + def do(self, method, path, query=None, body=None, **kw): + if (method, path) == ("GET", dash.GET_STATUS_PATH): + self.lookups += 1 + self.calls.append((method, path, query, body)) + if self.lookups == 1: + raise NotFound("missing") + return existing_status() + if (method, path) == ("POST", dash.DASHBOARDS_PATH): + self.calls.append((method, path, query, body)) + raise RuntimeError("RESOURCE_ALREADY_EXISTS: dashboard exists") + return super().do(method, path, query=query, body=body) + + api = Api() + result = ensure(api) + assert api.lookups == 2 and result["dashboard_id"] == "dash-1" + assert result["action"] == "unchanged" + + +def test_already_exists_but_refind_fails_surfaces_error(): + api = FakeApi(create=RuntimeError("RESOURCE_ALREADY_EXISTS: x")) + with pytest.raises(CLIError) as ei: + ensure(api) + assert ei.value.event == "test_results_dashboard_failed" + + +def test_notebook_at_the_path_is_refused(): + api = FakeApi(status=existing_status("NOTEBOOK")) + with pytest.raises(CLIError) as ei: + ensure(api) + assert ei.value.event == "test_results_dashboard_path_taken" + assert ei.value.context["found"] == "NOTEBOOK" + assert [v[0] for v in api.verbs()] == ["GET"] + + +def test_other_failure_is_exit_1_with_scrubbed_message(monkeypatch): + monkeypatch.setenv("DATABRICKS_TOKEN", FAKE_TOKEN) + api = FakeApi(create=PermissionError(f"denied for token {FAKE_TOKEN}")) + with pytest.raises(CLIError) as ei: + ensure(api) + assert ei.value.exit_code == 1 and ei.value.event == "test_results_dashboard_failed" + error = ei.value.context["error"] + assert error.startswith("PermissionError:") and FAKE_TOKEN not in error + + +def test_get_status_non_404_failure_is_reported(): + api = FakeApi(status=RuntimeError("503 unavailable")) + with pytest.raises(CLIError) as ei: + ensure(api) + assert ei.value.event == "test_results_dashboard_failed" and "503" in ei.value.context["error"] + + +def test_invalid_path_is_rejected_before_any_call(): + api = FakeApi() + with pytest.raises(CLIError) as ei: + dash.ensure_dashboard( + "relative", TABLE, client_factory=lambda: FakeClient(api), warehouse_id="w" + ) + assert ei.value.exit_code == 2 and api.calls == [] + + +def test_no_host_means_no_url(): + api = FakeApi() + result = dash.ensure_dashboard( + PATH, TABLE, client_factory=lambda: FakeClient(api, host=""), warehouse_id="wh" + ) + assert result["url"] is None + + +def test_dashboard_without_the_sdk_names_the_extra(): + def no_sdk(): + raise ModuleNotFoundError("No module named 'databricks'") + + with pytest.raises(CLIError) as ei: + dash.ensure_dashboard(PATH, TABLE, client_factory=no_sdk, warehouse_id="wh") + assert ei.value.event == "test_results_dashboard_failed" + assert "data-product-forge[databricks]" in ei.value.context["fix"] + + +def test_latest_run_is_the_latest_one_that_tested_rules(definition): + sql = {d["name"]: "".join(d["queryLines"]) for d in definition["datasets"]} + latest_users = ("fluid_dq_overall", "fluid_dq_products", "fluid_dq_failing") + for name in latest_users: + assert "WITH last_run AS" in sql[name], name + assert "MAX(CASE WHEN" in sql[name] and "JOIN last_run" in sql[name], name + assert "COALESCE(" in sql[name] and "MAX(run_time)" in sql[name], name + assert not any("QUALIFY" in s for s in sql.values()) + + +def stale_published(update_time, revision_create_time): + dashboard = live_dashboard() + dashboard["update_time"] = update_time + return FakeApi( + status=existing_status(), + dashboard=dashboard, + published={"revision_create_time": revision_create_time}, + ) + + +PUBLISH_CALL = ("POST", f"{dash.DASHBOARDS_PATH}/dash-1/published") + + +def test_unchanged_draft_newer_than_published_revision_is_published(): + api = stale_published("2026-11-12T10:05:00Z", "2026-11-12T10:00:00Z") + result = ensure(api) + assert result["action"] == "unchanged" + assert PUBLISH_CALL in api.verbs() + assert "PATCH" not in [v[0] for v in api.verbs()] + + +def test_unchanged_draft_older_than_published_revision_is_not_republished(): + api = stale_published("2026-11-12T10:00:00Z", "2026-11-12T10:05:00Z") + assert ensure(api)["action"] == "unchanged" + assert PUBLISH_CALL not in api.verbs() + + +def test_unparseable_timestamps_do_not_republish(): + api = stale_published("garbage", "also garbage") + assert ensure(api)["action"] == "unchanged" + assert PUBLISH_CALL not in api.verbs() + + +@pytest.mark.parametrize( + "first, second, later", + [ + ("2026-11-12T21:16:40.1Z", "2026-11-12T21:16:40.05Z", True), + ("2026-11-12T21:16:40.123456789Z", "2026-11-12T21:16:40.1234Z", True), + ("2026-11-12T21:16:39Z", "2026-11-12T21:16:40.000001Z", False), + ], +) +def test_later_reads_any_fraction_length(first, second, later): + # Python 3.10's fromisoformat takes only 3 or 6 fraction digits. + assert dash._later(first, second) is later diff --git a/tests/cli/test_fluid_test_results_table.py b/tests/cli/test_fluid_test_results_table.py new file mode 100644 index 00000000..38aa9d80 --- /dev/null +++ b/tests/cli/test_fluid_test_results_table.py @@ -0,0 +1,541 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``fluid test``: JUnit golden output, the results-table flags and the Unity Catalog +results writer (rows, insert statements, write_results) against a fake client.""" + +from __future__ import annotations + +import argparse +import json +import logging +import xml.etree.ElementTree as ET +from datetime import datetime, timedelta, timezone + +import pytest + +from fluid_build.cli import _test_results_databricks as rt +from fluid_build.cli import test as fluid_test +from fluid_build.cli._common import CLIError +from fluid_build.cli.contract_validation import CheckOutcome, ValidationReport + +# ---------------------------------------------------------------- helpers + + +def make_report(*, version="0.0.0", when=None): + return ValidationReport( + contract_path="c.yaml", + contract_id="prod", + contract_version=version, + validation_time=when or datetime(2026, 11, 12, 10, 0, 0, tzinfo=timezone.utc), + duration=1.2, + ) + + +def rule_check(name, passed, **kw): + expose, _, rid = name.rpartition(".") + base = dict( + name=name, + category="quality", + passed=passed, + severity="error", + message="msg", + expected="1.0", + actual="0.5", + rule_id=rid, + rule_type="completeness", + expose_id=expose, + ) + base.update(kw) + return CheckOutcome(**base) + + +def contract(**extra): + base = { + "version": "2.0.0", + "metadata": {"layer": "Silver"}, + "exposes": [ + { + "id": "e1", + "version": "1.4.0", + "binding": { + "platform": "databricks", + "location": {"database": "main", "schema": "energy", "table": "meters"}, + }, + } + ], + } + base.update(extra) + return base + + +# ------------------------------------------------------------ JUnit golden + + +def test_junit_golden_testcase_names_order_and_failure_counts(tmp_path): + report = make_report() + report.record_check(rule_check("e1.c1", True)) + report.record_check(rule_check("e1.u1", False, rule_type="uniqueness")) + report.add_issue("error", "quality", "u1 failed", "exposes[0].dq.rules.u1") + report.add_issue("error", "missing_field", "column b missing", "fields.b") + report.add_issue("warning", "mode_mismatch", "b is nullable", "fields.b.mode") + report.add_issue("warning", "overlay", "overlay hit another entry", "x") + out = tmp_path / "j.xml" + fluid_test._output_junit(report, str(out)) + suite = ET.parse(out).getroot() + cases = [(c.get("classname"), c.get("name")) for c in suite.findall("testcase")] + assert cases == [ + ("fluid.test.prod.quality", "e1.c1"), + ("fluid.test.prod.quality", "e1.u1"), + *[ + ("fluid.test.prod", n) + for n in ( + "schema", + "connection", + "binding", + "missing_resource", + "missing_field", + "type_mismatch", + "mode_mismatch", + "extra_field", + "empty_table", + "row_count_below_threshold", + "metadata", + "drift", + "overlay", + ) + ], + ] + assert suite.get("tests") == "15" and suite.get("failures") == "2" + failing = {c.get("name") for c in suite.findall("testcase") if c.find("failure") is not None} + assert failing == {"e1.u1", "missing_field"} + warned = {c.get("name") for c in suite.findall("testcase") if c.find("system-out") is not None} + assert warned == {"mode_mismatch", "overlay"} + rule_fail = suite.findall("testcase")[1].find("failure") + assert rule_fail.get("type") == "DataQualityFailure[error]" + + +def test_junit_without_rule_checks_keeps_quality_rollup_case(tmp_path): + report = make_report() + out = tmp_path / "j.xml" + fluid_test._output_junit(report, str(out)) + names = [c.get("name") for c in ET.parse(out).getroot().findall("testcase")] + assert names == list(fluid_test_cases().STRUCTURAL_CATEGORIES) + + +def fluid_test_cases(): + from fluid_build.cli import _test_cases + + return _test_cases + + +# --------------------------------------------------------------- flags + + +def args_for(path, **kw): + ns = dict(contract=str(path), engine="native", results_table=None, results_dashboard=None) + ns.update(kw) + return argparse.Namespace(**ns) + + +@pytest.fixture +def contract_file(tmp_path): + p = tmp_path / "contract.fluid.yaml" + p.write_text("fluidVersion: 0.7.3\nid: x\n", encoding="utf-8") + return p + + +@pytest.fixture +def no_validator(monkeypatch): + def boom(*a, **k): + raise AssertionError("validation must not start") + + monkeypatch.setattr("fluid_build.cli.contract_validation.ContractValidator", boom) + + +LOG = logging.getLogger("t") + + +def test_dashboard_without_table_is_exit_2(contract_file, no_validator): + with pytest.raises(CLIError) as ei: + fluid_test.run(args_for(contract_file, results_dashboard="/Workspace/Shared/a/b"), LOG) + assert ei.value.exit_code == 2 and ei.value.event == "test_results_dashboard_needs_table" + + +def test_results_table_with_soda_engine_is_exit_2(contract_file, no_validator): + with pytest.raises(CLIError) as ei: + fluid_test.run(args_for(contract_file, results_table="c.s.t", engine="soda"), LOG) + assert ei.value.exit_code == 2 and ei.value.event == "test_results_table_needs_native_engine" + + +@pytest.mark.parametrize("bad", ["a.b", "a.b.c.d", "bad-name.s.t", "a..t", "a.b.t`x"]) +def test_malformed_results_table_fails_before_validation(contract_file, no_validator, bad): + with pytest.raises(CLIError) as ei: + fluid_test.run(args_for(contract_file, results_table=bad), LOG) + assert ei.value.exit_code == 2 and ei.value.event == "test_results_table_invalid" + + +def test_malformed_dashboard_path_fails_before_validation(contract_file, no_validator): + with pytest.raises(CLIError) as ei: + fluid_test.run( + args_for(contract_file, results_table="c.s.t", results_dashboard="relative/x"), LOG + ) + assert ei.value.exit_code == 2 + + +# ------------------------------------------------------ parse / refuse + + +def test_parse_results_table_returns_parts(): + assert rt.parse_results_table("main.dq.results") == ("main", "dq", "results") + + +def test_refuse_product_schema_is_case_insensitive(): + with pytest.raises(CLIError) as ei: + rt.refuse_product_schema(("MAIN", "Energy", "results"), contract()) + assert ei.value.exit_code == 2 and ei.value.event == "test_results_table_in_product_schema" + + +def test_refuse_product_schema_allows_other_schema(): + rt.refuse_product_schema(("main", "dq", "results"), contract()) + + +def test_refuse_product_schema_accepts_catalog_key(): + c = contract() + loc = c["exposes"][0]["binding"]["location"] + loc["catalog"] = loc.pop("database") + with pytest.raises(CLIError): + rt.refuse_product_schema(("main", "energy", "r"), c) + + +# ---------------------------------------------------------------- rows + + +def rows_for(report, c=None): + return rt.results_rows(report, c or contract(), run_id="run-1", fluid_version="9.9") + + +def test_rule_rows_then_one_structural_row_per_category(): + report = make_report() + report.record_check(rule_check("e1.c1", True)) + report.add_issue("warning", "overlay", "o", "p") + rows = rows_for(report) + builtin, extra, _ = fluid_test_cases().structural_categories(report) + assert [r["rule_id"] for r in rows] == ["c1", *builtin, *extra] + assert rows[0]["rule_type"] == "completeness" + assert {r["rule_type"] for r in rows[1:]} == {"structure"} + + +def test_rule_row_fields_come_from_check_and_expose(): + report = make_report() + report.record_check(rule_check("e1.c1", False, severity="Warning")) + row = rows_for(report)[0] + assert row["product_id"] == "prod" and row["layer"] == "Silver" + assert row["expose_id"] == "e1" and row["table_name"] == "main.energy.meters" + assert row["severity"] == "warning" and row["passed"] is False + assert (row["observed_value"], row["threshold"]) == ("0.5", "1.0") + assert row["fluid_version"] == "9.9" and row["run_id"] == "run-1" + + +def test_product_version_prefers_expose_version(): + report = make_report(version="0.0.0") + report.record_check(rule_check("e1.c1", True)) + assert rows_for(report)[0]["product_version"] == "1.4.0" + + +def test_product_version_falls_back_to_contract_version_never_report_placeholder(): + c = contract() + del c["exposes"][0]["version"] + report = make_report(version="0.0.0") + report.record_check(rule_check("e1.c1", True)) + assert {r["product_version"] for r in rows_for(report, c)} == {"2.0.0"} + + +def test_product_version_none_when_nothing_declared(): + c = contract() + del c["exposes"][0]["version"], c["version"] + report = make_report(version="0.0.0") + report.record_check(rule_check("e1.c1", True)) + assert rows_for(report, c)[0]["product_version"] is None + + +def test_run_time_is_converted_to_utc(): + tz = timezone(timedelta(hours=5)) + report = make_report(when=datetime(2026, 11, 12, 15, 0, 0, tzinfo=tz)) + row = rows_for(report)[0] + assert row["run_time"].utcoffset() == timedelta(0) and row["run_time"].hour == 10 + + +def test_message_is_truncated_to_2000(): + report = make_report() + report.record_check(rule_check("e1.c1", False, message="x" * 5000)) + msg = rows_for(report)[0]["message"] + assert len(msg) == 2000 and msg.endswith("...") + + +def test_structural_row_for_error_and_clean_category(): + report = make_report() + report.add_issue("error", "missing_field", "b missing", "p", expected="b", actual=None) + rows = {r["rule_id"]: r for r in rows_for(report)} + bad, ok = rows["missing_field"], rows["schema"] + assert bad["passed"] is False and bad["severity"] == "error" and bad["threshold"] == "b" + assert bad["message"] == "b missing" and bad["table_name"] == "main.energy.meters" + assert ok["passed"] is True and ok["message"] == "OK" and ok["severity"] is None + + +def test_structural_warning_does_not_fail_the_row(): + report = make_report() + report.add_issue("warning", "mode_mismatch", "nullable", "p") + row = {r["rule_id"]: r for r in rows_for(report)}["mode_mismatch"] + assert row["passed"] is True and row["severity"] == "warning" + + +# --------------------------------------------------- insert statements + + +def row_dict(i): + return { + "run_id": "r", + "run_time": datetime(2026, 1, 2, 3, 4, 5, 6, tzinfo=timezone.utc), + "product_id": "p", + "rule_id": f"rule{i}", + "passed": i % 2 == 0, + } + + +def col(name): + return [n for n, _, _ in rt.RESULT_COLUMNS].index(name) + + +@pytest.mark.parametrize("count", [1, 60]) +def test_insert_is_one_statement_with_one_rows_parameter(count): + sql, params = rt.insert_statement("`c`.`s`.`t`", [row_dict(i) for i in range(count)]) + assert isinstance(sql, str) + assert [p["name"] for p in params] == ["rows"] + payload = json.loads(params[0]["value"]) + assert isinstance(payload, list) and len(payload) == count + names = {n for n, _, _ in rt.RESULT_COLUMNS} + assert all(set(item) == names for item in payload) + assert [item["rule_id"] for item in payload] == [f"rule{i}" for i in range(count)] + + +def test_insert_statement_unpacks_the_json_array_in_sql(): + sql, _ = rt.insert_statement("`c`.`s`.`t`", [row_dict(0)]) + assert sql.startswith("INSERT INTO `c`.`s`.`t` (run_id, run_time,") + assert "from_json(:rows, 'ARRAY.py +""" diff --git a/tests/exporters/test_sodacl_column_values.py b/tests/exporters/test_sodacl_column_values.py new file mode 100644 index 00000000..405bb626 --- /dev/null +++ b/tests/exporters/test_sodacl_column_values.py @@ -0,0 +1,90 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""SodaCL export of ``valid_values`` rules from the column's own declarations, and of +multi-column selectors.""" + +from __future__ import annotations + +import yaml + +from fluid_build.exporters.sodacl import render_sodacl_document + +COLUMNS = [ + {"name": "status", "validationRules": [{"type": "enum", "constraint": "A | B"}]}, + {"name": "code", "validationRules": [{"type": "regex", "constraint": "^x$"}]}, + {"name": "pct", "validationRules": [{"type": "range", "constraint": ">= 0 and <= 100"}]}, + {"name": "pos", "validationRules": [{"type": "range", "constraint": "> 0"}]}, + {"name": "site", "labels": {"references": "acme.sites.site_id"}}, + {"name": "tier", "validationRules": [{"type": "enum", "constraint": "P | Q"}]}, +] + + +def render(rules): + contract = { + "exposes": [ + { + "id": "e1", + "binding": { + "platform": "databricks", + "location": {"database": "c", "schema": "s", "table": "t"}, + }, + "contract": {"schema": COLUMNS, "dq": {"rules": rules}}, + } + ] + } + result = render_sodacl_document(contract) + checks = (yaml.safe_load(result.text) or {}).get("checks for t", []) + return result, checks + + +def valid_values(rule_id, selector, **extra): + return {"id": rule_id, "type": "valid_values", "selector": selector, **extra} + + +def test_enum_becomes_valid_values(): + result, checks = render([valid_values("r", "status")]) + assert checks == [{"invalid_count(status) = 0": {"valid values": ["A", "B"]}}] + assert result.mapped == ["e1.r"] + + +def test_regex_becomes_valid_regex(): + _, checks = render([valid_values("r", "code")]) + assert checks == [{"invalid_count(code) = 0": {"valid regex": "^x$"}}] + + +def test_inclusive_range_becomes_valid_min_and_max(): + _, checks = render([valid_values("r", "pct")]) + assert checks == [{"invalid_count(pct) = 0": {"valid min": 0, "valid max": 100}}] + + +def test_strict_bound_is_unmapped_naming_the_bound(): + result, checks = render([valid_values("r", "pos")]) + assert checks == [] and result.mapped == [] + (rule,) = result.unmapped + assert rule.rule_id == "r" and "> 0" in rule.reason + + +def test_reference_column_is_unmapped_naming_the_product(): + result, checks = render([valid_values("r", "site")]) + assert checks == [] + (rule,) = result.unmapped + assert "acme.sites" in rule.reason + + +def test_rule_description_list_wins_over_the_column_enum(): + _, checks = render([valid_values("r", "tier", description="tier valid values: X, Y.")]) + assert checks == [{"invalid_count(tier) = 0": {"valid values": ["X", "Y"]}}] + + +def test_multi_column_completeness_is_one_check_per_column_and_mapped(): + result, checks = render([{"id": "c", "type": "completeness", "selector": "a,b"}]) + assert checks == ["missing_count(a) = 0", "missing_count(b) = 0"] + assert result.mapped == ["e1.c"] and result.unmapped == [] + + +def test_multi_column_uniqueness_is_unmapped(): + result, checks = render([{"id": "u", "type": "uniqueness", "selector": "a,b"}]) + assert checks == [] and result.mapped == [] + (rule,) = result.unmapped + assert "takes one column" in rule.reason diff --git a/tests/providers/test_databricks_statements.py b/tests/providers/test_databricks_statements.py new file mode 100644 index 00000000..fd841f08 --- /dev/null +++ b/tests/providers/test_databricks_statements.py @@ -0,0 +1,219 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``providers/databricks/statements.py``: rows from a SQL warehouse over +``databricks_enforce.SqlRunner`` (request shape with parameters, polling, +timeout/cancel, failure scrubbing, type conversion, chunk paging), against a fake +``api_client.do``.""" + +from __future__ import annotations + +import pytest + +from fluid_build.iac.providers.databricks_enforce import StatementFailed +from fluid_build.providers.databricks.statements import ( + STATEMENTS_PATH, + StatementError, + StatementRunner, + param, + warehouse_id_from_env, +) + +FAKE_TOKEN = "fake-test-token-0000" # pragma: allowlist secret + + +class FakeApi: + def __init__(self, responses): + self.responses = list(responses) + self.calls = [] + + def do(self, method, path, query=None, body=None, **kw): + self.calls.append((method, path, body)) + item = self.responses.pop(0) + if isinstance(item, Exception): + raise item + return item + + +class FakeClient: + def __init__(self, responses): + self.api_client = FakeApi(responses) + + +def ok(rows=None, columns=(), **result_extra): + return { + "statement_id": "sid-1", + "status": {"state": "SUCCEEDED"}, + "manifest": {"schema": {"columns": [{"name": n, "type_name": t} for n, t in columns]}}, + "result": {"chunk_index": 0, "data_array": rows or [], **result_extra}, + } + + +def running(state="RUNNING"): + return {"statement_id": "sid-1", "status": {"state": state}} + + +def runner(responses, **kw): + client = FakeClient(responses) + kw.setdefault("sleep", lambda s: None) + return StatementRunner(client, "wh-1", **kw), client.api_client + + +def test_statement_error_is_the_enforce_runner_failure(): + assert StatementError is StatementFailed + + +def test_post_body_without_parameters(): + r, api = runner([ok([["1"]], [("one", "INT")])]) + assert r.run("SELECT 1") == [(1,)] + method, path, body = api.calls[0] + assert (method, path.rstrip("/")) == ("POST", STATEMENTS_PATH) + assert body == { + "warehouse_id": "wh-1", + "statement": "SELECT 1", + "wait_timeout": "30s", + "on_wait_timeout": "CONTINUE", + } + + +def test_parameters_are_bound_not_spliced(): + r, api = runner([ok()]) + r.run("SELECT :v", [param("v", "x'; DROP TABLE t; --")]) + body = api.calls[0][2] + assert body["statement"] == "SELECT :v" + assert body["parameters"] == [{"name": "v", "type": "STRING", "value": "x'; DROP TABLE t; --"}] + + +def test_live_response_shape_is_converted_by_type_name(): + # The shape a Free Edition warehouse returned for + # SELECT 1.5 AS d, CAST(2 AS BIGINT) AS l, 'x' AS s, true AS b + live = ok( + [["1.5", "2", "x", "true"]], + [("d", "DECIMAL"), ("l", "LONG"), ("s", "STRING"), ("b", "BOOLEAN")], + row_count=1, + row_offset=0, + ) + r, _ = runner([live]) + assert r.run("SELECT ...") == [(1.5, 2, "x", True)] + + +def test_null_stays_none_and_false_boolean_and_odd_value_kept(): + r, _ = runner([ok([[None, "false", "n/a"]], [("a", "LONG"), ("b", "BOOLEAN"), ("c", "INT")])]) + assert r.run("q") == [(None, False, "n/a")] + + +def test_pending_then_running_then_succeeded_polls_get(): + r, api = runner([running("PENDING"), running(), ok([["7"]], [("n", "LONG")])]) + assert r.run("q") == [(7,)] + assert [c[0] for c in api.calls] == ["POST", "GET", "GET"] + assert api.calls[1][1] == f"{STATEMENTS_PATH}/sid-1" + + +def test_timeout_cancels_then_raises(): + pending = [running("PENDING")] * 50 + r, api = runner(pending + [{}], timeout_seconds=0) + with pytest.raises(StatementError, match="did not finish"): + r.run("q") + assert any(c[0] == "POST" and c[1].endswith("/sid-1/cancel") for c in api.calls) + + +def test_failed_state_raises_with_scrubbed_api_message(monkeypatch): + monkeypatch.setenv("DATABRICKS_TOKEN", FAKE_TOKEN) + failed = { + "statement_id": "sid-1", + "status": { + "state": "FAILED", + "error": {"message": f"[TABLE_OR_VIEW_NOT_FOUND] t ({FAKE_TOKEN})"}, + }, + } + r, _ = runner([failed]) + with pytest.raises(StatementError) as exc: + r.run("q") + assert "TABLE_OR_VIEW_NOT_FOUND" in str(exc.value) + assert FAKE_TOKEN not in str(exc.value) + + +def test_transport_exception_is_wrapped_and_scrubbed(monkeypatch): + monkeypatch.setenv("DATABRICKS_TOKEN", FAKE_TOKEN) + r, _ = runner([RuntimeError(f"connection reset, token {FAKE_TOKEN}")]) + with pytest.raises(StatementError) as exc: + r.run("q") + assert "RuntimeError" in str(exc.value) + assert FAKE_TOKEN not in str(exc.value) + + +def test_canceled_state_raises(): + r, _ = runner([{"statement_id": "sid-1", "status": {"state": "CANCELED"}}]) + with pytest.raises(StatementError, match="CANCELED"): + r.run("q") + + +def test_next_chunk_index_fetches_further_chunks(): + first = ok([["1"]], [("n", "LONG")], next_chunk_index=1) + second = {"chunk_index": 1, "data_array": [["2"], ["3"]]} + r, api = runner([first, second]) + assert r.run("q") == [(1,), (2,), (3,)] + assert api.calls[1] == ("GET", f"{STATEMENTS_PATH}/sid-1/result/chunks/1", None) + + +def test_chunk_fetch_failure_is_a_statement_error(monkeypatch): + monkeypatch.setenv("DATABRICKS_TOKEN", FAKE_TOKEN) + first = ok([["1"]], [("n", "LONG")], next_chunk_index=1) + r, _ = runner([first, RuntimeError(f"chunk gone {FAKE_TOKEN}")]) + with pytest.raises(StatementError) as exc: + r.run("q") + assert FAKE_TOKEN not in str(exc.value) + + +def test_param_omits_value_for_none_and_takes_type(): + assert param("n", None) == {"name": "n", "type": "STRING"} + assert param("b", True, "BOOLEAN") == {"name": "b", "type": "BOOLEAN", "value": "True"} + + +def test_warehouse_id_from_env_unset_or_blank_names_the_variable(): + for env in ({}, {"DATABRICKS_WAREHOUSE_ID": " "}): + with pytest.raises(StatementError, match="DATABRICKS_WAREHOUSE_ID"): + warehouse_id_from_env(env) + + +def test_warehouse_id_from_env_trims_and_reads_os_environ(monkeypatch): + assert warehouse_id_from_env({"DATABRICKS_WAREHOUSE_ID": " wh "}) == "wh" + monkeypatch.setenv("DATABRICKS_WAREHOUSE_ID", "from-env") + assert warehouse_id_from_env() == "from-env" + + +# ------------------------------------------------------- circuit breaker + + +def test_timeout_stops_the_statements_after_it_without_an_api_call(): + r, api = runner([running("PENDING")] * 50 + [{}], timeout_seconds=0) + with pytest.raises(StatementError, match="did not finish"): + r.run("q1") + calls = len(api.calls) + with pytest.raises(StatementError) as exc: + r.run("q2") + assert str(exc.value).startswith("not run:") + assert len(api.calls) == calls + + +def test_transport_failure_stops_the_statements_after_it_without_an_api_call(): + r, api = runner([RuntimeError("connection reset")]) + with pytest.raises(StatementError, match="RuntimeError"): + r.run("q1") + with pytest.raises(StatementError) as exc: + r.run("q2") + assert str(exc.value).startswith("not run:") + assert len(api.calls) == 1 + + +def test_a_sql_error_does_not_stop_the_statements_after_it(): + failed = { + "statement_id": "sid-1", + "status": {"state": "FAILED", "error": {"message": "[TABLE_OR_VIEW_NOT_FOUND] t"}}, + } + r, api = runner([failed, ok([["7"]], [("n", "LONG")])]) + with pytest.raises(StatementError) as first: + r.run("q1") + assert not str(first.value).startswith("not run:") + assert r.run("q2") == [(7,)] + assert len([c for c in api.calls if c[0] == "POST"]) == 2 diff --git a/tests/providers/test_databricks_validation.py b/tests/providers/test_databricks_validation.py new file mode 100644 index 00000000..f1ceb73d --- /dev/null +++ b/tests/providers/test_databricks_validation.py @@ -0,0 +1,453 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""DatabricksValidationProvider against a fake workspace client (no SDK, no network).""" + +from __future__ import annotations + +import pytest + +from fluid_build.providers.databricks.statements import StatementError +from fluid_build.providers.databricks_validation import ( + DatabricksValidationProvider, + sql_table_ref, + table_name, +) +from fluid_build.providers.validation_provider import FieldSchema, ResourceSchema, ResourceType + + +class NotFound(Exception): + status_code = 404 + + +def table_json(name="smart_meters", columns=None): + cols = columns or [ + ("site_id", "string", "STRING", False), + ("qty", "int", "INT", True), + ("price", "decimal(12,3)", "DECIMAL", True), + ] + return { + "name": name, + "table_type": "MANAGED", + "columns": [ + {"name": n, "type_text": tt, "type_name": tn, "nullable": nb} for n, tt, tn, nb in cols + ], + } + + +class FakeApi: + """Routes api_client.do by method+path; records every call.""" + + def __init__(self): + self.calls = [] + self.tables = {} # full name -> json or exception + self.warehouses = {"w": {"id": "w"}} + self.statements = [] # (predicate, rows) + self.sql = [] + + def do(self, method, path, query=None, body=None, **kw): + self.calls.append((method, path, body)) + if method == "GET" and path.startswith("/api/2.0/sql/warehouses/"): + item = self.warehouses.get(path.rsplit("/", 1)[1], NotFound("no wh")) + elif method == "GET" and path.startswith("/api/2.1/unity-catalog/tables/"): + from urllib.parse import unquote + + item = self.tables.get(unquote(path.rsplit("/", 1)[1]), NotFound("no table")) + elif method == "POST" and path.rstrip("/") == "/api/2.0/sql/statements": + self.sql.append(body) + item = self._answer(body) + else: + raise AssertionError(f"unexpected call {method} {path}") + if isinstance(item, Exception): + raise item + return item + + def _answer(self, body): + text = body["statement"] + for pred, rows, cols in self.statements: + if pred in text: + return { + "statement_id": "s", + "status": {"state": "SUCCEEDED"}, + "manifest": { + "schema": {"columns": [{"name": c, "type_name": t} for c, t in cols]} + }, + "result": {"data_array": rows}, + } + return { + "statement_id": "s", + "status": {"state": "SUCCEEDED"}, + "manifest": {"schema": {"columns": [{"name": "n", "type_name": "LONG"}]}}, + "result": {"data_array": [["0"]]}, + } + + def stmt_calls(self): + return [c for c in self.calls if c[0] == "POST"] + + +class FakeClient: + def __init__(self, api): + self.api_client = api + + +@pytest.fixture +def api(): + return FakeApi() + + +@pytest.fixture +def provider(api): + return DatabricksValidationProvider( + {"client_factory": lambda: FakeClient(api), "warehouse_id": "w"} + ) + + +EXPOSE = { + "id": "meters", + "binding": { + "platform": "databricks", + "location": {"database": "main", "schema": "energy", "table": "smart_meters"}, + }, + "contract": { + "schema": [ + {"name": "site_id", "type": "VARCHAR", "required": True}, + {"name": "qty", "type": "INTEGER"}, + {"name": "price", "type": "DECIMAL(12,3)"}, + ] + }, +} +FULL = "main.energy.smart_meters" + + +# ---------------------------------------------------------------- names + + +def test_table_name_validates_parts(): + assert table_name(EXPOSE) == ("main", "energy", "smart_meters") + assert table_name({"binding": {}}) is None + bad = {"binding": {"location": {"database": "m", "schema": "s", "table": "bad-name"}}} + with pytest.raises(ValueError): + table_name(bad) + + +def test_table_name_rejects_backtick(): + bad = {"binding": {"location": {"database": "m", "schema": "s", "table": "t`x"}}} + with pytest.raises(ValueError): + table_name(bad) + + +def test_sql_table_ref_backticks_each_part(): + assert sql_table_ref(("a", "b", "c")) == "`a`.`b`.`c`" + + +# ----------------------------------------------------------- connection + + +def test_validate_connection_gets_warehouse(provider, api): + assert provider.validate_connection() is True + assert api.calls == [("GET", "/api/2.0/sql/warehouses/w", None)] + + +def test_validate_connection_404_names_warehouse(api): + p = DatabricksValidationProvider( + {"client_factory": lambda: FakeClient(api), "warehouse_id": "gone"} + ) + with pytest.raises(StatementError, match="gone"): + p.validate_connection() + + +def test_validate_connection_other_error_is_wrapped(api): + api.warehouses["w"] = RuntimeError("boom") + p = DatabricksValidationProvider( + {"client_factory": lambda: FakeClient(api), "warehouse_id": "w"} + ) + with pytest.raises(StatementError, match="RuntimeError: boom"): + p.validate_connection() + + +def test_validate_connection_needs_warehouse_env(api, monkeypatch): + monkeypatch.delenv("DATABRICKS_WAREHOUSE_ID", raising=False) + p = DatabricksValidationProvider({"client_factory": lambda: FakeClient(api)}) + with pytest.raises(StatementError, match="DATABRICKS_WAREHOUSE_ID"): + p.validate_connection() + + +# ---------------------------------------------------------------- schema + + +def test_get_resource_schema_missing_table_is_none(provider): + assert provider.get_resource_schema(EXPOSE) is None + + +def test_get_resource_schema_maps_columns_and_counts_rows(provider, api): + api.tables[FULL] = table_json() + api.statements.append(("COUNT(*)", [["42"]], [("c", "LONG")])) + schema = provider.get_resource_schema(EXPOSE) + assert [(f.name, f.type, f.mode) for f in schema.fields] == [ + ("site_id", "string", "REQUIRED"), + ("qty", "int", "NULLABLE"), + ("price", "decimal(12,3)", "NULLABLE"), + ] + assert schema.row_count == 42 and schema.fully_qualified_name == FULL + assert schema.metadata["table_type"] == "MANAGED" + assert api.sql[0]["statement"] == "SELECT COUNT(*) FROM `main`.`energy`.`smart_meters`" + + +def test_get_resource_schema_requires_location(provider): + with pytest.raises(ValueError, match="binding.location"): + provider.get_resource_schema({"binding": {}}) + + +def test_get_resource_schema_other_error_wrapped(provider, api): + api.tables[FULL] = RuntimeError("kaboom") + with pytest.raises(StatementError, match="kaboom"): + provider.get_resource_schema(EXPOSE) + + +def live_schema(fields, rows=5): + return ResourceSchema(ResourceType.TABLE, FULL, fields, row_count=rows) + + +def live_fields(): + return [ + FieldSchema("site_id", "string", "REQUIRED"), + FieldSchema("qty", "int", "NULLABLE"), + FieldSchema("price", "decimal(12,3)", "NULLABLE"), + ] + + +def test_validate_resource_contract_types_match_live_types(provider): + res = provider.validate_resource(EXPOSE, live_schema(live_fields())) + assert res.success and res.issues == [] + + +def test_validate_resource_missing_table(provider): + res = provider.validate_resource(EXPOSE, None) + assert not res.success + (issue,) = res.issues + assert issue.category == "missing_resource" and FULL in issue.message + + +def test_validate_resource_real_type_change_is_type_mismatch(provider): + fields = live_fields() + fields[1] = FieldSchema("qty", "bigint", "NULLABLE") + res = provider.validate_resource(EXPOSE, live_schema(fields)) + assert not res.success + assert [(i.category, i.severity) for i in res.issues] == [("type_mismatch", "error")] + + +def test_validate_resource_missing_column(provider): + res = provider.validate_resource(EXPOSE, live_schema(live_fields()[:2])) + assert [i.category for i in res.issues] == ["missing_field"] + + +def test_validate_resource_empty_table_is_warning_only(provider): + res = provider.validate_resource(EXPOSE, live_schema(live_fields(), rows=0)) + assert res.success + assert [(i.category, i.severity) for i in res.issues] == [("empty_table", "warning")] + + +def test_validate_resource_sla_row_count_min(provider): + spec = {**EXPOSE, "sla": {"row_count_min": 10}} + res = provider.validate_resource(spec, live_schema(live_fields(), rows=3)) + assert not res.success + assert [i.category for i in res.issues] == ["row_count_below_threshold"] + + +# ----------------------------------------------------------- quality rules + + +@pytest.fixture +def live(api): + """The tested table exists (the rule tests read data from it).""" + api.tables[FULL] = table_json() + return api + + +def test_rules_on_a_missing_table_run_no_statement(provider, api): + rules = [ + {"id": "c1", "type": "completeness", "selector": "site_id", "severity": "error"}, + {"id": "u1", "type": "uniqueness", "selector": "site_id"}, + ] + results = provider.run_quality_results(EXPOSE, rules) + assert [r.rule_id for r in results] == ["c1", "u1"] + assert all(not r.passed and r.actual == "not executed" for r in results) + assert results[0].severity == "error" + assert FULL in results[0].message and "does not exist" in results[0].message + assert api.sql == [] + + +def test_table_metadata_is_read_once_for_schema_and_rules(provider, live): + live.statements.append(("COUNT(*)", [["4"]], [("n", "LONG")])) + provider.get_resource_schema(EXPOSE) + live.statements.insert(0, ("NULLIF(COUNT(*), 0)", [["1.0"]], [("r", "DOUBLE")])) + provider.run_quality_results( + EXPOSE, [{"id": "c1", "type": "completeness", "selector": "site_id"}] + ) + gets = [ + c for c in live.calls if c[0] == "GET" and FULL.replace(".", "") in c[1].replace(".", "") + ] + assert len(gets) == 1 + + +def test_run_quality_results_uses_databricks_dialect_and_backticks(provider, live): + rules = [{"id": "c1", "type": "completeness", "selector": "site_id"}] + live.statements.insert(0, ("NULLIF(COUNT(*), 0)", [["1.0"]], [("r", "DOUBLE")])) + (res,) = provider.run_quality_results(EXPOSE, rules) + stmt = live.sql[0]["statement"] + assert res.passed and res.rule_id == "c1" + assert "FROM `main`.`energy`.`smart_meters`" in stmt + assert "`site_id`" in stmt and "AS DOUBLE" in stmt + + +def test_run_quality_results_passes_columns_for_enum(provider, live): + spec = { + **EXPOSE, + "contract": { + "schema": [ + { + "name": "status", + "validationRules": [{"type": "enum", "constraint": "A|B"}], + } + ] + }, + } + (res,) = provider.run_quality_results( + spec, [{"id": "v", "type": "valid_values", "selector": "status"}] + ) + assert res.passed and "NOT IN ('A', 'B')" in live.sql[0]["statement"] + + +def test_run_quality_results_requires_location(provider): + with pytest.raises(ValueError): + provider.run_quality_results({"binding": {}}, []) + + +# ---------------------------------------------------------- references + +REF_COLUMNS = [{"name": "site_id", "labels": {"references": "acme.sites.site_id"}}] +REF_EXPOSE = {**EXPOSE, "contract": {"schema": REF_COLUMNS}} +REF_RULE = {"id": "fk", "type": "valid_values", "selector": "site_id"} + + +def tag_rows(live, rows): + live.statements.insert( + 0, + ( + "table_tags", + rows, + [("catalog_name", "STRING"), ("schema_name", "STRING"), ("table_name", "STRING")], + ), + ) + + +def test_reference_lookup_uses_tag_query_with_params_then_anti_join(provider, live): + tag_rows(live, [["main", "ref", "sites"], ["main", "ref", "other"]]) + live.tables["main.ref.sites"] = table_json("sites", [("site_id", "string", "STRING", True)]) + live.tables["main.ref.other"] = table_json("other", [("x", "string", "STRING", True)]) + live.statements.insert(1, ("NOT EXISTS", [["0"]], [("m", "LONG")])) + (res,) = provider.run_quality_results(REF_EXPOSE, [REF_RULE]) + tag_call = live.sql[0] + assert "system.information_schema.table_tags" in tag_call["statement"] + assert tag_call["parameters"] == [ + {"name": "tag", "type": "STRING", "value": "fluid_product_id"}, + {"name": "product", "type": "STRING", "value": "acme.sites"}, + ] + assert "FROM `main`.`ref`.`sites` AS p" in live.sql[1]["statement"] + assert res.passed + + +def test_reference_no_candidate_is_not_executed_with_lookup_text(provider, live): + tag_rows(live, []) + (res,) = provider.run_quality_results(REF_EXPOSE, [REF_RULE]) + assert not res.passed and res.actual == "not executed" + assert "fluid_product_id=acme.sites" in res.message + + +def test_reference_ambiguous_is_not_executed(provider, live): + tag_rows(live, [["main", "a", "sites"], ["main", "b", "sites"]]) + for s in ("a", "b"): + live.tables[f"main.{s}.sites"] = table_json( + "sites", [("site_id", "string", "STRING", True)] + ) + (res,) = provider.run_quality_results(REF_EXPOSE, [REF_RULE]) + assert res.actual == "not executed" and "ambiguous" in res.message + # The message lands in the shared results table: other tables are not named. + assert "main.a.sites" not in res.message and "main.b.sites" not in res.message + + +def test_reference_lookup_cached_per_product_and_column(provider, live): + tag_rows(live, [["main", "ref", "sites"]]) + live.tables["main.ref.sites"] = table_json("sites", [("site_id", "string", "STRING", True)]) + rules = [REF_RULE, {**REF_RULE, "id": "fk2"}] + provider.run_quality_results(REF_EXPOSE, rules) + tag_queries = [b for b in live.sql if "table_tags" in b["statement"]] + assert len(tag_queries) == 1 + anti_joins = [b for b in live.sql if "NOT EXISTS" in b["statement"]] + assert len(anti_joins) == 2 + + +def test_reference_unreadable_candidate_is_skipped(provider, live): + tag_rows(live, [["main", "ref", "sites"]]) # table GET raises NotFound + (res,) = provider.run_quality_results(REF_EXPOSE, [REF_RULE]) + assert res.actual == "not executed" + + +def test_run_quality_checks_returns_issues_for_failures(provider, live): + live.statements.insert(0, ("NULLIF(COUNT(*), 0)", [["0.5"]], [("r", "DOUBLE")])) + issues = provider.run_quality_checks( + EXPOSE, [{"id": "c1", "type": "completeness", "selector": "site_id", "severity": "error"}] + ) + assert [(i.category, i.severity) for i in issues] == [("quality", "error")] + + +def test_normalize_type_and_provider_name(provider): + assert provider.provider_name == "databricks" + assert provider.normalize_type("STRING") == "string" + + +def test_reference_prefers_the_tested_tables_catalog(provider, live): + # A look-alike table tagged with the same product id in another catalog does + # not make the lookup ambiguous, and is not the one read. + tag_rows(live, [["main", "ref", "sites"], ["shadow", "ref", "sites"]]) + for catalog in ("main", "shadow"): + live.tables[f"{catalog}.ref.sites"] = table_json( + "sites", [("site_id", "string", "STRING", True)] + ) + live.statements.insert(0, ("NOT EXISTS", [["0"]], [("n", "LONG")])) + (res,) = provider.run_quality_results(REF_EXPOSE, [REF_RULE]) + assert res.passed, res.message + anti_join = next(b for b in live.sql if "NOT EXISTS" in b["statement"])["statement"] + assert "`main`.`ref`.`sites`" in anti_join and "shadow" not in anti_join + + +def test_reference_two_candidates_elsewhere_stay_ambiguous(provider, live): + tag_rows(live, [["other1", "ref", "sites"], ["other2", "ref", "sites"]]) + for catalog in ("other1", "other2"): + live.tables[f"{catalog}.ref.sites"] = table_json( + "sites", [("site_id", "string", "STRING", True)] + ) + (res,) = provider.run_quality_results(REF_EXPOSE, [REF_RULE]) + assert res.actual == "not executed" and "ambiguous" in res.message + + +def test_failed_tag_lookup_is_cached_as_a_lookup_error(provider, live, monkeypatch): + answer = live._answer + + def failing_tags(body): + if "table_tags" in body["statement"]: + return { + "statement_id": "s", + "status": {"state": "FAILED", "error": {"message": "[INSUFFICIENT_PERMISSIONS]"}}, + } + return answer(body) + + monkeypatch.setattr(live, "_answer", failing_tags) + rules = [REF_RULE, {**REF_RULE, "id": "fk2"}] + results = provider.run_quality_results(REF_EXPOSE, rules) + tag_queries = [b for b in live.sql if "table_tags" in b["statement"]] + assert len(tag_queries) == 1 + assert len(results) == 2 + for res in results: + assert not res.passed and res.actual == "not executed" + assert "lookup" in res.message and "failed" in res.message diff --git a/tests/providers/test_quality_engine_column_rules.py b/tests/providers/test_quality_engine_column_rules.py new file mode 100644 index 00000000..cf94a412 --- /dev/null +++ b/tests/providers/test_quality_engine_column_rules.py @@ -0,0 +1,533 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""Column-rule behaviour of the native quality engine: dialect quoting, multi-column +rules, value/pattern/reference sources and the backslash-safe string literal.""" + +from __future__ import annotations + +import pytest + +from fluid_build.providers import quality_engine as qe +from fluid_build.providers._sql_safety import quote_string_literal_for + +T = "`c`.`s`.`t`" + + +class Recorder: + """execute_fn double: records SQL, answers from a queue (or a fixed row).""" + + def __init__(self, *answers): + self.sql = [] + self._answers = list(answers) + + def __call__(self, sql): + self.sql.append(sql) + if not self._answers: + return [(0,)] + return self._answers.pop(0) + + +def run(rule, execute_fn, dialect="databricks", **kw): + (res,) = qe.execute_quality_checks([rule], T, execute_fn, dialect, **kw) + return res + + +# ---------------------------------------------------------------- helpers + + +@pytest.mark.parametrize( + "dialect,expected", + [ + ("databricks", "`x`"), + ("bigquery", "`x`"), + ("BigQuery", "`x`"), + ("snowflake", '"x"'), + ("ansi", '"x"'), + (None, '"x"'), + ], +) +def test_qi_quotes_per_dialect(dialect, expected): + assert qe._qi("x", dialect) == expected + + +def test_qi_rejects_unsafe_identifier(): + with pytest.raises(ValueError): + qe._qi("a`b", "databricks") + + +@pytest.mark.parametrize( + "dialect,expected", + [ + ("databricks", "DOUBLE"), + ("bigquery", "FLOAT64"), + ("snowflake", "DOUBLE PRECISION"), + (None, "DOUBLE PRECISION"), + ], +) +def test_double_type_per_dialect(dialect, expected): + assert qe._double(dialect) == expected + + +def test_rule_columns_splits_and_strips(): + assert qe.rule_columns("a, b ,,c") == ["a", "b", "c"] + assert qe.rule_columns("*") == ["*"] + assert qe.rule_columns(None) == [] + + +def test_expose_columns_prefers_contract_schema(): + expose = {"contract": {"schema": [{"name": "a"}, "junk"]}, "schema": [{"name": "z"}]} + assert qe.expose_columns(expose) == [{"name": "a"}] + assert qe.expose_columns({"schema": [{"name": "z"}]}) == [{"name": "z"}] + assert qe.expose_columns("nope") == [] + + +# ----------------------------------------------------- valid value sources + +COLS = [ + { + "name": "status", + "validationRules": [{"type": "enum", "constraint": "GOOD | ESTIMATED, SUSPECT"}], + }, + {"name": "cc", "validationRules": [{"type": "regex", "constraint": "^[A-Z]{2}$"}]}, + {"name": "site_id", "labels": {"references": "acme.sites.site_id"}}, +] + + +def test_valid_values_explicit_wins(): + rule = {"selector": "status", "validValues": ["X"], "description": "status valid values: A."} + assert qe.extract_valid_values(rule, COLS) == ["X"] + + +def test_valid_values_from_description_beats_enum(): + rule = {"selector": "status", "description": "status valid values: A, B."} + assert qe.extract_valid_values(rule, COLS) == ["A", "B"] + + +def test_valid_values_from_enum_splits_on_pipe_and_comma(): + assert qe.extract_valid_values({"selector": "status"}, COLS) == [ + "GOOD", + "ESTIMATED", + "SUSPECT", + ] + + +def test_valid_values_empty_without_any_source(): + assert qe.extract_valid_values({"selector": "cc"}, COLS) == [] + + +def test_value_pattern_reads_regex_rule(): + assert qe.extract_value_pattern({"selector": "cc"}, COLS) == "^[A-Z]{2}$" + assert qe.extract_value_pattern({"selector": "status"}, COLS) is None + + +def test_reference_rpartitions_on_last_dot(): + assert qe.extract_reference({"selector": "site_id"}, COLS) == ("acme.sites", "site_id") + assert qe.extract_reference({"selector": "cc"}, COLS) is None + + +# ------------------------------------------------ multi-column completeness + + +def test_completeness_multi_column_is_one_statement_with_lowest_ratio(): + rec = Recorder([(1.0, 0.5, 0.9)]) + res = run( + { + "id": "r", + "type": "completeness", + "selector": "a,b,c", + "threshold": 0.95, + "operator": ">=", + }, + rec, + ) + assert len(rec.sql) == 1 + assert rec.sql[0].count("CAST(") == 3 and "AS DOUBLE" in rec.sql[0] + assert res.passed is False + assert res.actual == "0.5000" + assert "'b'" in res.message and "'c'" in res.message and "'a'" not in res.message + + +def test_completeness_multi_column_passes_when_all_meet_bound(): + res = run({"id": "r", "type": "completeness", "selector": "a,b"}, Recorder([(1.0, 1.0)])) + assert res.passed and res.expected == "1.0" + + +def test_completeness_null_ratio_counts_as_zero(): + res = run({"id": "r", "type": "completeness", "selector": "a,b"}, Recorder([(1.0, None)])) + assert not res.passed and res.actual == "0.0000" + + +# ------------------------------------------------ multi-column accuracy + + +@pytest.mark.parametrize( + "operator,func", [(">=", "MIN"), (">", "MIN"), ("<=", "MAX"), ("<", "MAX")] +) +def test_accuracy_multi_column_aggregate_follows_operator(operator, func): + rec = Recorder([(1, 2)]) + run( + {"id": "r", "type": "accuracy", "selector": "a,b", "threshold": 0, "operator": operator}, + rec, + ) + assert rec.sql[0].count(f"{func}(`") == 2 and len(rec.sql) == 1 + + +def test_accuracy_multi_column_lower_bound_fails_on_low_column(): + res = run( + {"id": "r", "type": "accuracy", "selector": "a,b", "threshold": 5, "operator": ">="}, + Recorder([(10, 1)]), + ) + assert not res.passed and "'b'" in res.message and res.actual == "1" + + +def test_accuracy_multi_column_equality_uses_violation_counts(): + rec = Recorder([(0, 3)]) + res = run( + {"id": "r", "type": "accuracy", "selector": "a,b", "threshold": 1, "operator": "=="}, rec + ) + assert "SUM(CASE WHEN" in rec.sql[0] and "<>" in rec.sql[0] + assert not res.passed and res.actual == "3 violating row(s)" + + +def test_accuracy_multi_column_no_data_fails(): + res = run( + {"id": "r", "type": "accuracy", "selector": "a,b", "threshold": 0, "operator": ">="}, + Recorder([(1, None)]), + ) + assert not res.passed and "no data in 'b'" in res.message + + +# ------------------------------------- single-column rules over >1 columns + + +@pytest.mark.parametrize("rtype", ["uniqueness", "valid_values", "freshness"]) +def test_single_column_rule_over_many_columns_not_executed(rtype): + rec = Recorder() + res = run({"id": "r", "type": rtype, "selector": "a,b"}, rec) + assert res.passed is False and res.actual == "not executed" and rec.sql == [] + + +# ------------------------------------------------------- pattern SQL + + +@pytest.mark.parametrize( + "dialect,fragment", + [ + ("databricks", "NOT (`cc` RLIKE '^[A-Z]{2}$')"), + ("bigquery", "NOT REGEXP_CONTAINS(`cc`, '^[A-Z]{2}$')"), + ("snowflake", "REGEXP_INSTR(\"cc\", '^[A-Z]{2}$') = 0"), + ("ansi", "NOT regexp_matches(\"cc\", '^[A-Z]{2}$')"), + ("duckdb", "NOT regexp_matches(\"cc\", '^[A-Z]{2}$')"), + ], +) +def test_validity_pattern_sql_per_dialect(dialect, fragment): + sql = qe._validity_pattern_sql(T, "cc", "^[A-Z]{2}$", dialect) + assert fragment in sql and "IS NOT NULL" in sql + + +def test_validity_pattern_unknown_dialect_is_error_result(): + res = run( + {"id": "r", "type": "valid_values", "selector": "cc"}, Recorder(), "oracle", columns=COLS + ) + assert not res.passed and res.message.startswith("Error executing rule 'r'") + assert "no regular-expression test" in res.message + + +def test_valid_values_via_enum_runs_not_in_and_keeps_type(): + rec = Recorder([(2,)]) + res = run({"id": "r", "type": "valid_values", "selector": "status"}, rec, columns=COLS) + assert "NOT IN ('GOOD', 'ESTIMATED', 'SUSPECT')" in rec.sql[0] + assert res.rule_type == "valid_values" and not res.passed + + +def test_legacy_validity_type_is_preserved(): + res = run( + {"id": "r", "type": "validity", "selector": "s", "validValues": ["A"]}, Recorder([(0,)]) + ) + assert res.rule_type == "validity" and res.passed + + +def test_pattern_rule_via_column_regex(): + rec = Recorder([(0,)]) + res = run({"id": "r", "type": "valid_values", "selector": "cc"}, rec, columns=COLS) + assert "RLIKE" in rec.sql[0] and res.passed and res.rule_type == "valid_values" + + +# ------------------------------------------------------- references + + +def test_reference_without_resolver_not_executed(): + res = run({"id": "r", "type": "valid_values", "selector": "site_id"}, Recorder(), columns=COLS) + assert not res.passed and res.actual == "not executed" + + +def test_reference_resolver_lookup_error_message_is_surfaced(): + def resolver(product, column): + raise LookupError("nothing tagged") + + res = run( + {"id": "r", "type": "valid_values", "selector": "site_id"}, + Recorder(), + columns=COLS, + resolve_reference=resolver, + ) + assert res.actual == "not executed" and "nothing tagged" in res.message + + +def test_reference_runs_anti_join(): + rec = Recorder([(3,)]) + seen = [] + + def resolver(product, column): + seen.append((product, column)) + return "`p`.`s`.`sites`" + + res = run( + {"id": "r", "type": "valid_values", "selector": "site_id"}, + rec, + columns=COLS, + resolve_reference=resolver, + ) + assert seen == [("acme.sites", "site_id")] + assert ( + "NOT EXISTS (SELECT 1 FROM `p`.`s`.`sites` AS p WHERE p.`site_id` = c.`site_id`)" + in rec.sql[0] + ) + assert not res.passed and res.actual == "3 value(s) missing" + + +# ------------------------------------------------------- freshness + + +def test_freshness_databricks_sql(): + sql = qe._freshness_sql(T, "ts", "databricks") + assert "timestampdiff(SECOND, MAX(`ts`), current_timestamp())" in sql + + +# ------------------------------------------- backslash-safe string literals + + +def scan_literal(text): + """Reference scanner for the backslash-escaping grammar: decode one literal and + return (value, index just past the closing quote).""" + assert text[0] == "'" + out, i = [], 1 + while i < len(text): + ch = text[i] + if ch == "\\": + out.append(text[i + 1]) + i += 2 + elif ch == "'": + return "".join(out), i + 1 + else: + out.append(ch) + i += 1 + raise AssertionError("unterminated literal") + + +@pytest.mark.parametrize("dialect", ["databricks", "bigquery", "spark", "Databricks"]) +@pytest.mark.parametrize( + "value", + ["plain", "trailing\\", "it's", "\\' OR 1=1 --", "a\\\\b", "'", "\\", "x'); DROP TABLE t; --"], +) +def test_backslash_dialect_literal_roundtrips_and_cannot_break_out(dialect, value): + literal = quote_string_literal_for(dialect, value) + decoded, end = scan_literal(literal + " OR 1=1") + assert decoded == value + assert literal[end - 1] == "'" and literal[:end] == literal + + +def test_backslash_dialect_escape_shape(): + assert quote_string_literal_for("databricks", "a'b") == "'a\\'b'" + assert quote_string_literal_for("databricks", "a\\") == "'a\\\\'" + + +def test_other_dialect_delegates_to_existing_quoting(): + assert quote_string_literal_for("duckdb", "it's") == "'it''s'" + + +def test_non_string_value_rejected(): + with pytest.raises(ValueError): + quote_string_literal_for("databricks", 5) # type: ignore[arg-type] + + +# ── range validationRules and valid_values over several columns ───────────── + +from fluid_build.providers.quality_engine import ( # noqa: E402 + ValueDomain, + execute_quality_checks, + parse_range, + value_domain, +) + + +@pytest.mark.parametrize( + "constraint, bounds", + [ + (">= 0", [(">=", 0.0)]), + (">= 0 and <= 100", [(">=", 0.0), ("<=", 100.0)]), + ("between -1.5 and 2", [(">=", -1.5), ("<=", 2.0)]), + ("> 0 AND < 1", [(">", 0.0), ("<", 1.0)]), + ("0..100", []), + (">= zero", []), + (None, []), + ], +) +def test_parse_range(constraint, bounds): + assert parse_range(constraint) == bounds + + +ENERGY = [ + {"name": "grid_kwh", "validationRules": [{"type": "range", "constraint": ">= 0"}]}, + {"name": "solar_kwh", "validationRules": [{"type": "range", "constraint": ">= 0"}]}, + {"name": "quality", "validationRules": [{"type": "enum", "constraint": "GOOD,BAD"}]}, + {"name": "free_text"}, +] + + +def _recording(answers): + seen = [] + + def execute(sql): + seen.append(sql) + return [(answers.pop(0),)] + + return seen, execute + + +def test_range_rule_counts_values_outside_the_bounds_on_databricks(): + seen, execute = _recording([0]) + rule = {"id": "r", "type": "valid_values", "selector": "grid_kwh", "severity": "error"} + (res,) = execute_quality_checks([rule], "`c`.`s`.`t`", execute, "databricks", columns=ENERGY) + assert res.passed and res.expected == "0 values outside >= 0" + assert seen[0].endswith("WHERE `grid_kwh` IS NOT NULL AND NOT (`grid_kwh` >= 0.0)") + + +def test_valid_values_over_columns_checks_each_against_its_own_domain(): + seen, execute = _recording([0, 3, 1]) + rule = { + "id": "nonneg", + "type": "valid_values", + "selector": "grid_kwh, solar_kwh, quality", + "severity": "error", + } + (res,) = execute_quality_checks([rule], "t", execute, "databricks", columns=ENERGY) + assert not res.passed and res.rule_type == "valid_values" + assert len(seen) == 3 + assert "NOT (`solar_kwh` >= 0.0)" in seen[1] and "NOT IN ('GOOD', 'BAD')" in seen[2] + assert "'solar_kwh' (3 invalid value(s))" in res.message + assert "'quality' (1 invalid value(s))" in res.message + assert res.actual == "4 invalid value(s)" + + +def test_valid_values_over_columns_all_clean_passes(): + _, execute = _recording([0, 0]) + rule = {"id": "nonneg", "type": "valid_values", "selector": "grid_kwh,solar_kwh"} + (res,) = execute_quality_checks([rule], "t", execute, "databricks", columns=ENERGY) + assert res.passed and res.message == "Validity OK for all 2 columns" + assert res.actual == "0 invalid value(s)" + + +def test_valid_values_over_columns_with_an_undeclared_column_is_not_executed(): + _, execute = _recording([0]) + rule = {"id": "v", "type": "valid_values", "selector": "grid_kwh,free_text"} + (res,) = execute_quality_checks([rule], "t", execute, "databricks", columns=ENERGY) + assert not res.passed and res.actual == "not executed" + assert "nothing checked for 'free_text'" in res.message + + +def test_value_domain_is_the_rule_list_else_every_column_declaration(): + columns = [ + { + "name": "a", + "validationRules": [ + {"type": "range", "constraint": ">= 0"}, + {"type": "regex", "constraint": "^x$"}, + {"type": "enum", "constraint": "X | Y"}, + ], + "labels": {"references": "p.q.col"}, + }, + { + "name": "b", + "validationRules": [ + {"type": "range", "constraint": ">= 0"}, + {"type": "regex", "constraint": "^x$"}, + ], + }, + {"name": "c", "validationRules": [{"type": "range", "constraint": ">= 0"}]}, + {"name": "d", "labels": {"references": "p.q.col"}}, + ] + listed = {"id": "r", "selector": "a", "description": "a valid values: K, L."} + assert value_domain(listed, columns) == ValueDomain(values=("K", "L")) + assert value_domain({"selector": "a"}, columns) == ValueDomain( + values=("X", "Y"), pattern="^x$", bounds=((">=", 0.0),), reference=("p.q", "col") + ) + assert value_domain({"selector": "b"}, columns) == ValueDomain( + pattern="^x$", bounds=((">=", 0.0),) + ) + assert value_domain({"selector": "c"}, columns) == ValueDomain(bounds=((">=", 0.0),)) + assert value_domain({"selector": "d"}, columns) == ValueDomain(reference=("p.q", "col")) + assert value_domain({"selector": "zz"}, columns) == ValueDomain() + + +def test_every_column_declaration_is_checked_in_one_statement(): + # A column with an enum, a regex and a range: a value that breaks any of them is + # invalid (the range used to be dropped when a regex came first). + columns = [ + { + "name": "pct", + "validationRules": [ + {"type": "regex", "constraint": "^[0-9]+$"}, + {"type": "range", "constraint": ">= 0 and <= 100"}, + {"type": "enum", "constraint": "0,50,100,150"}, + ], + } + ] + seen, execute = _recording([1]) + rule = {"id": "p", "type": "valid_values", "selector": "pct", "severity": "error"} + (res,) = execute_quality_checks([rule], "t", execute, "databricks", columns=columns) + assert len(seen) == 1 + where = seen[0].split("WHERE", 1)[1] + assert "NOT IN ('0', '50', '100', '150')" in where + assert "NOT (`pct` RLIKE '^[0-9]+$')" in where + assert "NOT (`pct` >= 0.0 AND `pct` <= 100.0)" in where + assert where.count(" OR ") == 2 + assert not res.passed and res.actual == "1 invalid value(s)" + + +def test_a_column_with_values_and_a_reference_checks_both(): + columns = [ + { + "name": "site_id", + "validationRules": [{"type": "regex", "constraint": "^S[0-9]+$"}], + "labels": {"references": "acme.sites.site_id"}, + } + ] + seen, execute = _recording([0, 2]) + rule = {"id": "ref", "type": "valid_values", "selector": "site_id", "severity": "warn"} + (res,) = execute_quality_checks( + [rule], + "t", + execute, + "databricks", + columns=columns, + resolve_reference=lambda product, column: "`m`.`s`.`sites`", + ) + assert len(seen) == 2 and "RLIKE" in seen[0] and "NOT EXISTS" in seen[1] + assert not res.passed and "2 value(s) of 'site_id' are not in acme.sites.site_id" in res.message + assert res.actual == "0 invalid value(s); 2 value(s) missing" + + +def test_accuracy_without_a_threshold_is_not_run(): + seen, execute = _recording([]) + for selector in ("a", "a,b"): + rule = {"id": "acc", "type": "accuracy", "selector": selector, "severity": "error"} + (res,) = execute_quality_checks([rule], "t", execute, "databricks") + assert not res.passed and res.actual == "not executed" and "no threshold" in res.message + assert seen == [] + + +def test_parse_range_refuses_a_bound_too_large_for_a_float(): + assert parse_range(">= " + "9" * 400) == [] + assert parse_range("between 0 and " + "9" * 400) == [] diff --git a/tests/test_contract_val_branches.py b/tests/test_contract_val_branches.py index 7def2b4d..f1453ba6 100644 --- a/tests/test_contract_val_branches.py +++ b/tests/test_contract_val_branches.py @@ -328,18 +328,20 @@ def test_detect_databricks_provider(self, mock_schema, mock_load): } mock_result = MagicMock(is_valid=True, errors=[], warnings=[]) mock_schema.return_value.validate_contract.return_value = mock_result + from fluid_build.providers.databricks_validation import DatabricksValidationProvider + v = ContractValidator( contract_path=Path("test.yaml"), use_cache=False, track_history=False, check_data=False ) - report = v.validate() - # No live Unity Catalog check yet: a warning, never an "unknown provider" error - warnings = [ - i - for i in report.issues - if i.message - == "Live validation against Unity Catalog is not available for provider 'databricks' yet" - ] - assert len(warnings) == 1 + with ( + patch("fluid_build.cli.contract_validation.DATABRICKS_VALIDATION_AVAILABLE", True), + patch.object(DatabricksValidationProvider, "validate_connection", return_value=True), + ): + report = v.validate() + # The Unity Catalog provider, never an "unknown provider" error + assert report.provider_name == "databricks" + assert isinstance(v.validation_provider, DatabricksValidationProvider) + assert not any("Unknown provider" in i.message for i in report.issues) @patch("fluid_build.cli.contract_validation.load_contract_with_overlay") @patch("fluid_build.cli.contract_validation.FluidSchemaManager") diff --git a/tests/test_contract_validation_branches.py b/tests/test_contract_validation_branches.py index 46b5b843..fa904583 100644 --- a/tests/test_contract_validation_branches.py +++ b/tests/test_contract_validation_branches.py @@ -268,7 +268,10 @@ def test_unknown_provider(self, MockSM, mock_load): @patch("fluid_build.cli.contract_validation.load_contract_with_overlay") @patch("fluid_build.cli.contract_validation.FluidSchemaManager") - def test_databricks_provider_not_implemented(self, MockSM, mock_load): + def test_databricks_connection_failure_says_what_to_fix(self, MockSM, mock_load): + from fluid_build.providers.databricks.statements import StatementError + from fluid_build.providers.databricks_validation import DatabricksValidationProvider + contract = self._make_contract() contract["exposes"][0]["binding"]["platform"] = "databricks" mock_load.return_value = contract @@ -277,8 +280,16 @@ def test_databricks_provider_not_implemented(self, MockSM, mock_load): mock_sm.validate_contract.return_value = mock_result v = ContractValidator(Path("test.yaml"), use_cache=False, check_data=False) - report = v.validate() - assert any("Unity Catalog is not available" in w.message for w in report.get_warnings()) + no_warehouse = StatementError("DATABRICKS_WAREHOUSE_ID is not set") + with ( + patch("fluid_build.cli.contract_validation.DATABRICKS_VALIDATION_AVAILABLE", True), + patch.object( + DatabricksValidationProvider, "validate_connection", side_effect=no_warehouse + ), + ): + report = v.validate() + errors = [i for i in report.get_errors() if i.category == "connection"] + assert errors and "DATABRICKS_WAREHOUSE_ID is not set" in errors[0].message @patch("fluid_build.cli.contract_validation.load_contract_with_overlay") @patch("fluid_build.cli.contract_validation.FluidSchemaManager") diff --git a/tests/test_contract_validation_ext2.py b/tests/test_contract_validation_ext2.py index 21d0b716..040f52e7 100644 --- a/tests/test_contract_validation_ext2.py +++ b/tests/test_contract_validation_ext2.py @@ -332,12 +332,27 @@ def test_unknown_provider_adds_error(self): errors = v.report.get_errors() assert any("Unknown provider" in e.message for e in errors) - def test_databricks_adds_warning(self): + def test_databricks_without_the_sdk_is_an_error(self): v = _make_validator_new(provider_name="databricks") v.contract = {"exposes": [{"binding": {"platform": "databricks"}}]} - v._detect_and_validate_provider() - warnings = v.report.get_warnings() - assert any("databricks" in w.message.lower() for w in warnings) + with patch("fluid_build.cli.contract_validation.DATABRICKS_VALIDATION_AVAILABLE", False): + v._detect_and_validate_provider() + errors = v.report.get_errors() + assert any("databricks-sdk" in e.message for e in errors) + assert any("data-product-forge[databricks]" in (e.suggestion or "") for e in errors) + + def test_databricks_builds_the_unity_catalog_provider(self): + from fluid_build.providers.databricks_validation import DatabricksValidationProvider + + v = _make_validator_new(provider_name="databricks") + v.contract = {"exposes": [{"binding": {"platform": "databricks"}}]} + with ( + patch("fluid_build.cli.contract_validation.DATABRICKS_VALIDATION_AVAILABLE", True), + patch.object(DatabricksValidationProvider, "validate_connection", return_value=True), + ): + v._detect_and_validate_provider() + assert isinstance(v.validation_provider, DatabricksValidationProvider) + assert not v.report.get_errors() def test_azure_warns_that_it_was_not_validated(self): # `fluid validate` accepts azure (schema enum) although no provider serves