From ceb7d488f0e1ea9e90b2f24ff9f1dd4d6bafbbc5 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:51:45 +0100 Subject: [PATCH 1/6] feat(quality,databricks): fluid test runs every declared rule, live on Unity Catalog fluid test answered "Live validation against Unity Catalog is not available" for platform: databricks. It now reads the table's columns from the Unity Catalog tables API (no compute) and runs the row count and every dq rule on DATABRICKS_WAREHOUSE_ID (providers/databricks_validation.py), through the existing databricks_enforce.SqlRunner, which gains :name parameters. A table that does not exist runs no statement. The native engine runs what a contract already declares: a valid_values rule's values from its column's enum, regex or range validationRule or its labels.references (resolved on Databricks by the fluid_product_id tag, the tested table's catalog first); comma-listed columns for completeness, accuracy and valid_values; schema rules from the live column comparison. Every rule's result, passing ones included, carries what was measured. Fixed in passing: columns were quoted "col" on BigQuery and Databricks, where that is a string literal, and string literals doubled their quotes, which Databricks reads as adjacent literals ('it''s' becomes its). New _sql_safety.quote_string_literal_for(dialect, value). --- fluid_build/cli/contract_validation.py | 198 ++++- .../iac/providers/databricks_enforce.py | 29 +- fluid_build/providers/_sql_safety.py | 22 + fluid_build/providers/bigquery_validation.py | 2 + .../providers/databricks/statements.py | 141 ++++ .../providers/databricks_validation.py | 411 +++++++++ fluid_build/providers/local_validation.py | 3 + fluid_build/providers/quality_engine.py | 782 +++++++++++++++--- fluid_build/providers/snowflake_validation.py | 2 + fluid_build/providers/validation_provider.py | 13 + .../test_contract_validation_schema_rules.py | 285 +++++++ tests/providers/test_databricks_statements.py | 182 ++++ tests/providers/test_databricks_validation.py | 429 ++++++++++ .../test_quality_engine_column_rules.py | 467 +++++++++++ tests/test_contract_val_branches.py | 20 +- tests/test_contract_validation_branches.py | 17 +- tests/test_contract_validation_ext2.py | 23 +- 17 files changed, 2884 insertions(+), 142 deletions(-) create mode 100644 fluid_build/providers/databricks/statements.py create mode 100644 fluid_build/providers/databricks_validation.py create mode 100644 tests/cli/test_contract_validation_schema_rules.py create mode 100644 tests/providers/test_databricks_statements.py create mode 100644 tests/providers/test_databricks_validation.py create mode 100644 tests/providers/test_quality_engine_column_rules.py diff --git a/fluid_build/cli/contract_validation.py b/fluid_build/cli/contract_validation.py index b7053517..db219e45 100644 --- a/fluid_build/cli/contract_validation.py +++ b/fluid_build/cli/contract_validation.py @@ -87,17 +87,10 @@ def _dist_installed(dist_name: str) -> bool: SNOWFLAKE_VALIDATION_AVAILABLE = _dist_installed("snowflake-connector-python") AWS_VALIDATION_AVAILABLE = _dist_installed("boto3") LOCAL_VALIDATION_AVAILABLE = _dist_installed("duckdb") +DATABRICKS_VALIDATION_AVAILABLE = _dist_installed("databricks-sdk") #: Known platforms with no live validation provider: ``(warning, suggestion)``. -#: Databricks has no Unity Catalog introspection on this path yet (no SDK); its -#: offline checks are `fluid validate`'s, and `fluid apply --mode dry-run` is a -#: real `tofu plan` against the workspace. _NO_LIVE_VALIDATION = { - "databricks": ( - "Live validation against Unity Catalog is not available for provider 'databricks' yet", - "Run 'fluid validate' for the offline binding checks and " - "'fluid apply --mode dry-run' to plan against the workspace.", - ), # `fluid validate` accepts `azure` (the schema enum lists it) although there is # no azure provider: say so as a warning, so the two commands agree. "azure": ( @@ -110,6 +103,12 @@ def _dist_installed(dist_name: str) -> bool: COMMAND = "contract-validation" +def _rule_kind(rule: Any) -> str: + """A ``dq.rules[]`` entry's ``type``, lower-cased ("" when it has none).""" + kind = rule.get("type") if isinstance(rule, dict) else None + return kind.strip().lower() if isinstance(kind, str) else "" + + @dataclass class ValidationIssue: """Represents a validation issue found during contract validation.""" @@ -157,6 +156,11 @@ class CheckOutcome: message: str = "" expected: Optional[Any] = None actual: Optional[Any] = None + #: The data-quality rule this outcome is for (``dq.rules[].id`` / ``.type``) + #: and the expose it ran against, so a results table can key on them. + rule_id: Optional[str] = None + rule_type: Optional[str] = None + expose_id: Optional[str] = None @dataclass @@ -573,6 +577,22 @@ def _detect_and_validate_provider(self) -> None: from ..providers.local_validation import LocalValidationProvider self.validation_provider = LocalValidationProvider({"base_dir": base_dir}) + elif self.provider_name == "databricks": + if not DATABRICKS_VALIDATION_AVAILABLE: + self.report.add_issue( + "error", + "binding", + "databricks-sdk package is required for Databricks validation", + "binding.platform", + suggestion="Install it with: pip install 'data-product-forge[databricks]'", + ) + return + from ..providers.databricks_validation import DatabricksValidationProvider + + # Host, credentials and the SQL warehouse come from the + # environment (DATABRICKS_HOST, DATABRICKS_WAREHOUSE_ID), as for + # `fluid apply`; the contract names neither. + self.validation_provider = DatabricksValidationProvider({}) elif self.provider_name in _NO_LIVE_VALIDATION: message, suggestion = _NO_LIVE_VALIDATION[self.provider_name] self.report.add_issue( @@ -734,12 +754,23 @@ def _validate_single_expose(self, expose: Dict[str, Any], idx: int) -> None: # Also check nested contract.dq.rules pattern if not dq_rules: dq_rules = expose.get("contract", {}).get("dq", {}).get("rules", []) + # A ``schema`` rule ("the table conforms to the contract schema") is + # decided by the live column comparison below, not by a query. + schema_rules: List[Dict[str, Any]] = [] + if isinstance(dq_rules, list): + schema_rules = [r for r in dq_rules if _rule_kind(r) == "schema"] + dq_rules = [r for r in dq_rules if _rule_kind(r) != "schema"] if dq_rules and self.validation_provider and self.check_data: self._run_expose_quality_checks(expose, dq_rules, path_prefix) # Validate against actual resource (if provider available) if self.check_data and self.provider_name: - self._validate_against_actual_resource(expose, path_prefix) + first = len(self.report.issues) + compared = self._validate_against_actual_resource(expose, path_prefix) + if schema_rules and self.validation_provider: + self._decide_schema_rules( + expose_id, schema_rules, self.report.issues[first:], compared, path_prefix + ) def _validate_binding(self, binding: Dict[str, Any], path: str, expose_id: str) -> None: """Validate binding configuration.""" @@ -799,8 +830,22 @@ def _run_expose_quality_checks( """Execute data-quality rules against the live resource via the provider.""" expose_id = expose.get("id", "unknown") LOG.debug("Running %d quality checks for %s", len(rules), expose_id) + # A provider that hands back every rule's result (passes included) + # lets each outcome carry what was measured, not only the failures. + # ``ValidationProvider.run_quality_results`` answers ``None`` for a + # provider that only reports failures. + run_results = getattr(self.validation_provider, "run_quality_results", None) + results = None try: - issues = self.validation_provider.run_quality_checks(expose, rules) + if callable(run_results): + results = run_results(expose, rules) + if isinstance(results, list): + from fluid_build.providers.quality_engine import quality_results_to_issues + + issues = quality_results_to_issues(results) + else: + results = None + issues = self.validation_provider.run_quality_checks(expose, rules) except Exception as exc: LOG.warning("Quality check execution failed for %s: %s", expose_id, exc) self.report.add_issue( @@ -830,13 +875,82 @@ def _run_expose_quality_checks( documentation_url=issue.documentation_url, ) - self._record_quality_outcomes(expose_id, rules, issues) + self._record_quality_outcomes(expose_id, rules, issues, results) + + def _decide_schema_rules( + self, + expose_id: str, + rules: List[Dict[str, Any]], + resource_issues: List[ValidationIssue], + compared: bool, + path_prefix: str, + ) -> None: + """Decide each ``schema`` rule from the live column comparison. + + It passes when every contract column is in the table with its type + (the comparison's errors are a missing column or a changed type; an + extra column or a nullability difference does not break the rule). A + comparison that could not run is a failed rule, not a pass. + """ + missing_table = any(i.category == "missing_resource" for i in resource_issues) + broken = [ + i + for i in resource_issues + if i.category in ("missing_field", "type_mismatch") and _is_error_severity(i.severity) + ] + expected = "every contract column, with its type" + for rule in rules: + if not isinstance(rule, dict) or not rule.get("id"): + continue + rule_id = str(rule["id"]) + severity = rule.get("severity", "error") + if missing_table: + passed, actual = False, "table does not exist" + message = f"Rule '{rule_id}': the table does not exist, so no column could conform" + elif not compared: + passed, actual = False, "not executed" + message = ( + f"Rule '{rule_id}' is a schema rule, but the live table's columns could " + "not be read, so it was not checked" + ) + elif broken: + gone = sum(1 for i in broken if i.category == "missing_field") + actual = f"{gone} column(s) missing, {len(broken) - gone} type change(s)" + passed = False + message = f"Rule '{rule_id}': the table does not conform to the contract ({actual})" + else: + passed, actual = True, "all contract columns match" + message = "Schema conforms to the contract" + if not passed: + self.report.add_issue( + severity, + "quality", + message, + f"{path_prefix}.contract.dq.rules.{rule_id}", + expected=expected, + actual=actual, + ) + self.report.record_check( + CheckOutcome( + name=f"{expose_id}.{rule_id}", + category="quality", + passed=passed, + severity=severity, + message=message, + expected=expected, + actual=actual, + rule_id=rule_id, + rule_type="schema", + expose_id=expose_id, + ) + ) def _record_quality_outcomes( self, expose_id: str, rules: List[Dict[str, Any]], issues: List[Any], + results: Optional[List[Any]] = None, ) -> None: """Record one :class:`CheckOutcome` per DQ rule that actually ran. @@ -855,6 +969,10 @@ def _record_quality_outcomes( if not declared: return + if results is not None: + self._record_result_outcomes(expose_id, rules, results) + return + by_rule: Dict[str, Any] = {} for issue in issues: rule_id = (getattr(issue, "path", "") or "").rsplit(".", 1)[-1] @@ -881,6 +999,9 @@ def _record_quality_outcomes( f"{rule.get('type', 'rule')} check passed for " f"'{rule.get('selector', '')}'" ), + rule_id=str(rule_id), + rule_type=_rule_kind(rule) or None, + expose_id=expose_id, ) else: outcome = CheckOutcome( @@ -891,9 +1012,39 @@ def _record_quality_outcomes( message=issue.message, expected=issue.expected, actual=issue.actual, + rule_id=str(rule_id), + rule_type=_rule_kind(rule) or None, + expose_id=expose_id, ) self.report.record_check(outcome) + def _record_result_outcomes( + self, expose_id: str, rules: List[Dict[str, Any]], results: List[Any] + ) -> None: + """One :class:`CheckOutcome` per rule from the provider's full results, so a + passing rule carries what was measured too.""" + by_id = {getattr(r, "rule_id", None): r for r in results} + for rule in rules: + if not isinstance(rule, dict) or not rule.get("id"): + continue + result = by_id.get(rule["id"]) + if result is None: + continue + self.report.record_check( + CheckOutcome( + name=f"{expose_id}.{rule['id']}", + category="quality", + passed=bool(result.passed), + severity=result.severity or rule.get("severity", "error"), + message=result.message, + expected=result.expected, + actual=result.actual, + rule_id=str(rule["id"]), + rule_type=result.rule_type or _rule_kind(rule) or None, + expose_id=expose_id, + ) + ) + def _validate_schema_definition( self, schema: List[Dict[str, Any]], path: str, expose_id: str ) -> None: @@ -952,8 +1103,12 @@ def _validate_schema_definition( f"{col_path}.type", ) - def _validate_against_actual_resource(self, expose: Dict[str, Any], path: str) -> None: - """Validate contract schema against actual deployed resource.""" + def _validate_against_actual_resource(self, expose: Dict[str, Any], path: str) -> bool: + """Validate contract schema against actual deployed resource. + + Returns whether the live columns were compared with the contract (a + ``schema`` data-quality rule is decided by that comparison). + """ expose_id = expose.get("id") or expose.get("exposeId") or "unknown" LOG.debug("Validating actual resource for %s", expose_id) @@ -964,12 +1119,14 @@ def _validate_against_actual_resource(self, expose: Dict[str, Any], path: str) - location = binding.get("location", {}) props = location.get("properties", {}) self._validate_bigquery_resource(expose, path, props) + return True elif self.provider_name == "aws" and self._iceberg_outside_glue(expose, path): - return - elif self.provider_name in ("snowflake", "aws", "local"): - self._validate_generic_resource(expose, path) + return False + elif self.provider_name in ("snowflake", "aws", "local", "databricks"): + return self._validate_generic_resource(expose, path) else: LOG.debug("Provider '%s' resource validation not yet implemented", self.provider_name) + return False def _validate_bigquery_resource( self, expose: Dict[str, Any], path: str, props: Dict[str, Any] @@ -1128,8 +1285,9 @@ def _iceberg_outside_glue(self, expose: Dict[str, Any], path: str) -> bool: ) return True - def _validate_generic_resource(self, expose: Dict[str, Any], path: str) -> None: - """Validate a resource using the active validation provider (Snowflake, AWS, local).""" + def _validate_generic_resource(self, expose: Dict[str, Any], path: str) -> bool: + """Validate a resource with the active validation provider (Snowflake, AWS, + local, Databricks). Returns whether the comparison ran.""" if not self.validation_provider: self.report.add_issue( "warning", @@ -1137,7 +1295,7 @@ def _validate_generic_resource(self, expose: Dict[str, Any], path: str) -> None: "No validation provider initialized, skipping resource validation", path, ) - return + return False expose_id = expose.get("id") or expose.get("exposeId") or "unknown" @@ -1206,6 +1364,7 @@ def _validate_generic_resource(self, expose: Dict[str, Any], path: str) -> None: LOG.info("Resource '%s' validated successfully", expose_id) else: LOG.warning("Resource '%s' validation found issues", expose_id) + return True except Exception as e: self.report.add_issue( @@ -1215,6 +1374,7 @@ def _validate_generic_resource(self, expose: Dict[str, Any], path: str) -> None: path, suggestion="Check provider credentials and connectivity", ) + return False def _resource_cache_key(self, expose: Dict[str, Any], expose_id: str) -> str: """Cache identity for an expose's bound resource. diff --git a/fluid_build/iac/providers/databricks_enforce.py b/fluid_build/iac/providers/databricks_enforce.py index d78d8c11..e8cbd38b 100644 --- a/fluid_build/iac/providers/databricks_enforce.py +++ b/fluid_build/iac/providers/databricks_enforce.py @@ -281,21 +281,26 @@ def run(self, statement: str) -> None: """Run one statement to ``SUCCEEDED``; :class:`StatementFailed` otherwise.""" self.execute(statement) - def execute(self, statement: str) -> Mapping[str, Any]: - """:meth:`run`, and the finished statement's document (its ``result`` and ``manifest``).""" + def execute( + self, statement: str, parameters: Optional[Sequence[Mapping[str, Any]]] = None + ) -> Mapping[str, Any]: + """:meth:`run`, and the finished statement's document (its ``result`` and ``manifest``). + + ``parameters`` binds the statement's ``:name`` markers (``{"name", "value", "type"}`` + each), so a value never becomes statement text. + """ do = self._client.api_client.do doc: Any = None + body: Dict[str, Any] = { + "warehouse_id": self._warehouse, + "statement": statement, + "wait_timeout": "30s", + "on_wait_timeout": "CONTINUE", + } + if parameters: + body["parameters"] = [dict(p) for p in parameters] try: - doc = do( - "POST", - "/api/2.0/sql/statements/", - body={ - "warehouse_id": self._warehouse, - "statement": statement, - "wait_timeout": "30s", - "on_wait_timeout": "CONTINUE", - }, - ) + doc = do("POST", "/api/2.0/sql/statements/", body=body) deadline = time.monotonic() + self._timeout while self._state(doc) in ("PENDING", "RUNNING"): if time.monotonic() > deadline: diff --git a/fluid_build/providers/_sql_safety.py b/fluid_build/providers/_sql_safety.py index b282325f..470f5bdd 100644 --- a/fluid_build/providers/_sql_safety.py +++ b/fluid_build/providers/_sql_safety.py @@ -67,6 +67,28 @@ def quote_string_literal(value: str, *, backslash_escapes: bool = True) -> str: return "'" + escaped.replace("'", "''") + "'" +# Dialects that take a quote inside ``'...'`` only as ``\'``. Spark SQL +# (Databricks) reads ``'it''s'`` as the adjacent literals ``'it'`` and ``'s'`` +# and joins them into ``its`` (checked on a SQL warehouse); BigQuery rejects +# it. Both honour backslash escapes, so ``\\`` is doubled first. +_BACKSLASH_QUOTE_DIALECTS = frozenset({"bigquery", "databricks", "spark"}) + + +def quote_string_literal_for(dialect: str | None, value: str) -> str: + """Quote ``value`` as a string literal of ``dialect`` that reads back as ``value``. + + :func:`quote_string_literal` for every dialect that doubles a quote, and + ``'...\\'...'`` for the ones in :data:`_BACKSLASH_QUOTE_DIALECTS`: with + every backslash doubled before the quotes are escaped, no value can end + the literal early on either. + """ + if not isinstance(value, str): + raise ValueError(f"Invalid SQL string literal: {value!r}") + if (dialect or "").strip().lower() in _BACKSLASH_QUOTE_DIALECTS: + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" + return quote_string_literal(value, backslash_escapes=dialect_uses_backslash_escapes(dialect)) + + def quote_ansi_string_literal(value: str) -> str: """Quote a SQL string literal for a *standard-conforming* dialect. diff --git a/fluid_build/providers/bigquery_validation.py b/fluid_build/providers/bigquery_validation.py index 3c660dfa..a352b4b7 100644 --- a/fluid_build/providers/bigquery_validation.py +++ b/fluid_build/providers/bigquery_validation.py @@ -17,6 +17,7 @@ from fluid_build.providers._sql_safety import validate_ident from fluid_build.providers.quality_engine import ( execute_quality_checks, + expose_columns, quality_results_to_issues, ) from fluid_build.providers.validation_provider import ( @@ -416,5 +417,6 @@ def _exec(sql): table_ref=table_ref, execute_fn=_exec, dialect="bigquery", + columns=expose_columns(resource_spec), ) return quality_results_to_issues(results) diff --git a/fluid_build/providers/databricks/statements.py b/fluid_build/providers/databricks/statements.py new file mode 100644 index 00000000..5c206473 --- /dev/null +++ b/fluid_build/providers/databricks/statements.py @@ -0,0 +1,141 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""Rows from a Databricks SQL warehouse, for ``fluid test``. + +Statements run through ``iac/providers/databricks_enforce.SqlRunner``, the Statement +Execution API runner ``fluid apply`` already uses for masks and row filters: it polls a +long statement, cancels one that times out or is interrupted, and scrubs every error. +This module adds what reading results needs: ``:name`` parameters, and the rows of an +``INLINE`` ``JSON_ARRAY`` result (the API's defaults), converted by each column's +``type_name``, since every value arrives as a string, across every chunk of the result. + +The client is ``databricks_enforce.default_client()``, ``WorkspaceClient`` built from +``databricks_auth.sdk_config`` (the policy ``fluid apply`` and ``fluid verify`` share); +the SDK is the ``[databricks]`` extra, imported only when a client is made. +""" + +from __future__ import annotations + +import os +import time +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence +from urllib.parse import quote + +from fluid_build.iac.providers.databricks_enforce import ( + DATABRICKS_EXTRA_INSTALL, + WAREHOUSE_ENV, + SqlRunner, + default_client, + scrub, +) +from fluid_build.iac.providers.databricks_enforce import StatementFailed as StatementError + +STATEMENTS_PATH = "/api/2.0/sql/statements" + +#: Converters for the ``type_name`` of a ``JSON_ARRAY`` result column. +_INTEGER_TYPES = frozenset({"BYTE", "SHORT", "INT", "LONG"}) +_FLOAT_TYPES = frozenset({"FLOAT", "DOUBLE", "DECIMAL"}) + +#: A guard against a result that never stops naming a next chunk. +_MAX_CHUNKS = 10_000 + + +def warehouse_id_from_env(env: Optional[Mapping[str, str]] = None) -> str: + """The SQL warehouse id from ``DATABRICKS_WAREHOUSE_ID``, or a fix-it error.""" + value = str((env if env is not None else os.environ).get(WAREHOUSE_ENV) or "").strip() + if not value: + raise StatementError( + f"{WAREHOUSE_ENV} is not set, so there is no SQL warehouse to run the checks on. " + "Set it to the id of a SQL warehouse you can use (SQL Warehouses > the " + "warehouse > Connection details)." + ) + return value + + +def param(name: str, value: Any, type_name: str = "STRING") -> Dict[str, Any]: + """A ``parameters`` entry binding ``value`` to ``:name`` as ``type_name``. + + ``None`` binds NULL (the API reads an omitted value as NULL). + """ + entry: Dict[str, Any] = {"name": name, "type": type_name} + if value is not None: + entry["value"] = str(value) + return entry + + +def _convert(value: Any, type_name: str) -> Any: + if value is None: + return None + try: + if type_name in _INTEGER_TYPES: + return int(value) + if type_name in _FLOAT_TYPES: + return float(value) + if type_name == "BOOLEAN": + return str(value).lower() == "true" + except (TypeError, ValueError): + return value + return value + + +class StatementRunner: + """Run statements on one warehouse and return their rows as tuples.""" + + def __init__( + self, + client: Any, + warehouse_id: str, + *, + timeout_seconds: float = 300.0, + poll_seconds: float = 1.0, + sleep: Callable[[float], None] = time.sleep, + ) -> None: + self._client = client + self._sql = SqlRunner( + client, warehouse_id, timeout=timeout_seconds, poll=poll_seconds, sleep=sleep + ) + + def run( + self, sql: str, parameters: Optional[Sequence[Mapping[str, Any]]] = None + ) -> List[tuple]: + """Run ``sql`` (``:name`` markers bound from ``parameters``) and return its rows.""" + doc = self._sql.execute(sql, parameters) + columns = ((doc.get("manifest") or {}).get("schema") or {}).get("columns") or [] + types = [str(c.get("type_name") or "").upper() for c in columns] + statement_id = str(doc.get("statement_id") or "") + result = doc.get("result") or {} + rows: List[tuple] = [] + for _ in range(_MAX_CHUNKS): + for raw in result.get("data_array") or []: + rows.append( + tuple( + _convert(v, types[i] if i < len(types) else "") for i, v in enumerate(raw) + ) + ) + index = result.get("next_chunk_index") + if index is None or not statement_id: + break + result = self._chunk(statement_id, int(index)) + return rows + + def _chunk(self, statement_id: str, index: int) -> Mapping[str, Any]: + path = f"{STATEMENTS_PATH}/{quote(statement_id, safe='')}/result/chunks/{index}" + try: + out = self._client.api_client.do("GET", path) + except Exception as exc: # noqa: BLE001 - every transport failure is the statement's + raise StatementError(f"{type(exc).__name__}: {scrub(exc)}") from None + return out if isinstance(out, dict) else {} + + +__all__ = [ + "DATABRICKS_EXTRA_INSTALL", + "STATEMENTS_PATH", + "StatementError", + "StatementRunner", + "WAREHOUSE_ENV", + "default_client", + "param", + "scrub", + "warehouse_id_from_env", +] diff --git a/fluid_build/providers/databricks_validation.py b/fluid_build/providers/databricks_validation.py new file mode 100644 index 00000000..efc3fbfa --- /dev/null +++ b/fluid_build/providers/databricks_validation.py @@ -0,0 +1,411 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``fluid test`` against a Unity Catalog table (``binding.platform: databricks``). + +What is read, and what it costs: + +- **Columns**: ``GET /api/2.1/unity-catalog/tables/{catalog.schema.table}``, a + metadata call that starts no compute. A 404 means the table does not exist. + The contract's types go through the emitter's own FLUID → Databricks type + table (``engines/dbt/_types.sql_type``) and both sides through the + provider's normaliser, so ``VARCHAR`` matches the ``string`` that + ``fluid apply`` created. +- **Row count and data-quality rules**: SQL on ``DATABRICKS_WAREHOUSE_ID`` + through the Statement Execution API (``databricks/statements.py``), one + ``SELECT`` per rule; a rule over several columns is still one statement. + Nothing is written. + +The table is ``binding.location`` ``database`` (the Unity Catalog catalog, the +convention ``fluid apply`` uses), ``schema`` and ``table``; every part is +validated before it reaches a URL or a statement, and backtick-quoted in SQL. + +Sign-in is ``databricks_auth.sdk_config`` (``DATABRICKS_HOST`` plus a token, +an OAuth client or a named profile), the policy ``fluid apply`` and +``fluid verify`` share. +""" + +from __future__ import annotations + +import os +from typing import Any, Callable, Dict, List, Optional, Tuple +from urllib.parse import quote + +from fluid_build.providers._sql_safety import validate_ident +from fluid_build.providers.quality_engine import ( + QualityCheckResult, + execute_quality_checks, + expose_columns, + quality_results_to_issues, +) +from fluid_build.providers.validation_provider import ( + FieldSchema, + ResourceSchema, + ResourceType, + ValidationIssue, + ValidationProvider, + ValidationResult, +) + +TABLES_PATH = "/api/2.1/unity-catalog/tables" + +#: The table tag ``fluid apply`` puts a product's id in (``iac/providers/databricks.py`` +#: ``_PRODUCT_TAGS``); a reference to another product finds its table by it. +PRODUCT_TAG = "fluid_product_id" +WAREHOUSES_PATH = "/api/2.0/sql/warehouses" + + +def _is_not_found(exc: BaseException) -> bool: + for attr in ("status_code", "status", "http_status"): + if getattr(exc, attr, None) == 404: + return True + if type(exc).__name__ in {"NotFound", "ResourceDoesNotExist"}: + return True + code = str(getattr(exc, "error_code", "") or "") + return code in {"TABLE_DOES_NOT_EXIST", "RESOURCE_DOES_NOT_EXIST", "NOT_FOUND"} + + +def table_name(resource_spec: Dict[str, Any]) -> Optional[Tuple[str, str, str]]: + """``(catalog, schema, table)`` of an expose, each validated, or ``None``. + + Raises ``ValueError`` for a part that is not a plain identifier. + """ + binding = resource_spec.get("binding") or {} + location = binding.get("location") if isinstance(binding, dict) else None + if not isinstance(location, dict): + return None + catalog = location.get("database") or location.get("catalog") + schema = location.get("schema") + table = location.get("table") + if not (catalog and schema and table): + return None + return validate_ident(str(catalog)), validate_ident(str(schema)), validate_ident(str(table)) + + +def sql_table_ref(parts: Tuple[str, str, str]) -> str: + """```catalog`.`schema`.`table``` for validated parts.""" + return ".".join(f"`{validate_ident(p)}`" for p in parts) + + +def _not_run(rule: Dict[str, Any], full_name: str) -> QualityCheckResult: + """A rule on a table that does not exist: failed at its severity, never run.""" + rule_id = str(rule.get("id") or "unnamed") + return QualityCheckResult( + rule_id=rule_id, + rule_type=str(rule.get("type") or "").lower(), + selector=str(rule.get("selector") or ""), + passed=False, + severity=rule.get("severity", "warning"), + message=f"Rule '{rule_id}' was not checked: table {full_name} does not exist", + expected="the table to exist", + actual="not executed", + ) + + +class DatabricksValidationProvider(ValidationProvider): + """Validate an expose against its live Unity Catalog table.""" + + def __init__(self, config: Dict[str, Any]): + super().__init__(config) + self._client_factory: Optional[Callable[[], Any]] = config.get("client_factory") + self._warehouse_id: Optional[str] = config.get("warehouse_id") + self._client: Any = None + self._runner: Any = None + self._references: Dict[Tuple[str, str, str], Any] = {} + self._tables: Dict[str, Optional[Dict[str, Any]]] = {} + + @property + def provider_name(self) -> str: + return "databricks" + + # ── plumbing ──────────────────────────────────────────────────────── + + def _get_client(self) -> Any: + if self._client is None: + from fluid_build.providers.databricks.statements import default_client + + self._client = (self._client_factory or default_client)() + return self._client + + def _sql(self): + if self._runner is None: + from fluid_build.providers.databricks.statements import ( + StatementRunner, + warehouse_id_from_env, + ) + + warehouse = self._warehouse_id or warehouse_id_from_env(os.environ) + self._runner = StatementRunner(self._get_client(), warehouse) + return self._runner + + def _table(self, parts: Tuple[str, str, str]) -> Optional[Dict[str, Any]]: + """The table's Unity Catalog metadata, read once per run; ``None`` when it + does not exist. A metadata read, so no compute starts.""" + full_name = ".".join(parts) + if full_name not in self._tables: + try: + self._tables[full_name] = self._get(f"{TABLES_PATH}/{quote(full_name, safe='')}") + except Exception as exc: # noqa: BLE001 + if not _is_not_found(exc): + from fluid_build.providers.databricks.statements import StatementError, scrub + + raise StatementError(f"{type(exc).__name__}: {scrub(exc)}") from None + self._tables[full_name] = None + return self._tables[full_name] + + def _get(self, path: str) -> Dict[str, Any]: + out = self._get_client().api_client.do("GET", path) + return out if isinstance(out, dict) else {} + + # ── ValidationProvider ────────────────────────────────────────────── + + def validate_connection(self) -> bool: + """Sign in and find the warehouse; neither starts compute. + + Raises with a message that says what to fix: no host or credential, + no ``DATABRICKS_WAREHOUSE_ID``, or a warehouse this identity cannot see. + """ + from fluid_build.providers.databricks.statements import ( + StatementError, + scrub, + warehouse_id_from_env, + ) + + warehouse = self._warehouse_id or warehouse_id_from_env(os.environ) + try: + self._get(f"{WAREHOUSES_PATH}/{quote(warehouse, safe='')}") + except Exception as exc: # noqa: BLE001 - one safe message per failure + if _is_not_found(exc): + raise StatementError( + f"SQL warehouse {warehouse!r} was not found in the workspace " + "DATABRICKS_HOST names, or this identity cannot use it" + ) from None + raise StatementError(f"{type(exc).__name__}: {scrub(exc)}") from None + return True + + def get_resource_schema(self, resource_spec: Dict[str, Any]) -> Optional[ResourceSchema]: + parts = table_name(resource_spec) + if parts is None: + raise ValueError( + "binding.location needs database (the Unity Catalog catalog), schema and table" + ) + full_name = ".".join(parts) + table = self._table(parts) + if table is None: + return None + from fluid_build.iac.providers.databricks import normalize_column_type + + fields: List[FieldSchema] = [] + for col in table.get("columns") or []: + if not isinstance(col, dict) or not col.get("name"): + continue + raw = col.get("type_text") or col.get("type_name") or "" + fields.append( + FieldSchema( + name=str(col["name"]), + type=normalize_column_type(raw), + mode="NULLABLE" if col.get("nullable", True) is not False else "REQUIRED", + description=col.get("comment"), + ) + ) + rows = self._sql().run(f"SELECT COUNT(*) FROM {sql_table_ref(parts)}") + row_count = int(rows[0][0]) if rows and rows[0] and rows[0][0] is not None else None + return ResourceSchema( + resource_type=ResourceType.TABLE, + fully_qualified_name=full_name, + fields=fields, + row_count=row_count, + metadata={"table_type": table.get("table_type")}, + ) + + def validate_resource( + self, + contract_spec: Dict[str, Any], + actual_schema: Optional[ResourceSchema], + ) -> ValidationResult: + issues: List[ValidationIssue] = [] + resource_name = contract_spec.get("id") or contract_spec.get("exposeId") or "unknown" + if actual_schema is None: + parts = table_name(contract_spec) + name = ".".join(parts) if parts else "unknown" + issues.append( + ValidationIssue( + severity="error", + category="missing_resource", + message=f"Table '{name}' does not exist in Unity Catalog", + path="exposes[].binding.location", + expected=name, + actual=None, + suggestion="Run 'fluid apply' for this contract, or correct " + "binding.location (database is the catalog)", + ) + ) + return ValidationResult(resource_name, False, issues, None) + + expected = self._expected_fields(contract_spec) + if expected: + issues.extend(self.compare_schemas(expected, actual_schema.fields)) + + if actual_schema.row_count == 0: + issues.append( + ValidationIssue( + severity="warning", + category="empty_table", + message=f"Table '{actual_schema.fully_qualified_name}' has no rows", + path="exposes[].binding.location", + expected="> 0 rows", + actual="0 rows", + suggestion="Run the product's build, then test again", + ) + ) + sla = contract_spec.get("sla") or {} + min_rows = sla.get("row_count_min") if isinstance(sla, dict) else None + if min_rows is not None and actual_schema.row_count is not None: + if actual_schema.row_count < min_rows: + issues.append( + ValidationIssue( + severity="error", + category="row_count_below_threshold", + message=( + f"Table has {actual_schema.row_count} rows, below minimum of " + f"{min_rows}" + ), + path="exposes[].sla.row_count_min", + expected=f">= {min_rows} rows", + actual=f"{actual_schema.row_count} rows", + suggestion="Check data pipeline for issues", + ) + ) + return ValidationResult( + resource_name=resource_name, + success=not any(i.severity == "error" for i in issues), + issues=issues, + schema=actual_schema, + ) + + def normalize_type(self, provider_type: str) -> str: + from fluid_build.iac.providers.databricks import normalize_column_type + + return normalize_column_type(provider_type) + + @staticmethod + def _expected_fields(resource_spec: Dict[str, Any]) -> List[FieldSchema]: + """The contract's columns, typed as ``fluid apply`` creates them.""" + from fluid_build.engines.dbt._types import sql_type + + fields: List[FieldSchema] = [] + for col in expose_columns(resource_spec): + if "required" in col: + mode = "REQUIRED" if col["required"] else "NULLABLE" + elif "nullable" in col: + mode = "NULLABLE" if col["nullable"] else "REQUIRED" + else: + mode = str(col.get("mode") or "NULLABLE").upper() + fields.append( + FieldSchema( + name=str(col.get("name", "")), + type=sql_type(str(col.get("type") or "string"), "databricks"), + mode=mode, + description=col.get("description"), + ) + ) + return fields + + # ── data-quality rules ────────────────────────────────────────────── + + def run_quality_results( + self, resource_spec: Dict[str, Any], rules: List[Dict[str, Any]] + ) -> List[QualityCheckResult]: + """Every rule's result, passing ones included (with what was measured). + + A table that does not exist runs no statement: each rule is reported as not + executed, and the missing table itself is the resource check's error. + """ + parts = table_name(resource_spec) + if parts is None: + raise ValueError( + "binding.location needs database (the Unity Catalog catalog), schema and table" + ) + if self._table(parts) is None: + return [_not_run(rule, ".".join(parts)) for rule in rules if isinstance(rule, dict)] + return execute_quality_checks( + rules=rules, + table_ref=sql_table_ref(parts), + execute_fn=self._sql().run, + dialect="databricks", + columns=expose_columns(resource_spec), + resolve_reference=lambda product, column: self._reference_table( + product, column, prefer_catalog=parts[0] + ), + ) + + def _reference_table( + self, product_id: str, column: str, prefer_catalog: Optional[str] = None + ) -> str: + """The live table of ``product_id`` that has ``column``, as a SQL reference. + + Found by the ``fluid_product_id`` tag ``fluid apply`` puts on every table it + makes (one ``system.information_schema.table_tags`` query, which lists only + what this identity can see), then by the column, from each candidate's + Unity Catalog metadata. Tables in ``prefer_catalog`` (the tested table's + catalog) win over tables elsewhere, so a look-alike tagged table in another + catalog cannot stand in for the product's own. Raises ``LookupError`` saying + why when there is no such table, or more than one. + """ + key = (product_id, column.lower(), (prefer_catalog or "").lower()) + if key not in self._references: + self._references[key] = self._find_reference(product_id, column, prefer_catalog) + found = self._references[key] + if isinstance(found, LookupError): + raise found + return found + + def _find_reference( + self, product_id: str, column: str, prefer_catalog: Optional[str] = None + ) -> Any: + from fluid_build.providers.databricks.statements import param + + rows = self._sql().run( + "SELECT catalog_name, schema_name, table_name " + "FROM system.information_schema.table_tags " + "WHERE tag_name = :tag AND tag_value = :product", + [param("tag", PRODUCT_TAG), param("product", product_id)], + ) + candidates = sorted({tuple(str(v) for v in row[:3]) for row in rows if len(row) >= 3}) + having = [] + for parts in candidates: + try: + table = self._get(f"{TABLES_PATH}/{quote('.'.join(parts), safe='')}") + except Exception: # noqa: BLE001 - a table this identity cannot read is no match + continue + names = {str(c.get("name", "")).lower() for c in table.get("columns") or []} + if column.lower() in names: + having.append(parts) + local = [p for p in having if prefer_catalog and p[0].lower() == prefer_catalog.lower()] + if local: + having = local + if not having: + return LookupError( + f"no table tagged {PRODUCT_TAG}={product_id} with a column {column!r} is " + "visible to this identity (has that product been applied?)" + ) + if len(having) > 1: + listed = ", ".join(".".join(p) for p in having[:3]) + return LookupError( + f"{len(having)} tables tagged {PRODUCT_TAG}={product_id} have a column " + f"{column!r} ({listed}), so which one it references is ambiguous" + ) + try: + return sql_table_ref(having[0]) + except ValueError: + return LookupError( + f"the table of {product_id} has a name that is not a plain identifier" + ) + + def run_quality_checks( + self, resource_spec: Dict[str, Any], rules: List[Dict[str, Any]] + ) -> List[ValidationIssue]: + return quality_results_to_issues(self.run_quality_results(resource_spec, rules)) + + +__all__ = ["DatabricksValidationProvider", "sql_table_ref", "table_name"] diff --git a/fluid_build/providers/local_validation.py b/fluid_build/providers/local_validation.py index 4ab17792..00be176c 100644 --- a/fluid_build/providers/local_validation.py +++ b/fluid_build/providers/local_validation.py @@ -20,6 +20,7 @@ from fluid_build.providers._sql_safety import quote_ansi_string_literal, validate_ident from fluid_build.providers.quality_engine import ( execute_quality_checks, + expose_columns, quality_results_to_issues, ) from fluid_build.providers.validation_provider import ( @@ -432,6 +433,7 @@ def _exec(sql): table_ref=table_ref, execute_fn=_exec, dialect="ansi", + columns=expose_columns(resource_spec), ) return quality_results_to_issues(results) finally: @@ -465,6 +467,7 @@ def _exec(sql): table_ref=table_ref, execute_fn=_exec, dialect="ansi", + columns=expose_columns(resource_spec), ) return quality_results_to_issues(results) finally: diff --git a/fluid_build/providers/quality_engine.py b/fluid_build/providers/quality_engine.py index 13d96719..e30ee386 100644 --- a/fluid_build/providers/quality_engine.py +++ b/fluid_build/providers/quality_engine.py @@ -19,12 +19,9 @@ import logging import re from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Tuple -from fluid_build.providers._sql_safety import ( - dialect_uses_backslash_escapes, - quote_string_literal, -) +from fluid_build.providers._sql_safety import quote_string_literal_for from fluid_build.providers.validation_provider import ValidationIssue LOG = logging.getLogger("fluid.providers.quality_engine") @@ -39,6 +36,163 @@ def _validate_ident(name: str) -> str: return name +# Dialects that read ``"x"`` as a string literal, not as a column: an +# identifier there needs backticks. ``COUNT(DISTINCT "col")`` on Databricks +# or BigQuery counts one constant, so a uniqueness or completeness rule +# quoted the ANSI way passes whatever the data holds. +_BACKTICK_DIALECTS = frozenset({"bigquery", "databricks"}) + +# The floating-point type the ratio casts use. ``DOUBLE PRECISION`` is ANSI, +# DuckDB and Snowflake; Spark SQL and BigQuery do not accept it. +_DOUBLE_TYPES = {"bigquery": "FLOAT64", "databricks": "DOUBLE"} + + +def _dialect(dialect: Optional[str]) -> str: + return (dialect or "ansi").strip().lower() + + +def _qi(column: str, dialect: Optional[str] = "ansi") -> str: + """``column`` validated and quoted as an identifier of ``dialect``.""" + col = _validate_ident(column) + return f"`{col}`" if _dialect(dialect) in _BACKTICK_DIALECTS else f'"{col}"' + + +def _double(dialect: Optional[str]) -> str: + return _DOUBLE_TYPES.get(_dialect(dialect), "DOUBLE PRECISION") + + +def rule_columns(selector: Any) -> List[str]: + """The columns a rule's ``selector`` names, in order. + + ``dqRule.selector`` is one column, ``"*"``, or a comma-separated list of + columns (the industry packs write ``site_id,meter_id,reading_ts`` for + "all mandatory attributes"). Completeness and accuracy rules hold for + every listed column; the other column rules take exactly one. + """ + if not isinstance(selector, str): + return [] + return [part.strip() for part in selector.split(",") if part.strip()] + + +def _column_constraint(columns: Any, name: str, kind: str) -> Optional[str]: + """The ``constraint`` of the first ``validationRules[]`` entry of ``kind`` on + the contract column called ``name`` (case-insensitive), if any.""" + if not isinstance(columns, (list, tuple)) or not isinstance(name, str): + return None + for col in columns: + if not isinstance(col, dict) or str(col.get("name", "")).lower() != name.lower(): + continue + for entry in col.get("validationRules") or []: + if not isinstance(entry, dict): + continue + if str(entry.get("type") or "").strip().lower() != kind: + continue + constraint = entry.get("constraint") + if isinstance(constraint, str) and constraint.strip(): + return constraint.strip() + return None + + +def expose_columns(expose: Any) -> List[Dict[str, Any]]: + """The expose's declared columns (``contract.schema``, else ``schema``).""" + if not isinstance(expose, dict): + return [] + contract = expose.get("contract") + columns = contract.get("schema") if isinstance(contract, dict) else None + if not columns: + columns = expose.get("schema") + return [c for c in columns if isinstance(c, dict)] if isinstance(columns, list) else [] + + +def extract_reference(rule: Dict[str, Any], columns: Any = None) -> Optional[Tuple[str, str]]: + """``(product id, column)`` that a ``valid_values`` rule's column references. + + Used when the rule names no values and its column declares no ``enum`` or + ``regex``: the column's ``labels.references: .`` (the + foreign-key convention the Unity Catalog keys read too) makes a valid value + one that the referenced product's column holds. + """ + name = rule.get("selector") or "" + if not isinstance(columns, (list, tuple)) or not isinstance(name, str): + return None + for col in columns: + if not isinstance(col, dict) or str(col.get("name", "")).lower() != name.lower(): + continue + labels = col.get("labels") + target = labels.get("references") if isinstance(labels, dict) else None + if isinstance(target, str): + product, _, column = target.strip().rpartition(".") + if product and column: + return product, column + return None + + +_NUMBER = r"-?\d+(?:\.\d+)?" +_RANGE_BETWEEN = re.compile(rf"^\s*between\s+({_NUMBER})\s+and\s+({_NUMBER})\s*$", re.IGNORECASE) +_RANGE_BOUND = re.compile(rf"^\s*(>=|<=|>|<)\s*({_NUMBER})\s*$") +_RANGE_AND = re.compile(r"\s+and\s+", re.IGNORECASE) + + +def parse_range(constraint: Any) -> List[Tuple[str, float]]: + """The bounds of a ``range`` validationRule: ``">= 0"``, ``">= 0 and <= 100"`` + or ``"between 0 and 100"`` (``[]`` for anything else).""" + if not isinstance(constraint, str): + return [] + m = _RANGE_BETWEEN.match(constraint) + if m: + return [(">=", float(m.group(1))), ("<=", float(m.group(2)))] + bounds = [] + for part in _RANGE_AND.split(constraint.strip()): + m = _RANGE_BOUND.match(part) + if not m: + return [] + bounds.append((m.group(1), float(m.group(2)))) + return bounds + + +@dataclass(frozen=True) +class ValueDomain: + """What a valid value of one column is: the first declaration found of + the rule's own values, the column's ``enum``, ``regex`` or ``range`` + validationRule, or its ``labels.references``.""" + + values: Tuple[str, ...] = () + pattern: Optional[str] = None + bounds: Tuple[Tuple[str, float], ...] = () + reference: Optional[Tuple[str, str]] = None + + +def value_domain(rule: Dict[str, Any], columns: Any = None, column: Optional[str] = None): + """The :class:`ValueDomain` of ``column`` (default: the rule's selector).""" + sub = rule if column is None else {**rule, "selector": column} + values = extract_valid_values(sub, columns) + if values: + return ValueDomain(values=tuple(values)) + pattern = extract_value_pattern(sub, columns) + if pattern: + return ValueDomain(pattern=pattern) + bounds = parse_range(_column_constraint(columns, sub.get("selector") or "", "range")) + if bounds: + return ValueDomain(bounds=tuple(bounds)) + return ValueDomain(reference=extract_reference(sub, columns)) + + +#: ``resolve_reference(product id, column) -> table reference for SQL``; raises +#: ``LookupError`` (with a message that says why) when the table cannot be found. +ReferenceResolver = Callable[[str, str], str] + + +def extract_value_pattern(rule: Dict[str, Any], columns: Any = None) -> Optional[str]: + """The regular expression a ``valid_values`` rule's column must match. + + Used only when the rule names no allowed values (see + :func:`extract_valid_values`): the selected column's own ``regex`` + validationRule is then what "a valid value" means + (``country_code: ^[A-Z]{2}$``). + """ + return _column_constraint(columns, rule.get("selector") or "", "regex") + + @dataclass class QualityCheckResult: """Result of a single data quality check.""" @@ -58,61 +212,136 @@ class QualityCheckResult: # ------------------------------------------------------------------ -def _completeness_sql(table_ref: str, column: str) -> str: - """Generate SQL to compute non-null fraction for a column.""" - col = _validate_ident(column) +def _completeness_expr(column: str, dialect: str) -> str: return ( - "SELECT " - f'CAST(SUM(CASE WHEN "{col}" IS NOT NULL THEN 1 ELSE 0 END) AS DOUBLE PRECISION) ' - "/ NULLIF(COUNT(*), 0) AS completeness_ratio " - f"FROM {table_ref}" + f"CAST(SUM(CASE WHEN {_qi(column, dialect)} IS NOT NULL THEN 1 ELSE 0 END) " + f"AS {_double(dialect)}) / NULLIF(COUNT(*), 0)" ) -def _uniqueness_sql(table_ref: str, column: str) -> str: +def _completeness_sql(table_ref: str, column: str, dialect: str = "ansi") -> str: + """Generate SQL to compute non-null fraction for a column.""" + return f"SELECT {_completeness_expr(column, dialect)} AS completeness_ratio FROM {table_ref}" + + +def _completeness_columns_sql(table_ref: str, columns: List[str], dialect: str) -> str: + """One statement: the non-null fraction of each of ``columns``, in order.""" + ratios = ", ".join(_completeness_expr(c, dialect) for c in columns) + return f"SELECT {ratios} FROM {table_ref}" + + +def _uniqueness_sql(table_ref: str, column: str, dialect: str = "ansi") -> str: """Generate SQL to compute distinct-value fraction for a column.""" - col = _validate_ident(column) + col = _qi(column, dialect) return ( "SELECT " - f'CAST(COUNT(DISTINCT "{col}") AS DOUBLE PRECISION) ' - f'/ NULLIF(COUNT("{col}"), 0) AS uniqueness_ratio ' + f"CAST(COUNT(DISTINCT {col}) AS {_double(dialect)}) " + f"/ NULLIF(COUNT({col}), 0) AS uniqueness_ratio " f"FROM {table_ref}" ) -def _accuracy_min_sql(table_ref: str, column: str) -> str: +def _accuracy_min_sql(table_ref: str, column: str, dialect: str = "ansi") -> str: """Generate SQL to get the minimum value of a numeric column.""" - col = _validate_ident(column) - return f'SELECT MIN("{col}") AS min_val FROM {table_ref}' + return f"SELECT MIN({_qi(column, dialect)}) AS min_val FROM {table_ref}" -def _accuracy_max_sql(table_ref: str, column: str) -> str: +def _accuracy_max_sql(table_ref: str, column: str, dialect: str = "ansi") -> str: """Generate SQL to get the maximum value of a numeric column.""" - col = _validate_ident(column) - return f'SELECT MAX("{col}") AS max_val FROM {table_ref}' + return f"SELECT MAX({_qi(column, dialect)}) AS max_val FROM {table_ref}" + + +def _accuracy_bound_columns_sql( + table_ref: str, columns: List[str], aggregate: str, dialect: str +) -> str: + """One statement: ``MIN`` (or ``MAX``) of each of ``columns``, in order.""" + func = "MAX" if aggregate == "max" else "MIN" + bounds = ", ".join(f"{func}({_qi(c, dialect)})" for c in columns) + return f"SELECT {bounds} FROM {table_ref}" + + +def _violation_predicate(column: str, operator: str, threshold, dialect: str) -> str: + col = _qi(column, dialect) + negated = _NEGATED_OPERATORS[operator] + # ``threshold`` is schema-typed ``number``; coerce so nothing but a + # numeric literal can reach the SQL text. + literal = float(threshold) + return f"{col} IS NOT NULL AND {col} {negated} {literal}" -def _accuracy_violations_sql(table_ref: str, column: str, operator: str, threshold) -> str: +def _accuracy_violations_sql( + table_ref: str, column: str, operator: str, threshold, dialect: str = "ansi" +) -> str: """Count rows whose value does NOT satisfy ``value threshold``. Used for equality/inequality bounds, where no single aggregate can decide the rule: ``MIN(col) == t`` says nothing about the rest of the column. """ - col = _validate_ident(column) - negated = _NEGATED_OPERATORS[operator] - # ``threshold`` is schema-typed ``number``; coerce so nothing but a - # numeric literal can reach the SQL text. - literal = float(threshold) return ( f"SELECT COUNT(*) AS violation_count FROM {table_ref} " - f'WHERE "{col}" IS NOT NULL AND "{col}" {negated} {literal}' + f"WHERE {_violation_predicate(column, operator, threshold, dialect)}" + ) + + +def _accuracy_violations_columns_sql( + table_ref: str, columns: List[str], operator: str, threshold, dialect: str +) -> str: + """One statement: the violating-row count of each of ``columns``, in order.""" + counts = ", ".join( + f"SUM(CASE WHEN {_violation_predicate(c, operator, threshold, dialect)} " + "THEN 1 ELSE 0 END)" + for c in columns + ) + return f"SELECT {counts} FROM {table_ref}" + + +def _validity_pattern_sql(table_ref: str, column: str, pattern: str, dialect: str) -> str: + """Count non-null values of ``column`` that do not match ``pattern`` anywhere. + + Each dialect's search-anywhere regex test (Snowflake's ``REGEXP_LIKE`` + matches the whole value, so ``REGEXP_INSTR`` stands in for it); an + anchored pattern (``^[A-Z]{2}$``) means the same on all of them. + """ + col = _qi(column, dialect) + literal = quote_string_literal_for(dialect, pattern) + name = _dialect(dialect) + if name == "databricks": + mismatch = f"NOT ({col} RLIKE {literal})" + elif name == "bigquery": + mismatch = f"NOT REGEXP_CONTAINS({col}, {literal})" + elif name == "snowflake": + mismatch = f"REGEXP_INSTR({col}, {literal}) = 0" + elif name in ("ansi", "duckdb"): + mismatch = f"NOT regexp_matches({col}, {literal})" + else: + raise ValueError(f"no regular-expression test for SQL dialect {dialect!r}") + return ( + f"SELECT COUNT(*) AS invalid_count FROM {table_ref} " + f"WHERE {col} IS NOT NULL AND {mismatch}" + ) + + +def _validity_range_sql( + table_ref: str, column: str, bounds: Tuple[Tuple[str, float], ...], dialect: str +) -> str: + """Count non-null values of ``column`` outside the declared bounds.""" + col = _qi(column, dialect) + inside = " AND ".join(f"{col} {op} {float(n)!r}" for op, n in bounds if op in _OPERATORS) + return ( + f"SELECT COUNT(*) AS invalid_count FROM {table_ref} " + f"WHERE {col} IS NOT NULL AND NOT ({inside})" ) def _freshness_sql(table_ref: str, column: str, dialect: str = "ansi") -> str: """Generate SQL to get age of most-recent timestamp value (in seconds).""" col = _validate_ident(column) + if dialect == "databricks": + return ( + f"SELECT timestampdiff(SECOND, MAX(`{col}`), current_timestamp()) AS age_seconds " + f"FROM {table_ref}" + ) if dialect == "snowflake": return ( f"SELECT DATEDIFF('second', MAX(\"{col}\"), CURRENT_TIMESTAMP()) AS age_seconds " @@ -222,7 +451,7 @@ def _parse_duration_seconds(value: str) -> Optional[int]: parse_duration_seconds = _parse_duration_seconds -def extract_valid_values(rule: Dict[str, Any]) -> List[str]: +def extract_valid_values(rule: Dict[str, Any], columns: Any = None) -> List[str]: """Allowed values for a ``validity`` / ``valid_values`` rule. ``$defs.dqRule`` is ``additionalProperties: false`` and has no @@ -230,6 +459,12 @@ def extract_valid_values(rule: Dict[str, Any]) -> List[str]: the description: ``"COLUMN valid values: A, B, C."``. An explicit ``validValues`` list still wins when present (hand-written files). + With ``columns`` (the expose's ``contract.schema``), a rule that names no + values takes them from its column's own ``enum`` validationRule + (``constraint: "GOOD | ESTIMATED | SUSPECT"``, ``|`` or ``,`` separated): + that is where a contract already declares them, and where the industry + packs do. + Shared by the native engine and the SodaCL exporter so the two cannot disagree about which values a rule allows. """ @@ -237,12 +472,16 @@ def extract_valid_values(rule: Dict[str, Any]) -> List[str]: if explicit: return [str(v) for v in explicit] description = rule.get("description") or "" - if not isinstance(description, str) or " valid values:" not in description.lower(): - return [] - m = re.search(r"valid values:\s*([^.]+)", description, re.IGNORECASE) - if not m: - return [] - return [v.strip() for v in m.group(1).split(",") if v.strip()] + if isinstance(description, str) and " valid values:" in description.lower(): + m = re.search(r"valid values:\s*([^.]+)", description, re.IGNORECASE) + if m: + values = [v.strip() for v in m.group(1).split(",") if v.strip()] + if values: + return values + enum = _column_constraint(columns, rule.get("selector") or "", "enum") + if enum: + return [v.strip() for v in re.split(r"[|,]", enum) if v.strip()] + return [] # ------------------------------------------------------------------ @@ -255,6 +494,8 @@ def execute_quality_checks( table_ref: str, execute_fn, dialect: str = "ansi", + columns: Any = None, + resolve_reference: Optional[ReferenceResolver] = None, ) -> List[QualityCheckResult]: """ Execute a list of DQ rules against a live table. @@ -269,7 +510,15 @@ def execute_quality_checks( ``execute_fn(sql) -> list[tuple]`` — runs a SQL statement and returns the result rows. dialect : str - SQL dialect hint: ``"ansi"``, ``"snowflake"``, ``"bigquery"``. + SQL dialect hint: ``"ansi"``, ``"snowflake"``, ``"bigquery"``, + ``"databricks"``. + columns : list[dict], optional + The expose's ``contract.schema``. A ``valid_values`` rule that names + no values reads its column's ``enum`` (or ``regex``) validationRule, or + its ``labels.references``. + resolve_reference : callable, optional + ``(product id, column) -> table reference`` of a referenced product; + without it a reference rule is reported as not executed. Returns ------- @@ -303,22 +552,26 @@ def execute_quality_checks( # Support validValues from explicit key OR parsed from description # Description format: "column valid values: val1, val2, val3." # Shared with the SodaCL exporter so both engines read the same list. - explicit_vv = extract_valid_values(rule) - - result = _execute_single_rule( - rule_id=rule_id, - rule_type=rule_type, - selector=selector, - severity=severity, - description=description, - threshold=threshold, - operator=operator, - valid_values=explicit_vv, - window=rule.get("window", rule.get("freshness")), - table_ref=table_ref, - execute_fn=execute_fn, - dialect=dialect, - ) + if rule_type in _VALIDITY_TYPES and len(rule_columns(selector)) > 1: + result = _check_validity_columns( + rule, columns, table_ref, execute_fn, dialect, resolve_reference + ) + else: + result = _execute_single_rule( + rule_id=rule_id, + rule_type=rule_type, + selector=selector, + severity=severity, + description=description, + threshold=threshold, + operator=operator, + domain=value_domain(rule, columns), + resolve_reference=resolve_reference, + window=rule.get("window", rule.get("freshness")), + table_ref=table_ref, + execute_fn=execute_fn, + dialect=dialect, + ) results.append(result) except Exception as e: results.append( @@ -373,6 +626,13 @@ def quality_results_to_issues( } ) +# Rule types whose comma-separated selector means "every listed column" (a +# valid_values rule checks each listed column against its own declaration). +_MULTI_COLUMN_RULE_TYPES = frozenset( + {"completeness", "accuracy", "anomaly_detection", "validity", "valid_values"} +) +_VALIDITY_TYPES = frozenset({"validity", "valid_values"}) + # Rule types ``$defs.dqRule.type`` accepts but this engine cannot run. # They are reported as failures at the rule's declared severity so a gate # nobody is enforcing can never read as green. @@ -388,13 +648,34 @@ def _execute_single_rule( description: str, threshold, operator: str, - valid_values: List[str], + domain: ValueDomain, window: Optional[str], table_ref: str, execute_fn, dialect: str, + resolve_reference: Optional[ReferenceResolver] = None, ) -> QualityCheckResult: + columns = rule_columns(selector) + if len(columns) > 1 and rule_type not in _MULTI_COLUMN_RULE_TYPES: + # ``uniqueness`` over a list would mean a composite key and + # ``valid_values`` / ``freshness`` one column's domain or clock; + # guessing either would check something the author did not write. + return QualityCheckResult( + rule_id=rule_id, + rule_type=rule_type, + selector=selector, + passed=False, + severity=severity, + message=( + f"Rule '{rule_id}' is a {rule_type} rule over {len(columns)} columns; " + f"a {rule_type} rule takes one column, so nothing was checked. " + "Write one rule per column." + ), + expected="one column", + actual="not executed", + ) + if rule_type == "completeness": return _check_completeness( rule_id, @@ -405,6 +686,7 @@ def _execute_single_rule( operator, table_ref, execute_fn, + dialect, ) elif rule_type == "uniqueness": return _check_uniqueness( @@ -416,6 +698,7 @@ def _execute_single_rule( operator, table_ref, execute_fn, + dialect, ) elif rule_type == "accuracy": return _check_accuracy( @@ -427,17 +710,21 @@ def _execute_single_rule( operator, table_ref, execute_fn, + dialect, ) - elif rule_type == "validity": + elif rule_type in _VALIDITY_TYPES: + # ``valid_values`` is the schema's name; ``validity`` the older alias. return _check_validity( rule_id, selector, severity, description, - valid_values, + domain, table_ref, execute_fn, dialect, + rule_type=rule_type, + resolve_reference=resolve_reference, ) elif rule_type == "freshness": return _check_freshness( @@ -461,17 +748,6 @@ def _execute_single_rule( operator, table_ref, execute_fn, - ) - elif rule_type == "valid_values": - # Alias for validity — checks that column values are in the validValues list - return _check_validity( - rule_id, - selector, - severity, - description, - valid_values, - table_ref, - execute_fn, dialect, ) elif rule_type in _UNIMPLEMENTED_RULE_TYPES: @@ -528,8 +804,23 @@ def _check_completeness( operator, table_ref, execute_fn, + dialect: str = "ansi", ) -> QualityCheckResult: - sql = _completeness_sql(table_ref, selector) + columns = rule_columns(selector) + if len(columns) > 1: + return _check_completeness_columns( + rule_id, + selector, + columns, + severity, + description, + threshold, + operator, + table_ref, + execute_fn, + dialect, + ) + sql = _completeness_sql(table_ref, selector, dialect) rows = execute_fn(sql) ratio = rows[0][0] if rows and rows[0][0] is not None else 0.0 passed = _compare(ratio, threshold, operator) if threshold is not None else ratio == 1.0 @@ -549,6 +840,59 @@ def _check_completeness( ) +def _row_values(rows, width: int) -> List[Any]: + """The first result row padded to ``width`` (an empty result reads as NULLs).""" + first = list(rows[0]) if rows else [] + return [first[i] if i < len(first) else None for i in range(width)] + + +def _listed(pairs: List[tuple], fmt) -> str: + """``'a' (x), 'b' (y)`` for at most five pairs, then how many more.""" + shown = ", ".join(f"'{name}' ({fmt(value)})" for name, value in pairs[:5]) + return shown + (f" and {len(pairs) - 5} more" if len(pairs) > 5 else "") + + +def _check_completeness_columns( + rule_id, + selector, + columns: List[str], + severity, + description, + threshold, + operator, + table_ref, + execute_fn, + dialect: str, +) -> QualityCheckResult: + """A completeness rule over several columns: each must meet the bound. + + One statement measures every column. The observed value is the lowest + ratio, and the message names each column below the bound. + """ + rows = execute_fn(_completeness_columns_sql(table_ref, columns, dialect)) + ratios = [float(v) if v is not None else 0.0 for v in _row_values(rows, len(columns))] + + def meets(ratio: float) -> bool: + return _compare(ratio, threshold, operator) if threshold is not None else ratio == 1.0 + + failing = [(c, r) for c, r in zip(columns, ratios, strict=True) if not meets(r)] + return QualityCheckResult( + rule_id=rule_id, + rule_type="completeness", + selector=selector, + passed=not failing, + severity=severity, + message=( + f"{description or rule_id} — completeness is below the bound for " + f"{_listed(failing, lambda r: f'{r:.2%}')}" + if failing + else f"Completeness OK for all {len(columns)} columns" + ), + expected=f"{operator} {threshold}" if threshold is not None else "1.0", + actual=f"{min(ratios):.4f}", + ) + + def _check_uniqueness( rule_id, selector, @@ -558,8 +902,9 @@ def _check_uniqueness( operator, table_ref, execute_fn, + dialect: str = "ansi", ) -> QualityCheckResult: - sql = _uniqueness_sql(table_ref, selector) + sql = _uniqueness_sql(table_ref, selector, dialect) rows = execute_fn(sql) ratio = rows[0][0] if rows and rows[0][0] is not None else 0.0 passed = _compare(ratio, threshold, operator) if threshold is not None else ratio == 1.0 @@ -588,13 +933,28 @@ def _check_accuracy( operator, table_ref, execute_fn, + dialect: str = "ansi", ) -> QualityCheckResult: + columns = rule_columns(selector) + if len(columns) > 1: + return _check_accuracy_columns( + rule_id, + selector, + columns, + severity, + description, + threshold, + operator, + table_ref, + execute_fn, + dialect, + ) # A bound rule asserts something about EVERY row, so the aggregate # that decides it depends on the direction of the bound. Always # asking for MIN() made every upper bound ('<=', '<') trivially true # and every equality bound undecidable. if threshold is not None and operator in _NEGATED_OPERATORS: - sql = _accuracy_violations_sql(table_ref, selector, operator, threshold) + sql = _accuracy_violations_sql(table_ref, selector, operator, threshold, dialect) rows = execute_fn(sql) violations = rows[0][0] if rows and rows[0][0] is not None else 0 passed = int(violations) == 0 @@ -617,9 +977,9 @@ def _check_accuracy( use_max = operator in _UPPER_BOUND_OPERATORS aggregate = "max" if use_max else "min" sql = ( - _accuracy_max_sql(table_ref, selector) + _accuracy_max_sql(table_ref, selector, dialect) if use_max - else _accuracy_min_sql(table_ref, selector) + else _accuracy_min_sql(table_ref, selector, dialect) ) rows = execute_fn(sql) bound_val = rows[0][0] if rows and rows[0][0] is not None else None @@ -649,55 +1009,153 @@ def _check_accuracy( ) +def _check_accuracy_columns( + rule_id, + selector, + columns: List[str], + severity, + description, + threshold, + operator, + table_ref, + execute_fn, + dialect: str, +) -> QualityCheckResult: + """An accuracy rule over several columns: every column must hold the bound. + + One statement measures every column, with the same aggregate the + single-column rule uses (``MIN`` for a lower bound, ``MAX`` for an upper + one, a violating-row count for ``==`` / ``!=``). A column with no values + fails as "no data", as it does on its own. + """ + if threshold is not None and operator in _NEGATED_OPERATORS: + rows = execute_fn( + _accuracy_violations_columns_sql(table_ref, columns, operator, threshold, dialect) + ) + counts = [int(v) if v is not None else 0 for v in _row_values(rows, len(columns))] + failing = [(c, n) for c, n in zip(columns, counts, strict=True) if n] + return QualityCheckResult( + rule_id=rule_id, + rule_type="accuracy", + selector=selector, + passed=not failing, + severity=severity, + message=( + f"{description or rule_id} — rows that do not satisfy {operator} {threshold} " + f"in {_listed(failing, str)}" + if failing + else f"Accuracy OK for all {len(columns)} columns" + ), + expected=f"0 rows violating {operator} {threshold}", + actual=f"{sum(counts)} violating row(s)", + ) + + aggregate = "max" if operator in _UPPER_BOUND_OPERATORS else "min" + rows = execute_fn(_accuracy_bound_columns_sql(table_ref, columns, aggregate, dialect)) + bounds = _row_values(rows, len(columns)) + empty = [c for c, v in zip(columns, bounds, strict=True) if v is None] + failing = [ + (c, v) + for c, v in zip(columns, bounds, strict=True) + if v is not None and threshold is not None and not _compare(v, threshold, operator) + ] + problems = [] + if failing: + problems.append(f"{aggregate} value is outside the bound for {_listed(failing, str)}") + if empty: + problems.append("no data in " + ", ".join(f"'{c}'" for c in empty[:5])) + present = [v for v in bounds if v is not None] + worst = None + if present: + try: + worst = (max if aggregate == "max" else min)(float(v) for v in present) + except (TypeError, ValueError): + worst = None + return QualityCheckResult( + rule_id=rule_id, + rule_type="accuracy", + selector=selector, + passed=not problems, + severity=severity, + message=( + f"{description or rule_id} — " + "; ".join(problems) + if problems + else f"Accuracy OK for all {len(columns)} columns" + ), + expected=f"{operator} {threshold}" if threshold is not None else "pass", + actual=None if worst is None else f"{worst:g}", + ) + + def _check_validity( rule_id, selector, severity, description, - valid_values, + domain: ValueDomain, table_ref, execute_fn, dialect: str = "ansi", + *, + rule_type: str = "validity", + resolve_reference: Optional[ReferenceResolver] = None, ) -> QualityCheckResult: - if not valid_values: + if domain.values: + col = _qi(selector, dialect) + # The literals are contract-supplied, so the quoting must hold on the + # *target* dialect (a value ending in a backslash must not end the + # literal early and append live SQL to this predicate). + escaped = ", ".join(quote_string_literal_for(dialect, str(v)) for v in domain.values) + sql = ( + f"SELECT COUNT(*) AS invalid_count FROM {table_ref} " + f"WHERE {col} IS NOT NULL AND {col} NOT IN ({escaped})" + ) + expected = "0 invalid values" + elif domain.pattern: + sql = _validity_pattern_sql(table_ref, selector, domain.pattern, dialect) + expected = f"0 values not matching {domain.pattern}" + elif domain.bounds: + sql = _validity_range_sql(table_ref, selector, domain.bounds, dialect) + expected = "0 values outside " + " and ".join(f"{op} {n:g}" for op, n in domain.bounds) + elif domain.reference is not None: + return _check_reference( + rule_id, + selector, + severity, + description, + domain.reference, + resolve_reference, + table_ref, + execute_fn, + dialect, + rule_type, + ) + else: # A misconfigured gate is a failed gate. Forcing 'warning' here # overrode the author's declared severity, so a rule declared # `severity: error` that checks nothing at all exited 0. return QualityCheckResult( rule_id=rule_id, - rule_type="validity", + rule_type=rule_type, selector=selector, passed=False, severity=severity, message=( f"Rule '{rule_id}' is a valid_values rule but declares no allowed " "values, so nothing was checked. Declare them in the description " - f"(\"{selector or 'COLUMN'} valid values: A, B, C.\") or as a " - "'validValues' list." + f"(\"{selector or 'COLUMN'} valid values: A, B, C.\"), as an enum, regex " + "or range validationRule on the column, as the column's " + "labels.references, or as a 'validValues' list." ), expected="a non-empty list of allowed values", actual="no allowed values declared", ) - col = _validate_ident(selector) - # Build SQL with quoted string literals for valid values. The literals are - # contract-supplied, so the quoting must hold on the *target* dialect: - # Snowflake/BigQuery honour ``\`` escapes inside ``'...'``, and a value - # ending in a backslash would otherwise terminate the literal early and - # append live SQL to this predicate (silently flipping the check to pass). - backslash_escapes = dialect_uses_backslash_escapes(dialect) - escaped = ", ".join( - quote_string_literal(str(v), backslash_escapes=backslash_escapes) for v in valid_values - ) - sql = ( - f"SELECT COUNT(*) AS invalid_count FROM {table_ref} " - f'WHERE "{col}" IS NOT NULL AND "{col}" NOT IN ({escaped})' - ) rows = execute_fn(sql) invalid_count = rows[0][0] if rows and rows[0][0] is not None else 0 - passed = invalid_count == 0 + passed = int(invalid_count) == 0 return QualityCheckResult( rule_id=rule_id, - rule_type="validity", + rule_type=rule_type, selector=selector, passed=passed, severity=severity, @@ -706,11 +1164,136 @@ def _check_validity( if not passed else f"Validity OK for '{selector}'" ), - expected="0 invalid values", + expected=expected, actual=f"{invalid_count} invalid value(s)", ) +_NOT_RUN = ("not executed", "no allowed values declared") + + +def _check_validity_columns( + rule: Dict[str, Any], + columns: Any, + table_ref, + execute_fn, + dialect: str, + resolve_reference: Optional[ReferenceResolver], +) -> QualityCheckResult: + """A valid_values rule over several columns: each column against its own + declared values (the rule's list, else each column's validationRule or + reference). Passes when every column was checked and holds no invalid value.""" + rule_id = rule.get("id", "unnamed") + rule_type = rule.get("type", "").lower() + selector = rule.get("selector", "") + severity = rule.get("severity", "warning") + names = rule_columns(selector) + parts = [ + _check_validity( + rule_id, + name, + severity, + rule.get("description", ""), + value_domain(rule, columns, name), + table_ref, + execute_fn, + dialect, + rule_type=rule_type, + resolve_reference=resolve_reference, + ) + for name in names + ] + skipped = [n for n, r in zip(names, parts, strict=True) if r.actual in _NOT_RUN] + failing = [ + (n, r.actual) + for n, r in zip(names, parts, strict=True) + if not r.passed and n not in skipped + ] + problems = [] + if failing: + problems.append("invalid values in " + _listed(failing, str)) + if skipped: + problems.append("nothing checked for " + ", ".join(f"'{n}'" for n in skipped[:5])) + total = 0 + for result in parts: + head = str(result.actual or "").split(" ", 1)[0] + total += int(head) if head.isdigit() else 0 + return QualityCheckResult( + rule_id=rule_id, + rule_type=rule_type, + selector=selector, + passed=not problems, + severity=severity, + message=( + f"{rule.get('description') or rule_id} — " + "; ".join(problems) + if problems + else f"Validity OK for all {len(names)} columns" + ), + expected="0 invalid values in every column", + actual="not executed" if skipped else f"{total} invalid value(s)", + ) + + +def _check_reference( + rule_id, + selector, + severity, + description, + reference: Tuple[str, str], + resolve_reference: Optional[ReferenceResolver], + table_ref, + execute_fn, + dialect: str, + rule_type: str, +) -> QualityCheckResult: + """Every non-null value of ``selector`` exists in the referenced column.""" + product, column = reference + target = f"{product}.{column}" + expected = f"every value in {target}" + + def not_run(why: str) -> QualityCheckResult: + return QualityCheckResult( + rule_id=rule_id, + rule_type=rule_type, + selector=selector, + passed=False, + severity=severity, + message=f"Rule '{rule_id}' checks '{selector}' against {target}, but {why}, " + "so nothing was checked", + expected=expected, + actual="not executed", + ) + + if resolve_reference is None: + return not_run("this platform cannot look up another product's table") + try: + parent = resolve_reference(product, column) + except LookupError as exc: + return not_run(str(exc)) + col, pcol = _qi(selector, dialect), _qi(column, dialect) + sql = ( + f"SELECT COUNT(*) AS missing_count FROM {table_ref} AS c " + f"WHERE c.{col} IS NOT NULL AND NOT EXISTS " + f"(SELECT 1 FROM {parent} AS p WHERE p.{pcol} = c.{col})" + ) + rows = execute_fn(sql) + missing = int(rows[0][0]) if rows and rows[0][0] is not None else 0 + return QualityCheckResult( + rule_id=rule_id, + rule_type=rule_type, + selector=selector, + passed=missing == 0, + severity=severity, + message=( + f"{description or rule_id} — {missing} value(s) of '{selector}' are not in {target}" + if missing + else f"Every '{selector}' value is in {target}" + ), + expected=f"0 values missing from {target}", + actual=f"{missing} value(s) missing", + ) + + def _check_freshness( rule_id, selector, @@ -790,6 +1373,7 @@ def _check_anomaly_detection( operator, table_ref, execute_fn, + dialect: str = "ansi", ) -> QualityCheckResult: """Anomaly detection via row-count check (selector '*') or MIN/MAX check for a column.""" if selector == "*" or selector == "": @@ -817,5 +1401,13 @@ def _check_anomaly_detection( # Column-level anomaly: delegate to the bound check, which picks # MIN or MAX (or a violating-row count) from the operator. return _check_accuracy( - rule_id, selector, severity, description, threshold, operator, table_ref, execute_fn + rule_id, + selector, + severity, + description, + threshold, + operator, + table_ref, + execute_fn, + dialect, ) diff --git a/fluid_build/providers/snowflake_validation.py b/fluid_build/providers/snowflake_validation.py index 5a4c331f..f69fd914 100644 --- a/fluid_build/providers/snowflake_validation.py +++ b/fluid_build/providers/snowflake_validation.py @@ -17,6 +17,7 @@ from fluid_build.providers._sql_safety import validate_ident from fluid_build.providers.quality_engine import ( execute_quality_checks, + expose_columns, quality_results_to_issues, ) from fluid_build.providers.validation_provider import ( @@ -454,5 +455,6 @@ def run_quality_checks( table_ref=table_ref, execute_fn=conn.execute, dialect="snowflake", + columns=expose_columns(resource_spec), ) return quality_results_to_issues(results) diff --git a/fluid_build/providers/validation_provider.py b/fluid_build/providers/validation_provider.py index 1bbe1892..7a909e5b 100644 --- a/fluid_build/providers/validation_provider.py +++ b/fluid_build/providers/validation_provider.py @@ -322,3 +322,16 @@ def run_quality_checks( path="contract.dq.rules", ) ] + + def run_quality_results( + self, + resource_spec: Dict[str, Any], + rules: List[Dict[str, Any]], + ) -> Optional[List[Any]]: + """Every rule's ``QualityCheckResult``, the passing ones included. + + ``None`` (the default) means this provider reports failures only, + through :meth:`run_quality_checks`. A provider that returns the full + list lets a report record what each passing rule measured, too. + """ + return None diff --git a/tests/cli/test_contract_validation_schema_rules.py b/tests/cli/test_contract_validation_schema_rules.py new file mode 100644 index 00000000..6af4e4c4 --- /dev/null +++ b/tests/cli/test_contract_validation_schema_rules.py @@ -0,0 +1,285 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""ContractValidator: Databricks provider wiring, ``schema`` dq rules decided from the +live column comparison, and CheckOutcomes that carry rule id/type/expose id.""" + +from __future__ import annotations + +from datetime import datetime + +import pytest + +from fluid_build.cli import contract_validation as cv +from fluid_build.providers.quality_engine import QualityCheckResult +from fluid_build.providers.validation_provider import ( + FieldSchema, + ResourceSchema, + ResourceType, + ValidationProvider, + ValidationResult, +) + +CONTRACT_COLUMNS = [ + {"name": "a", "type": "string"}, + {"name": "b", "type": "int"}, +] + + +class FakeProvider(ValidationProvider): + """Deterministic provider: canned live schema, canned quality results.""" + + def __init__(self, live=None, results="unset", issues=None, schema_error=None): + super().__init__({}) + self.live = live + self.results = results + self.issues = issues or [] + self.schema_error = schema_error + self.quality_calls = [] + + @property + def provider_name(self): + return "databricks" + + def validate_connection(self): + return True + + def get_resource_schema(self, resource_spec): + if self.schema_error: + raise self.schema_error + return self.live + + def validate_resource(self, contract_spec, actual_schema): + if actual_schema is None: + from fluid_build.providers.validation_provider import ValidationIssue + + return ValidationResult( + "x", + False, + [ + ValidationIssue( + severity="error", + category="missing_resource", + message="Table does not exist", + path="p", + ) + ], + None, + ) + expected = [FieldSchema(c["name"], c["type"], "NULLABLE") for c in contract_spec["schema"]] + issues = self.compare_schemas(expected, actual_schema.fields) + return ValidationResult("x", not issues, issues, actual_schema) + + def normalize_type(self, provider_type): + return provider_type.lower() + + def run_quality_checks(self, resource_spec, rules): + self.quality_calls.append(("checks", rules)) + return self.issues + + def run_quality_results(self, resource_spec, rules): + self.quality_calls.append(("results", rules)) + return None if self.results == "unset" else self.results + + +def live(*cols): + return ResourceSchema( + ResourceType.TABLE, "c.s.t", [FieldSchema(n, t, "NULLABLE") for n, t in cols], row_count=1 + ) + + +def make_validator(provider, tmp_path): + v = cv.ContractValidator( + tmp_path / "contract.fluid.yaml", + provider_name="databricks", + use_cache=False, + track_history=False, + ) + v.report = cv.ValidationReport( + contract_path="x", + contract_id="id", + contract_version="1", + validation_time=datetime.now(), + duration=0.0, + ) + v.validation_provider = provider + return v + + +def expose(rules): + return { + "id": "e1", + "type": "table", + "binding": { + "platform": "databricks", + "location": {"database": "c", "schema": "s", "table": "t"}, + }, + "schema": CONTRACT_COLUMNS, + "contract": {"dq": {"rules": rules}}, + } + + +SCHEMA_RULE = {"id": "conforms", "type": "schema", "selector": "*", "severity": "error"} + + +def quality_issues(v): + return [i for i in v.report.issues if i.category == "quality"] + + +def check(v, name): + (c,) = [c for c in v.report.checks if c.name == name] + return c + + +# ------------------------------------------------------------- provider wiring + + +def detect(tmp_path, monkeypatch, available): + v = cv.ContractValidator(tmp_path / "c.yaml", use_cache=False, track_history=False) + v.contract = {"exposes": [{"binding": {"platform": "databricks"}}]} + v.report = cv.ValidationReport("x", "id", "1", datetime.now(), 0.0) + monkeypatch.setattr(cv, "DATABRICKS_VALIDATION_AVAILABLE", available) + v._detect_and_validate_provider() + return v + + +def test_databricks_provider_is_built_when_sdk_available(tmp_path, monkeypatch): + from fluid_build.providers.databricks_validation import DatabricksValidationProvider + + v = detect(tmp_path, monkeypatch, True) + assert isinstance(v.validation_provider, DatabricksValidationProvider) + assert [i for i in v.report.issues if i.category == "binding"] == [] + + +def test_databricks_without_sdk_is_binding_error_naming_sdk(tmp_path, monkeypatch): + v = detect(tmp_path, monkeypatch, False) + assert v.validation_provider is None + (issue,) = v.report.issues + assert issue.severity == "error" and issue.category == "binding" + assert "databricks-sdk" in issue.message + + +# --------------------------------------------------- schema rules are not sent + + +def test_schema_rules_are_not_sent_to_the_provider(tmp_path): + other = {"id": "c1", "type": "completeness", "selector": "a"} + p = FakeProvider(live=live(("a", "string"), ("b", "int")), results=[]) + v = make_validator(p, tmp_path) + v._validate_single_expose(expose([SCHEMA_RULE, other]), 0) + assert p.quality_calls and all( + [r["id"] for r in rules] == ["c1"] for _, rules in p.quality_calls + ) + + +def test_only_schema_rules_means_no_provider_quality_call(tmp_path): + p = FakeProvider(live=live(("a", "string"), ("b", "int"))) + v = make_validator(p, tmp_path) + v._validate_single_expose(expose([SCHEMA_RULE]), 0) + assert p.quality_calls == [] + + +def test_schema_rule_passes_when_columns_match(tmp_path): + v = make_validator(FakeProvider(live=live(("a", "string"), ("b", "int"))), tmp_path) + v._validate_single_expose(expose([SCHEMA_RULE]), 0) + c = check(v, "e1.conforms") + assert c.passed and c.rule_type == "schema" and c.rule_id == "conforms" and c.expose_id == "e1" + assert c.actual == "all contract columns match" and quality_issues(v) == [] + + +def test_schema_rule_ignores_extra_live_column(tmp_path): + v = make_validator( + FakeProvider(live=live(("a", "string"), ("b", "int"), ("extra", "string"))), tmp_path + ) + v._validate_single_expose(expose([SCHEMA_RULE]), 0) + assert check(v, "e1.conforms").passed + + +def test_schema_rule_fails_on_missing_column_at_rule_severity(tmp_path): + rule = {**SCHEMA_RULE, "severity": "warning"} + v = make_validator(FakeProvider(live=live(("a", "string"))), tmp_path) + v._validate_single_expose(expose([rule]), 0) + c = check(v, "e1.conforms") + assert not c.passed and c.severity == "warning" + assert c.actual == "1 column(s) missing, 0 type change(s)" + (issue,) = quality_issues(v) + assert issue.severity == "warning" and issue.path.endswith("dq.rules.conforms") + + +def test_schema_rule_fails_on_type_change(tmp_path): + v = make_validator(FakeProvider(live=live(("a", "string"), ("b", "bigint"))), tmp_path) + v._validate_single_expose(expose([SCHEMA_RULE]), 0) + c = check(v, "e1.conforms") + assert not c.passed and c.actual == "0 column(s) missing, 1 type change(s)" + assert quality_issues(v)[0].severity == "error" + + +def test_schema_rule_fails_when_table_missing(tmp_path): + v = make_validator(FakeProvider(live=None), tmp_path) + v._validate_single_expose(expose([SCHEMA_RULE]), 0) + c = check(v, "e1.conforms") + assert not c.passed and c.actual == "table does not exist" + + +def test_schema_rule_not_executed_when_comparison_did_not_run(tmp_path): + p = FakeProvider(schema_error=RuntimeError("no network")) + v = make_validator(p, tmp_path) + v._validate_single_expose(expose([SCHEMA_RULE]), 0) + c = check(v, "e1.conforms") + assert not c.passed and c.actual == "not executed" + + +def test_schema_rule_not_decided_without_check_data(tmp_path): + v = make_validator(FakeProvider(live=live(("a", "string"), ("b", "int"))), tmp_path) + v.check_data = False + v._validate_single_expose(expose([SCHEMA_RULE]), 0) + assert v.report.checks == [] + + +# ------------------------------------------------- outcomes for passing rules + + +def test_provider_results_list_gives_passing_outcomes_expected_and_actual(tmp_path): + results = [ + QualityCheckResult("c1", "completeness", "a", True, "error", "ok", "1.0", "1.0000"), + QualityCheckResult("u1", "uniqueness", "a", False, "warning", "dupes", "1.0", "0.5000"), + ] + rules = [ + {"id": "c1", "type": "completeness", "selector": "a"}, + {"id": "u1", "type": "uniqueness", "selector": "a", "severity": "warning"}, + ] + p = FakeProvider(live=live(("a", "string"), ("b", "int")), results=results) + v = make_validator(p, tmp_path) + v._validate_single_expose(expose(rules), 0) + ok = check(v, "e1.c1") + assert ok.passed and (ok.expected, ok.actual) == ("1.0", "1.0000") + assert (ok.rule_id, ok.rule_type, ok.expose_id) == ("c1", "completeness", "e1") + bad = check(v, "e1.u1") + assert not bad.passed and bad.actual == "0.5000" and bad.severity == "warning" + assert [i.path for i in quality_issues(v)] == ["exposes[0].contract.dq.rules.u1"] + + +def test_results_none_falls_back_to_run_quality_checks_and_still_tags_outcomes(tmp_path): + rules = [{"id": "c1", "type": "completeness", "selector": "a"}] + p = FakeProvider(live=live(("a", "string"), ("b", "int")), results="unset") + v = make_validator(p, tmp_path) + v._validate_single_expose(expose(rules), 0) + assert [k for k, _ in p.quality_calls] == ["results", "checks"] + c = check(v, "e1.c1") + assert c.passed and (c.rule_id, c.rule_type, c.expose_id) == ("c1", "completeness", "e1") + + +def test_provider_exception_becomes_issue_not_crash(tmp_path): + class Boom(FakeProvider): + def run_quality_results(self, resource_spec, rules): + raise RuntimeError("warehouse down") + + v = make_validator(Boom(live=live(("a", "string"), ("b", "int"))), tmp_path) + v._validate_single_expose(expose([{"id": "c1", "type": "completeness", "selector": "a"}]), 0) + assert any("warehouse down" in i.message for i in v.report.issues) + + +@pytest.mark.parametrize("value", ["Schema", " schema "]) +def test_rule_kind_normalises_case_and_space(value): + assert cv._rule_kind({"type": value}) == "schema" + assert cv._rule_kind("nope") == "" and cv._rule_kind({}) == "" diff --git a/tests/providers/test_databricks_statements.py b/tests/providers/test_databricks_statements.py new file mode 100644 index 00000000..d8649dc8 --- /dev/null +++ b/tests/providers/test_databricks_statements.py @@ -0,0 +1,182 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``providers/databricks/statements.py``: rows from a SQL warehouse over +``databricks_enforce.SqlRunner`` (request shape with parameters, polling, +timeout/cancel, failure scrubbing, type conversion, chunk paging), against a fake +``api_client.do``.""" + +from __future__ import annotations + +import pytest + +from fluid_build.iac.providers.databricks_enforce import StatementFailed +from fluid_build.providers.databricks.statements import ( + STATEMENTS_PATH, + StatementError, + StatementRunner, + param, + warehouse_id_from_env, +) + +FAKE_TOKEN = "fake-test-token-0000" # pragma: allowlist secret + + +class FakeApi: + def __init__(self, responses): + self.responses = list(responses) + self.calls = [] + + def do(self, method, path, query=None, body=None, **kw): + self.calls.append((method, path, body)) + item = self.responses.pop(0) + if isinstance(item, Exception): + raise item + return item + + +class FakeClient: + def __init__(self, responses): + self.api_client = FakeApi(responses) + + +def ok(rows=None, columns=(), **result_extra): + return { + "statement_id": "sid-1", + "status": {"state": "SUCCEEDED"}, + "manifest": {"schema": {"columns": [{"name": n, "type_name": t} for n, t in columns]}}, + "result": {"chunk_index": 0, "data_array": rows or [], **result_extra}, + } + + +def running(state="RUNNING"): + return {"statement_id": "sid-1", "status": {"state": state}} + + +def runner(responses, **kw): + client = FakeClient(responses) + kw.setdefault("sleep", lambda s: None) + return StatementRunner(client, "wh-1", **kw), client.api_client + + +def test_statement_error_is_the_enforce_runner_failure(): + assert StatementError is StatementFailed + + +def test_post_body_without_parameters(): + r, api = runner([ok([["1"]], [("one", "INT")])]) + assert r.run("SELECT 1") == [(1,)] + method, path, body = api.calls[0] + assert (method, path.rstrip("/")) == ("POST", STATEMENTS_PATH) + assert body == { + "warehouse_id": "wh-1", + "statement": "SELECT 1", + "wait_timeout": "30s", + "on_wait_timeout": "CONTINUE", + } + + +def test_parameters_are_bound_not_spliced(): + r, api = runner([ok()]) + r.run("SELECT :v", [param("v", "x'; DROP TABLE t; --")]) + body = api.calls[0][2] + assert body["statement"] == "SELECT :v" + assert body["parameters"] == [{"name": "v", "type": "STRING", "value": "x'; DROP TABLE t; --"}] + + +def test_live_response_shape_is_converted_by_type_name(): + # The shape a Free Edition warehouse returned for + # SELECT 1.5 AS d, CAST(2 AS BIGINT) AS l, 'x' AS s, true AS b + live = ok( + [["1.5", "2", "x", "true"]], + [("d", "DECIMAL"), ("l", "LONG"), ("s", "STRING"), ("b", "BOOLEAN")], + row_count=1, + row_offset=0, + ) + r, _ = runner([live]) + assert r.run("SELECT ...") == [(1.5, 2, "x", True)] + + +def test_null_stays_none_and_false_boolean_and_odd_value_kept(): + r, _ = runner([ok([[None, "false", "n/a"]], [("a", "LONG"), ("b", "BOOLEAN"), ("c", "INT")])]) + assert r.run("q") == [(None, False, "n/a")] + + +def test_pending_then_running_then_succeeded_polls_get(): + r, api = runner([running("PENDING"), running(), ok([["7"]], [("n", "LONG")])]) + assert r.run("q") == [(7,)] + assert [c[0] for c in api.calls] == ["POST", "GET", "GET"] + assert api.calls[1][1] == f"{STATEMENTS_PATH}/sid-1" + + +def test_timeout_cancels_then_raises(): + pending = [running("PENDING")] * 50 + r, api = runner(pending + [{}], timeout_seconds=0) + with pytest.raises(StatementError, match="did not finish"): + r.run("q") + assert any(c[0] == "POST" and c[1].endswith("/sid-1/cancel") for c in api.calls) + + +def test_failed_state_raises_with_scrubbed_api_message(monkeypatch): + monkeypatch.setenv("DATABRICKS_TOKEN", FAKE_TOKEN) + failed = { + "statement_id": "sid-1", + "status": { + "state": "FAILED", + "error": {"message": f"[TABLE_OR_VIEW_NOT_FOUND] t ({FAKE_TOKEN})"}, + }, + } + r, _ = runner([failed]) + with pytest.raises(StatementError) as exc: + r.run("q") + assert "TABLE_OR_VIEW_NOT_FOUND" in str(exc.value) + assert FAKE_TOKEN not in str(exc.value) + + +def test_transport_exception_is_wrapped_and_scrubbed(monkeypatch): + monkeypatch.setenv("DATABRICKS_TOKEN", FAKE_TOKEN) + r, _ = runner([RuntimeError(f"connection reset, token {FAKE_TOKEN}")]) + with pytest.raises(StatementError) as exc: + r.run("q") + assert "RuntimeError" in str(exc.value) + assert FAKE_TOKEN not in str(exc.value) + + +def test_canceled_state_raises(): + r, _ = runner([{"statement_id": "sid-1", "status": {"state": "CANCELED"}}]) + with pytest.raises(StatementError, match="CANCELED"): + r.run("q") + + +def test_next_chunk_index_fetches_further_chunks(): + first = ok([["1"]], [("n", "LONG")], next_chunk_index=1) + second = {"chunk_index": 1, "data_array": [["2"], ["3"]]} + r, api = runner([first, second]) + assert r.run("q") == [(1,), (2,), (3,)] + assert api.calls[1] == ("GET", f"{STATEMENTS_PATH}/sid-1/result/chunks/1", None) + + +def test_chunk_fetch_failure_is_a_statement_error(monkeypatch): + monkeypatch.setenv("DATABRICKS_TOKEN", FAKE_TOKEN) + first = ok([["1"]], [("n", "LONG")], next_chunk_index=1) + r, _ = runner([first, RuntimeError(f"chunk gone {FAKE_TOKEN}")]) + with pytest.raises(StatementError) as exc: + r.run("q") + assert FAKE_TOKEN not in str(exc.value) + + +def test_param_omits_value_for_none_and_takes_type(): + assert param("n", None) == {"name": "n", "type": "STRING"} + assert param("b", True, "BOOLEAN") == {"name": "b", "type": "BOOLEAN", "value": "True"} + + +def test_warehouse_id_from_env_unset_or_blank_names_the_variable(): + for env in ({}, {"DATABRICKS_WAREHOUSE_ID": " "}): + with pytest.raises(StatementError, match="DATABRICKS_WAREHOUSE_ID"): + warehouse_id_from_env(env) + + +def test_warehouse_id_from_env_trims_and_reads_os_environ(monkeypatch): + assert warehouse_id_from_env({"DATABRICKS_WAREHOUSE_ID": " wh "}) == "wh" + monkeypatch.setenv("DATABRICKS_WAREHOUSE_ID", "from-env") + assert warehouse_id_from_env() == "from-env" diff --git a/tests/providers/test_databricks_validation.py b/tests/providers/test_databricks_validation.py new file mode 100644 index 00000000..fb4bd8c0 --- /dev/null +++ b/tests/providers/test_databricks_validation.py @@ -0,0 +1,429 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""DatabricksValidationProvider against a fake workspace client (no SDK, no network).""" + +from __future__ import annotations + +import pytest + +from fluid_build.providers.databricks.statements import StatementError +from fluid_build.providers.databricks_validation import ( + DatabricksValidationProvider, + sql_table_ref, + table_name, +) +from fluid_build.providers.validation_provider import FieldSchema, ResourceSchema, ResourceType + + +class NotFound(Exception): + status_code = 404 + + +def table_json(name="smart_meters", columns=None): + cols = columns or [ + ("site_id", "string", "STRING", False), + ("qty", "int", "INT", True), + ("price", "decimal(12,3)", "DECIMAL", True), + ] + return { + "name": name, + "table_type": "MANAGED", + "columns": [ + {"name": n, "type_text": tt, "type_name": tn, "nullable": nb} for n, tt, tn, nb in cols + ], + } + + +class FakeApi: + """Routes api_client.do by method+path; records every call.""" + + def __init__(self): + self.calls = [] + self.tables = {} # full name -> json or exception + self.warehouses = {"w": {"id": "w"}} + self.statements = [] # (predicate, rows) + self.sql = [] + + def do(self, method, path, query=None, body=None, **kw): + self.calls.append((method, path, body)) + if method == "GET" and path.startswith("/api/2.0/sql/warehouses/"): + item = self.warehouses.get(path.rsplit("/", 1)[1], NotFound("no wh")) + elif method == "GET" and path.startswith("/api/2.1/unity-catalog/tables/"): + from urllib.parse import unquote + + item = self.tables.get(unquote(path.rsplit("/", 1)[1]), NotFound("no table")) + elif method == "POST" and path.rstrip("/") == "/api/2.0/sql/statements": + self.sql.append(body) + item = self._answer(body) + else: + raise AssertionError(f"unexpected call {method} {path}") + if isinstance(item, Exception): + raise item + return item + + def _answer(self, body): + text = body["statement"] + for pred, rows, cols in self.statements: + if pred in text: + return { + "statement_id": "s", + "status": {"state": "SUCCEEDED"}, + "manifest": { + "schema": {"columns": [{"name": c, "type_name": t} for c, t in cols]} + }, + "result": {"data_array": rows}, + } + return { + "statement_id": "s", + "status": {"state": "SUCCEEDED"}, + "manifest": {"schema": {"columns": [{"name": "n", "type_name": "LONG"}]}}, + "result": {"data_array": [["0"]]}, + } + + def stmt_calls(self): + return [c for c in self.calls if c[0] == "POST"] + + +class FakeClient: + def __init__(self, api): + self.api_client = api + + +@pytest.fixture +def api(): + return FakeApi() + + +@pytest.fixture +def provider(api): + return DatabricksValidationProvider( + {"client_factory": lambda: FakeClient(api), "warehouse_id": "w"} + ) + + +EXPOSE = { + "id": "meters", + "binding": { + "platform": "databricks", + "location": {"database": "main", "schema": "energy", "table": "smart_meters"}, + }, + "contract": { + "schema": [ + {"name": "site_id", "type": "VARCHAR", "required": True}, + {"name": "qty", "type": "INTEGER"}, + {"name": "price", "type": "DECIMAL(12,3)"}, + ] + }, +} +FULL = "main.energy.smart_meters" + + +# ---------------------------------------------------------------- names + + +def test_table_name_validates_parts(): + assert table_name(EXPOSE) == ("main", "energy", "smart_meters") + assert table_name({"binding": {}}) is None + bad = {"binding": {"location": {"database": "m", "schema": "s", "table": "bad-name"}}} + with pytest.raises(ValueError): + table_name(bad) + + +def test_table_name_rejects_backtick(): + bad = {"binding": {"location": {"database": "m", "schema": "s", "table": "t`x"}}} + with pytest.raises(ValueError): + table_name(bad) + + +def test_sql_table_ref_backticks_each_part(): + assert sql_table_ref(("a", "b", "c")) == "`a`.`b`.`c`" + + +# ----------------------------------------------------------- connection + + +def test_validate_connection_gets_warehouse(provider, api): + assert provider.validate_connection() is True + assert api.calls == [("GET", "/api/2.0/sql/warehouses/w", None)] + + +def test_validate_connection_404_names_warehouse(api): + p = DatabricksValidationProvider( + {"client_factory": lambda: FakeClient(api), "warehouse_id": "gone"} + ) + with pytest.raises(StatementError, match="gone"): + p.validate_connection() + + +def test_validate_connection_other_error_is_wrapped(api): + api.warehouses["w"] = RuntimeError("boom") + p = DatabricksValidationProvider( + {"client_factory": lambda: FakeClient(api), "warehouse_id": "w"} + ) + with pytest.raises(StatementError, match="RuntimeError: boom"): + p.validate_connection() + + +def test_validate_connection_needs_warehouse_env(api, monkeypatch): + monkeypatch.delenv("DATABRICKS_WAREHOUSE_ID", raising=False) + p = DatabricksValidationProvider({"client_factory": lambda: FakeClient(api)}) + with pytest.raises(StatementError, match="DATABRICKS_WAREHOUSE_ID"): + p.validate_connection() + + +# ---------------------------------------------------------------- schema + + +def test_get_resource_schema_missing_table_is_none(provider): + assert provider.get_resource_schema(EXPOSE) is None + + +def test_get_resource_schema_maps_columns_and_counts_rows(provider, api): + api.tables[FULL] = table_json() + api.statements.append(("COUNT(*)", [["42"]], [("c", "LONG")])) + schema = provider.get_resource_schema(EXPOSE) + assert [(f.name, f.type, f.mode) for f in schema.fields] == [ + ("site_id", "string", "REQUIRED"), + ("qty", "int", "NULLABLE"), + ("price", "decimal(12,3)", "NULLABLE"), + ] + assert schema.row_count == 42 and schema.fully_qualified_name == FULL + assert schema.metadata["table_type"] == "MANAGED" + assert api.sql[0]["statement"] == "SELECT COUNT(*) FROM `main`.`energy`.`smart_meters`" + + +def test_get_resource_schema_requires_location(provider): + with pytest.raises(ValueError, match="binding.location"): + provider.get_resource_schema({"binding": {}}) + + +def test_get_resource_schema_other_error_wrapped(provider, api): + api.tables[FULL] = RuntimeError("kaboom") + with pytest.raises(StatementError, match="kaboom"): + provider.get_resource_schema(EXPOSE) + + +def live_schema(fields, rows=5): + return ResourceSchema(ResourceType.TABLE, FULL, fields, row_count=rows) + + +def live_fields(): + return [ + FieldSchema("site_id", "string", "REQUIRED"), + FieldSchema("qty", "int", "NULLABLE"), + FieldSchema("price", "decimal(12,3)", "NULLABLE"), + ] + + +def test_validate_resource_contract_types_match_live_types(provider): + res = provider.validate_resource(EXPOSE, live_schema(live_fields())) + assert res.success and res.issues == [] + + +def test_validate_resource_missing_table(provider): + res = provider.validate_resource(EXPOSE, None) + assert not res.success + (issue,) = res.issues + assert issue.category == "missing_resource" and FULL in issue.message + + +def test_validate_resource_real_type_change_is_type_mismatch(provider): + fields = live_fields() + fields[1] = FieldSchema("qty", "bigint", "NULLABLE") + res = provider.validate_resource(EXPOSE, live_schema(fields)) + assert not res.success + assert [(i.category, i.severity) for i in res.issues] == [("type_mismatch", "error")] + + +def test_validate_resource_missing_column(provider): + res = provider.validate_resource(EXPOSE, live_schema(live_fields()[:2])) + assert [i.category for i in res.issues] == ["missing_field"] + + +def test_validate_resource_empty_table_is_warning_only(provider): + res = provider.validate_resource(EXPOSE, live_schema(live_fields(), rows=0)) + assert res.success + assert [(i.category, i.severity) for i in res.issues] == [("empty_table", "warning")] + + +def test_validate_resource_sla_row_count_min(provider): + spec = {**EXPOSE, "sla": {"row_count_min": 10}} + res = provider.validate_resource(spec, live_schema(live_fields(), rows=3)) + assert not res.success + assert [i.category for i in res.issues] == ["row_count_below_threshold"] + + +# ----------------------------------------------------------- quality rules + + +@pytest.fixture +def live(api): + """The tested table exists (the rule tests read data from it).""" + api.tables[FULL] = table_json() + return api + + +def test_rules_on_a_missing_table_run_no_statement(provider, api): + rules = [ + {"id": "c1", "type": "completeness", "selector": "site_id", "severity": "error"}, + {"id": "u1", "type": "uniqueness", "selector": "site_id"}, + ] + results = provider.run_quality_results(EXPOSE, rules) + assert [r.rule_id for r in results] == ["c1", "u1"] + assert all(not r.passed and r.actual == "not executed" for r in results) + assert results[0].severity == "error" + assert FULL in results[0].message and "does not exist" in results[0].message + assert api.sql == [] + + +def test_table_metadata_is_read_once_for_schema_and_rules(provider, live): + live.statements.append(("COUNT(*)", [["4"]], [("n", "LONG")])) + provider.get_resource_schema(EXPOSE) + live.statements.insert(0, ("NULLIF(COUNT(*), 0)", [["1.0"]], [("r", "DOUBLE")])) + provider.run_quality_results( + EXPOSE, [{"id": "c1", "type": "completeness", "selector": "site_id"}] + ) + gets = [ + c for c in live.calls if c[0] == "GET" and FULL.replace(".", "") in c[1].replace(".", "") + ] + assert len(gets) == 1 + + +def test_run_quality_results_uses_databricks_dialect_and_backticks(provider, live): + rules = [{"id": "c1", "type": "completeness", "selector": "site_id"}] + live.statements.insert(0, ("NULLIF(COUNT(*), 0)", [["1.0"]], [("r", "DOUBLE")])) + (res,) = provider.run_quality_results(EXPOSE, rules) + stmt = live.sql[0]["statement"] + assert res.passed and res.rule_id == "c1" + assert "FROM `main`.`energy`.`smart_meters`" in stmt + assert "`site_id`" in stmt and "AS DOUBLE" in stmt + + +def test_run_quality_results_passes_columns_for_enum(provider, live): + spec = { + **EXPOSE, + "contract": { + "schema": [ + { + "name": "status", + "validationRules": [{"type": "enum", "constraint": "A|B"}], + } + ] + }, + } + (res,) = provider.run_quality_results( + spec, [{"id": "v", "type": "valid_values", "selector": "status"}] + ) + assert res.passed and "NOT IN ('A', 'B')" in live.sql[0]["statement"] + + +def test_run_quality_results_requires_location(provider): + with pytest.raises(ValueError): + provider.run_quality_results({"binding": {}}, []) + + +# ---------------------------------------------------------- references + +REF_COLUMNS = [{"name": "site_id", "labels": {"references": "acme.sites.site_id"}}] +REF_EXPOSE = {**EXPOSE, "contract": {"schema": REF_COLUMNS}} +REF_RULE = {"id": "fk", "type": "valid_values", "selector": "site_id"} + + +def tag_rows(live, rows): + live.statements.insert( + 0, + ( + "table_tags", + rows, + [("catalog_name", "STRING"), ("schema_name", "STRING"), ("table_name", "STRING")], + ), + ) + + +def test_reference_lookup_uses_tag_query_with_params_then_anti_join(provider, live): + tag_rows(live, [["main", "ref", "sites"], ["main", "ref", "other"]]) + live.tables["main.ref.sites"] = table_json("sites", [("site_id", "string", "STRING", True)]) + live.tables["main.ref.other"] = table_json("other", [("x", "string", "STRING", True)]) + live.statements.insert(1, ("NOT EXISTS", [["0"]], [("m", "LONG")])) + (res,) = provider.run_quality_results(REF_EXPOSE, [REF_RULE]) + tag_call = live.sql[0] + assert "system.information_schema.table_tags" in tag_call["statement"] + assert tag_call["parameters"] == [ + {"name": "tag", "type": "STRING", "value": "fluid_product_id"}, + {"name": "product", "type": "STRING", "value": "acme.sites"}, + ] + assert "FROM `main`.`ref`.`sites` AS p" in live.sql[1]["statement"] + assert res.passed + + +def test_reference_no_candidate_is_not_executed_with_lookup_text(provider, live): + tag_rows(live, []) + (res,) = provider.run_quality_results(REF_EXPOSE, [REF_RULE]) + assert not res.passed and res.actual == "not executed" + assert "fluid_product_id=acme.sites" in res.message + + +def test_reference_ambiguous_is_not_executed(provider, live): + tag_rows(live, [["main", "a", "sites"], ["main", "b", "sites"]]) + for s in ("a", "b"): + live.tables[f"main.{s}.sites"] = table_json( + "sites", [("site_id", "string", "STRING", True)] + ) + (res,) = provider.run_quality_results(REF_EXPOSE, [REF_RULE]) + assert res.actual == "not executed" and "ambiguous" in res.message + + +def test_reference_lookup_cached_per_product_and_column(provider, live): + tag_rows(live, [["main", "ref", "sites"]]) + live.tables["main.ref.sites"] = table_json("sites", [("site_id", "string", "STRING", True)]) + rules = [REF_RULE, {**REF_RULE, "id": "fk2"}] + provider.run_quality_results(REF_EXPOSE, rules) + tag_queries = [b for b in live.sql if "table_tags" in b["statement"]] + assert len(tag_queries) == 1 + anti_joins = [b for b in live.sql if "NOT EXISTS" in b["statement"]] + assert len(anti_joins) == 2 + + +def test_reference_unreadable_candidate_is_skipped(provider, live): + tag_rows(live, [["main", "ref", "sites"]]) # table GET raises NotFound + (res,) = provider.run_quality_results(REF_EXPOSE, [REF_RULE]) + assert res.actual == "not executed" + + +def test_run_quality_checks_returns_issues_for_failures(provider, live): + live.statements.insert(0, ("NULLIF(COUNT(*), 0)", [["0.5"]], [("r", "DOUBLE")])) + issues = provider.run_quality_checks( + EXPOSE, [{"id": "c1", "type": "completeness", "selector": "site_id", "severity": "error"}] + ) + assert [(i.category, i.severity) for i in issues] == [("quality", "error")] + + +def test_normalize_type_and_provider_name(provider): + assert provider.provider_name == "databricks" + assert provider.normalize_type("STRING") == "string" + + +def test_reference_prefers_the_tested_tables_catalog(provider, live): + # A look-alike table tagged with the same product id in another catalog does + # not make the lookup ambiguous, and is not the one read. + tag_rows(live, [["main", "ref", "sites"], ["shadow", "ref", "sites"]]) + for catalog in ("main", "shadow"): + live.tables[f"{catalog}.ref.sites"] = table_json( + "sites", [("site_id", "string", "STRING", True)] + ) + live.statements.insert(0, ("NOT EXISTS", [["0"]], [("n", "LONG")])) + (res,) = provider.run_quality_results(REF_EXPOSE, [REF_RULE]) + assert res.passed, res.message + anti_join = next(b for b in live.sql if "NOT EXISTS" in b["statement"])["statement"] + assert "`main`.`ref`.`sites`" in anti_join and "shadow" not in anti_join + + +def test_reference_two_candidates_elsewhere_stay_ambiguous(provider, live): + tag_rows(live, [["other1", "ref", "sites"], ["other2", "ref", "sites"]]) + for catalog in ("other1", "other2"): + live.tables[f"{catalog}.ref.sites"] = table_json( + "sites", [("site_id", "string", "STRING", True)] + ) + (res,) = provider.run_quality_results(REF_EXPOSE, [REF_RULE]) + assert res.actual == "not executed" and "ambiguous" in res.message diff --git a/tests/providers/test_quality_engine_column_rules.py b/tests/providers/test_quality_engine_column_rules.py new file mode 100644 index 00000000..b713b4d2 --- /dev/null +++ b/tests/providers/test_quality_engine_column_rules.py @@ -0,0 +1,467 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""Column-rule behaviour of the native quality engine: dialect quoting, multi-column +rules, value/pattern/reference sources and the backslash-safe string literal.""" + +from __future__ import annotations + +import pytest + +from fluid_build.providers import quality_engine as qe +from fluid_build.providers._sql_safety import quote_string_literal_for + +T = "`c`.`s`.`t`" + + +class Recorder: + """execute_fn double: records SQL, answers from a queue (or a fixed row).""" + + def __init__(self, *answers): + self.sql = [] + self._answers = list(answers) + + def __call__(self, sql): + self.sql.append(sql) + if not self._answers: + return [(0,)] + return self._answers.pop(0) + + +def run(rule, execute_fn, dialect="databricks", **kw): + (res,) = qe.execute_quality_checks([rule], T, execute_fn, dialect, **kw) + return res + + +# ---------------------------------------------------------------- helpers + + +@pytest.mark.parametrize( + "dialect,expected", + [ + ("databricks", "`x`"), + ("bigquery", "`x`"), + ("BigQuery", "`x`"), + ("snowflake", '"x"'), + ("ansi", '"x"'), + (None, '"x"'), + ], +) +def test_qi_quotes_per_dialect(dialect, expected): + assert qe._qi("x", dialect) == expected + + +def test_qi_rejects_unsafe_identifier(): + with pytest.raises(ValueError): + qe._qi("a`b", "databricks") + + +@pytest.mark.parametrize( + "dialect,expected", + [ + ("databricks", "DOUBLE"), + ("bigquery", "FLOAT64"), + ("snowflake", "DOUBLE PRECISION"), + (None, "DOUBLE PRECISION"), + ], +) +def test_double_type_per_dialect(dialect, expected): + assert qe._double(dialect) == expected + + +def test_rule_columns_splits_and_strips(): + assert qe.rule_columns("a, b ,,c") == ["a", "b", "c"] + assert qe.rule_columns("*") == ["*"] + assert qe.rule_columns(None) == [] + + +def test_expose_columns_prefers_contract_schema(): + expose = {"contract": {"schema": [{"name": "a"}, "junk"]}, "schema": [{"name": "z"}]} + assert qe.expose_columns(expose) == [{"name": "a"}] + assert qe.expose_columns({"schema": [{"name": "z"}]}) == [{"name": "z"}] + assert qe.expose_columns("nope") == [] + + +# ----------------------------------------------------- valid value sources + +COLS = [ + { + "name": "status", + "validationRules": [{"type": "enum", "constraint": "GOOD | ESTIMATED, SUSPECT"}], + }, + {"name": "cc", "validationRules": [{"type": "regex", "constraint": "^[A-Z]{2}$"}]}, + {"name": "site_id", "labels": {"references": "acme.sites.site_id"}}, +] + + +def test_valid_values_explicit_wins(): + rule = {"selector": "status", "validValues": ["X"], "description": "status valid values: A."} + assert qe.extract_valid_values(rule, COLS) == ["X"] + + +def test_valid_values_from_description_beats_enum(): + rule = {"selector": "status", "description": "status valid values: A, B."} + assert qe.extract_valid_values(rule, COLS) == ["A", "B"] + + +def test_valid_values_from_enum_splits_on_pipe_and_comma(): + assert qe.extract_valid_values({"selector": "status"}, COLS) == [ + "GOOD", + "ESTIMATED", + "SUSPECT", + ] + + +def test_valid_values_empty_without_any_source(): + assert qe.extract_valid_values({"selector": "cc"}, COLS) == [] + + +def test_value_pattern_reads_regex_rule(): + assert qe.extract_value_pattern({"selector": "cc"}, COLS) == "^[A-Z]{2}$" + assert qe.extract_value_pattern({"selector": "status"}, COLS) is None + + +def test_reference_rpartitions_on_last_dot(): + assert qe.extract_reference({"selector": "site_id"}, COLS) == ("acme.sites", "site_id") + assert qe.extract_reference({"selector": "cc"}, COLS) is None + + +# ------------------------------------------------ multi-column completeness + + +def test_completeness_multi_column_is_one_statement_with_lowest_ratio(): + rec = Recorder([(1.0, 0.5, 0.9)]) + res = run( + { + "id": "r", + "type": "completeness", + "selector": "a,b,c", + "threshold": 0.95, + "operator": ">=", + }, + rec, + ) + assert len(rec.sql) == 1 + assert rec.sql[0].count("CAST(") == 3 and "AS DOUBLE" in rec.sql[0] + assert res.passed is False + assert res.actual == "0.5000" + assert "'b'" in res.message and "'c'" in res.message and "'a'" not in res.message + + +def test_completeness_multi_column_passes_when_all_meet_bound(): + res = run({"id": "r", "type": "completeness", "selector": "a,b"}, Recorder([(1.0, 1.0)])) + assert res.passed and res.expected == "1.0" + + +def test_completeness_null_ratio_counts_as_zero(): + res = run({"id": "r", "type": "completeness", "selector": "a,b"}, Recorder([(1.0, None)])) + assert not res.passed and res.actual == "0.0000" + + +# ------------------------------------------------ multi-column accuracy + + +@pytest.mark.parametrize( + "operator,func", [(">=", "MIN"), (">", "MIN"), ("<=", "MAX"), ("<", "MAX")] +) +def test_accuracy_multi_column_aggregate_follows_operator(operator, func): + rec = Recorder([(1, 2)]) + run( + {"id": "r", "type": "accuracy", "selector": "a,b", "threshold": 0, "operator": operator}, + rec, + ) + assert rec.sql[0].count(f"{func}(`") == 2 and len(rec.sql) == 1 + + +def test_accuracy_multi_column_lower_bound_fails_on_low_column(): + res = run( + {"id": "r", "type": "accuracy", "selector": "a,b", "threshold": 5, "operator": ">="}, + Recorder([(10, 1)]), + ) + assert not res.passed and "'b'" in res.message and res.actual == "1" + + +def test_accuracy_multi_column_equality_uses_violation_counts(): + rec = Recorder([(0, 3)]) + res = run( + {"id": "r", "type": "accuracy", "selector": "a,b", "threshold": 1, "operator": "=="}, rec + ) + assert "SUM(CASE WHEN" in rec.sql[0] and "<>" in rec.sql[0] + assert not res.passed and res.actual == "3 violating row(s)" + + +def test_accuracy_multi_column_no_data_fails(): + res = run( + {"id": "r", "type": "accuracy", "selector": "a,b", "threshold": 0, "operator": ">="}, + Recorder([(1, None)]), + ) + assert not res.passed and "no data in 'b'" in res.message + + +# ------------------------------------- single-column rules over >1 columns + + +@pytest.mark.parametrize("rtype", ["uniqueness", "valid_values", "freshness"]) +def test_single_column_rule_over_many_columns_not_executed(rtype): + rec = Recorder() + res = run({"id": "r", "type": rtype, "selector": "a,b"}, rec) + assert res.passed is False and res.actual == "not executed" and rec.sql == [] + + +# ------------------------------------------------------- pattern SQL + + +@pytest.mark.parametrize( + "dialect,fragment", + [ + ("databricks", "NOT (`cc` RLIKE '^[A-Z]{2}$')"), + ("bigquery", "NOT REGEXP_CONTAINS(`cc`, '^[A-Z]{2}$')"), + ("snowflake", "REGEXP_INSTR(\"cc\", '^[A-Z]{2}$') = 0"), + ("ansi", "NOT regexp_matches(\"cc\", '^[A-Z]{2}$')"), + ("duckdb", "NOT regexp_matches(\"cc\", '^[A-Z]{2}$')"), + ], +) +def test_validity_pattern_sql_per_dialect(dialect, fragment): + sql = qe._validity_pattern_sql(T, "cc", "^[A-Z]{2}$", dialect) + assert fragment in sql and "IS NOT NULL" in sql + + +def test_validity_pattern_unknown_dialect_is_error_result(): + res = run( + {"id": "r", "type": "valid_values", "selector": "cc"}, Recorder(), "oracle", columns=COLS + ) + assert not res.passed and res.message.startswith("Error executing rule 'r'") + assert "no regular-expression test" in res.message + + +def test_valid_values_via_enum_runs_not_in_and_keeps_type(): + rec = Recorder([(2,)]) + res = run({"id": "r", "type": "valid_values", "selector": "status"}, rec, columns=COLS) + assert "NOT IN ('GOOD', 'ESTIMATED', 'SUSPECT')" in rec.sql[0] + assert res.rule_type == "valid_values" and not res.passed + + +def test_legacy_validity_type_is_preserved(): + res = run( + {"id": "r", "type": "validity", "selector": "s", "validValues": ["A"]}, Recorder([(0,)]) + ) + assert res.rule_type == "validity" and res.passed + + +def test_pattern_rule_via_column_regex(): + rec = Recorder([(0,)]) + res = run({"id": "r", "type": "valid_values", "selector": "cc"}, rec, columns=COLS) + assert "RLIKE" in rec.sql[0] and res.passed and res.rule_type == "valid_values" + + +# ------------------------------------------------------- references + + +def test_reference_without_resolver_not_executed(): + res = run({"id": "r", "type": "valid_values", "selector": "site_id"}, Recorder(), columns=COLS) + assert not res.passed and res.actual == "not executed" + + +def test_reference_resolver_lookup_error_message_is_surfaced(): + def resolver(product, column): + raise LookupError("nothing tagged") + + res = run( + {"id": "r", "type": "valid_values", "selector": "site_id"}, + Recorder(), + columns=COLS, + resolve_reference=resolver, + ) + assert res.actual == "not executed" and "nothing tagged" in res.message + + +def test_reference_runs_anti_join(): + rec = Recorder([(3,)]) + seen = [] + + def resolver(product, column): + seen.append((product, column)) + return "`p`.`s`.`sites`" + + res = run( + {"id": "r", "type": "valid_values", "selector": "site_id"}, + rec, + columns=COLS, + resolve_reference=resolver, + ) + assert seen == [("acme.sites", "site_id")] + assert ( + "NOT EXISTS (SELECT 1 FROM `p`.`s`.`sites` AS p WHERE p.`site_id` = c.`site_id`)" + in rec.sql[0] + ) + assert not res.passed and res.actual == "3 value(s) missing" + + +# ------------------------------------------------------- freshness + + +def test_freshness_databricks_sql(): + sql = qe._freshness_sql(T, "ts", "databricks") + assert "timestampdiff(SECOND, MAX(`ts`), current_timestamp())" in sql + + +# ------------------------------------------- backslash-safe string literals + + +def scan_literal(text): + """Reference scanner for the backslash-escaping grammar: decode one literal and + return (value, index just past the closing quote).""" + assert text[0] == "'" + out, i = [], 1 + while i < len(text): + ch = text[i] + if ch == "\\": + out.append(text[i + 1]) + i += 2 + elif ch == "'": + return "".join(out), i + 1 + else: + out.append(ch) + i += 1 + raise AssertionError("unterminated literal") + + +@pytest.mark.parametrize("dialect", ["databricks", "bigquery", "spark", "Databricks"]) +@pytest.mark.parametrize( + "value", + ["plain", "trailing\\", "it's", "\\' OR 1=1 --", "a\\\\b", "'", "\\", "x'); DROP TABLE t; --"], +) +def test_backslash_dialect_literal_roundtrips_and_cannot_break_out(dialect, value): + literal = quote_string_literal_for(dialect, value) + decoded, end = scan_literal(literal + " OR 1=1") + assert decoded == value + assert literal[end - 1] == "'" and literal[:end] == literal + + +def test_backslash_dialect_escape_shape(): + assert quote_string_literal_for("databricks", "a'b") == "'a\\'b'" + assert quote_string_literal_for("databricks", "a\\") == "'a\\\\'" + + +def test_other_dialect_delegates_to_existing_quoting(): + assert quote_string_literal_for("duckdb", "it's") == "'it''s'" + + +def test_non_string_value_rejected(): + with pytest.raises(ValueError): + quote_string_literal_for("databricks", 5) # type: ignore[arg-type] + + +# ── range validationRules and valid_values over several columns ───────────── + +from fluid_build.providers.quality_engine import ( # noqa: E402 + ValueDomain, + execute_quality_checks, + parse_range, + value_domain, +) + + +@pytest.mark.parametrize( + "constraint, bounds", + [ + (">= 0", [(">=", 0.0)]), + (">= 0 and <= 100", [(">=", 0.0), ("<=", 100.0)]), + ("between -1.5 and 2", [(">=", -1.5), ("<=", 2.0)]), + ("> 0 AND < 1", [(">", 0.0), ("<", 1.0)]), + ("0..100", []), + (">= zero", []), + (None, []), + ], +) +def test_parse_range(constraint, bounds): + assert parse_range(constraint) == bounds + + +ENERGY = [ + {"name": "grid_kwh", "validationRules": [{"type": "range", "constraint": ">= 0"}]}, + {"name": "solar_kwh", "validationRules": [{"type": "range", "constraint": ">= 0"}]}, + {"name": "quality", "validationRules": [{"type": "enum", "constraint": "GOOD,BAD"}]}, + {"name": "free_text"}, +] + + +def _recording(answers): + seen = [] + + def execute(sql): + seen.append(sql) + return [(answers.pop(0),)] + + return seen, execute + + +def test_range_rule_counts_values_outside_the_bounds_on_databricks(): + seen, execute = _recording([0]) + rule = {"id": "r", "type": "valid_values", "selector": "grid_kwh", "severity": "error"} + (res,) = execute_quality_checks([rule], "`c`.`s`.`t`", execute, "databricks", columns=ENERGY) + assert res.passed and res.expected == "0 values outside >= 0" + assert "WHERE `grid_kwh` IS NOT NULL AND NOT (`grid_kwh` >= 0.0)" in seen[0] + + +def test_valid_values_over_columns_checks_each_against_its_own_domain(): + seen, execute = _recording([0, 3, 1]) + rule = { + "id": "nonneg", + "type": "valid_values", + "selector": "grid_kwh, solar_kwh, quality", + "severity": "error", + } + (res,) = execute_quality_checks([rule], "t", execute, "databricks", columns=ENERGY) + assert not res.passed and res.rule_type == "valid_values" + assert len(seen) == 3 + assert "NOT (`solar_kwh` >= 0.0)" in seen[1] and "NOT IN ('GOOD', 'BAD')" in seen[2] + assert "'solar_kwh' (3 invalid value(s))" in res.message + assert "'quality' (1 invalid value(s))" in res.message + assert res.actual == "4 invalid value(s)" + + +def test_valid_values_over_columns_all_clean_passes(): + _, execute = _recording([0, 0]) + rule = {"id": "nonneg", "type": "valid_values", "selector": "grid_kwh,solar_kwh"} + (res,) = execute_quality_checks([rule], "t", execute, "databricks", columns=ENERGY) + assert res.passed and res.message == "Validity OK for all 2 columns" + assert res.actual == "0 invalid value(s)" + + +def test_valid_values_over_columns_with_an_undeclared_column_is_not_executed(): + _, execute = _recording([0]) + rule = {"id": "v", "type": "valid_values", "selector": "grid_kwh,free_text"} + (res,) = execute_quality_checks([rule], "t", execute, "databricks", columns=ENERGY) + assert not res.passed and res.actual == "not executed" + assert "nothing checked for 'free_text'" in res.message + + +def test_value_domain_order_rule_list_then_enum_regex_range_reference(): + columns = [ + { + "name": "a", + "validationRules": [ + {"type": "range", "constraint": ">= 0"}, + {"type": "regex", "constraint": "^x$"}, + {"type": "enum", "constraint": "X | Y"}, + ], + "labels": {"references": "p.q.col"}, + }, + { + "name": "b", + "validationRules": [ + {"type": "range", "constraint": ">= 0"}, + {"type": "regex", "constraint": "^x$"}, + ], + }, + {"name": "c", "validationRules": [{"type": "range", "constraint": ">= 0"}]}, + {"name": "d", "labels": {"references": "p.q.col"}}, + ] + listed = {"id": "r", "selector": "a", "description": "a valid values: K, L."} + assert value_domain(listed, columns) == ValueDomain(values=("K", "L")) + assert value_domain({"selector": "a"}, columns) == ValueDomain(values=("X", "Y")) + assert value_domain({"selector": "b"}, columns) == ValueDomain(pattern="^x$") + assert value_domain({"selector": "c"}, columns) == ValueDomain(bounds=((">=", 0.0),)) + assert value_domain({"selector": "d"}, columns) == ValueDomain(reference=("p.q", "col")) + assert value_domain({"selector": "zz"}, columns) == ValueDomain() diff --git a/tests/test_contract_val_branches.py b/tests/test_contract_val_branches.py index 7def2b4d..f1453ba6 100644 --- a/tests/test_contract_val_branches.py +++ b/tests/test_contract_val_branches.py @@ -328,18 +328,20 @@ def test_detect_databricks_provider(self, mock_schema, mock_load): } mock_result = MagicMock(is_valid=True, errors=[], warnings=[]) mock_schema.return_value.validate_contract.return_value = mock_result + from fluid_build.providers.databricks_validation import DatabricksValidationProvider + v = ContractValidator( contract_path=Path("test.yaml"), use_cache=False, track_history=False, check_data=False ) - report = v.validate() - # No live Unity Catalog check yet: a warning, never an "unknown provider" error - warnings = [ - i - for i in report.issues - if i.message - == "Live validation against Unity Catalog is not available for provider 'databricks' yet" - ] - assert len(warnings) == 1 + with ( + patch("fluid_build.cli.contract_validation.DATABRICKS_VALIDATION_AVAILABLE", True), + patch.object(DatabricksValidationProvider, "validate_connection", return_value=True), + ): + report = v.validate() + # The Unity Catalog provider, never an "unknown provider" error + assert report.provider_name == "databricks" + assert isinstance(v.validation_provider, DatabricksValidationProvider) + assert not any("Unknown provider" in i.message for i in report.issues) @patch("fluid_build.cli.contract_validation.load_contract_with_overlay") @patch("fluid_build.cli.contract_validation.FluidSchemaManager") diff --git a/tests/test_contract_validation_branches.py b/tests/test_contract_validation_branches.py index 46b5b843..fa904583 100644 --- a/tests/test_contract_validation_branches.py +++ b/tests/test_contract_validation_branches.py @@ -268,7 +268,10 @@ def test_unknown_provider(self, MockSM, mock_load): @patch("fluid_build.cli.contract_validation.load_contract_with_overlay") @patch("fluid_build.cli.contract_validation.FluidSchemaManager") - def test_databricks_provider_not_implemented(self, MockSM, mock_load): + def test_databricks_connection_failure_says_what_to_fix(self, MockSM, mock_load): + from fluid_build.providers.databricks.statements import StatementError + from fluid_build.providers.databricks_validation import DatabricksValidationProvider + contract = self._make_contract() contract["exposes"][0]["binding"]["platform"] = "databricks" mock_load.return_value = contract @@ -277,8 +280,16 @@ def test_databricks_provider_not_implemented(self, MockSM, mock_load): mock_sm.validate_contract.return_value = mock_result v = ContractValidator(Path("test.yaml"), use_cache=False, check_data=False) - report = v.validate() - assert any("Unity Catalog is not available" in w.message for w in report.get_warnings()) + no_warehouse = StatementError("DATABRICKS_WAREHOUSE_ID is not set") + with ( + patch("fluid_build.cli.contract_validation.DATABRICKS_VALIDATION_AVAILABLE", True), + patch.object( + DatabricksValidationProvider, "validate_connection", side_effect=no_warehouse + ), + ): + report = v.validate() + errors = [i for i in report.get_errors() if i.category == "connection"] + assert errors and "DATABRICKS_WAREHOUSE_ID is not set" in errors[0].message @patch("fluid_build.cli.contract_validation.load_contract_with_overlay") @patch("fluid_build.cli.contract_validation.FluidSchemaManager") diff --git a/tests/test_contract_validation_ext2.py b/tests/test_contract_validation_ext2.py index 21d0b716..040f52e7 100644 --- a/tests/test_contract_validation_ext2.py +++ b/tests/test_contract_validation_ext2.py @@ -332,12 +332,27 @@ def test_unknown_provider_adds_error(self): errors = v.report.get_errors() assert any("Unknown provider" in e.message for e in errors) - def test_databricks_adds_warning(self): + def test_databricks_without_the_sdk_is_an_error(self): v = _make_validator_new(provider_name="databricks") v.contract = {"exposes": [{"binding": {"platform": "databricks"}}]} - v._detect_and_validate_provider() - warnings = v.report.get_warnings() - assert any("databricks" in w.message.lower() for w in warnings) + with patch("fluid_build.cli.contract_validation.DATABRICKS_VALIDATION_AVAILABLE", False): + v._detect_and_validate_provider() + errors = v.report.get_errors() + assert any("databricks-sdk" in e.message for e in errors) + assert any("data-product-forge[databricks]" in (e.suggestion or "") for e in errors) + + def test_databricks_builds_the_unity_catalog_provider(self): + from fluid_build.providers.databricks_validation import DatabricksValidationProvider + + v = _make_validator_new(provider_name="databricks") + v.contract = {"exposes": [{"binding": {"platform": "databricks"}}]} + with ( + patch("fluid_build.cli.contract_validation.DATABRICKS_VALIDATION_AVAILABLE", True), + patch.object(DatabricksValidationProvider, "validate_connection", return_value=True), + ): + v._detect_and_validate_provider() + assert isinstance(v.validation_provider, DatabricksValidationProvider) + assert not v.report.get_errors() def test_azure_warns_that_it_was_not_validated(self): # `fluid validate` accepts azure (schema enum) although no provider serves From aa5fb88ca708dc156c8ecac7c5023a419b4e9914 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:51:45 +0100 Subject: [PATCH 2/6] feat(test): --results-table and --results-dashboard for Databricks --results-table CATALOG.SCHEMA.TABLE appends one row per check of the run, the same cases --output junit reports (cli/_test_cases.py), to a Unity Catalog table created if missing in a schema the operator already has; the catalog and schema are never created, the product's own schema is refused, and a table without the result columns is never written. Every value is a bound parameter. --results-dashboard WORKSPACE_PATH keeps a Lakeview dashboard over the table (pass rate of the latest runs, pass rate per product by layer, failing checks, daily trend), found by its workspace path so no state file is needed, updated only when its definition differs, and published without embedded credentials. The row shape follows Databricks Labs DQX's summary-metrics table and Elementary's test results; the path lookup follows databricks-solutions/ai-dev-kit; widget shapes follow DQX's .lvdash.json. No code is taken from them (DQX is under the Databricks License). --- fluid_build/_error_catalog.py | 69 +++ .../references/error-codes.md | 9 + fluid_build/cli/_test_cases.py | 63 +++ fluid_build/cli/_test_dashboard_databricks.py | 493 ++++++++++++++++++ fluid_build/cli/_test_results_databricks.py | 363 +++++++++++++ fluid_build/cli/test.py | 121 +++-- tests/cli/test_fluid_test_dashboard.py | 371 +++++++++++++ tests/cli/test_fluid_test_results_table.py | 438 ++++++++++++++++ 8 files changed, 1898 insertions(+), 29 deletions(-) create mode 100644 fluid_build/cli/_test_cases.py create mode 100644 fluid_build/cli/_test_dashboard_databricks.py create mode 100644 fluid_build/cli/_test_results_databricks.py create mode 100644 tests/cli/test_fluid_test_dashboard.py create mode 100644 tests/cli/test_fluid_test_results_table.py diff --git a/fluid_build/_error_catalog.py b/fluid_build/_error_catalog.py index a7f6f068..0c2d340c 100644 --- a/fluid_build/_error_catalog.py +++ b/fluid_build/_error_catalog.py @@ -638,6 +638,75 @@ def slug_for(event: str) -> str: ["Pass the product id to roll back: fluid rollback "], None, ), + # ── fluid test results table / dashboard (Databricks) ────────────────── + "test_results_table_invalid": ( + [ + "Name the table as catalog.schema.table, e.g. --results-table ops.dq.fluid_test_results", + "Each part may hold letters, digits and underscores only", + ], + None, + ), + "test_results_table_in_product_schema": ( + [ + "Put the results table in a schema of its own: every product writes to it, " + + "and fluid apply owns each product's schema", + "Create one with CREATE SCHEMA .dq, then pass " + + "--results-table .dq.fluid_test_results", + ], + None, + ), + "test_results_table_mismatch": ( + [ + "The table exists without the fluid test results columns, so nothing was written", + "Point --results-table at a new table name; forge creates it with the right columns", + ], + None, + ), + "test_results_write_failed": ( + [ + "Check DATABRICKS_HOST, the token or OAuth client, and DATABRICKS_WAREHOUSE_ID", + "The schema must exist and this identity needs USE CATALOG, USE SCHEMA, " + + "CREATE TABLE and MODIFY on it", + "The test report above is unaffected: rerun fluid test once the write can succeed", + ], + None, + ), + "test_results_dashboard_needs_table": ( + [ + "--results-dashboard draws the --results-table table: pass both", + ], + None, + ), + "test_results_table_needs_native_engine": ( + [ + "--results-table records the native engine's per-rule results; drop --engine soda", + "Or keep --engine soda and its JUnit or JSON output, without --results-table", + ], + None, + ), + "test_results_dashboard_invalid": ( + [ + "Give a workspace path whose last part is the name, e.g. " + + "--results-dashboard '/Workspace/Shared/fluid/Data quality'", + "The path must start with / and have no empty, . or .. parts", + ], + None, + ), + "test_results_dashboard_path_taken": ( + [ + "Something other than a dashboard already sits at that workspace path", + "Choose another --results-dashboard path, or move that object away", + ], + None, + ), + "test_results_dashboard_failed": ( + [ + "The results were written; only the dashboard step failed", + "This identity needs CAN MANAGE on the dashboard (or write access to its folder) " + + "and CAN USE on DATABRICKS_WAREHOUSE_ID", + ], + None, + ), # ── apply safety gates ───────────────────────────────────────────────── "iceberg_catalog_move_blocked": ( [ diff --git a/fluid_build/agent_skills/skills/troubleshooting-fluid-errors/references/error-codes.md b/fluid_build/agent_skills/skills/troubleshooting-fluid-errors/references/error-codes.md index 1bff8355..6a08fd48 100644 --- a/fluid_build/agent_skills/skills/troubleshooting-fluid-errors/references/error-codes.md +++ b/fluid_build/agent_skills/skills/troubleshooting-fluid-errors/references/error-codes.md @@ -91,4 +91,13 @@ Docs pages are relative to https://agenticstiger.github.io/forge_docs/ | `ERR_SCHEDULE_SYNC_DAGS_DIR_NOT_DIRECTORY` | The --dags-dir value must be an existing directory. Create it (mkdir -p) or point --dags-dir at your Airflow dags/ folder. | advanced/production-troubleshooting.html | | `ERR_SCHEDULE_SYNC_UNHANDLED_SCHEME` | Use a supported destination scheme (file / scp / git+ssh) for --to. | advanced/production-troubleshooting.html | | `ERR_ROLLBACK_PRODUCT_ID_EMPTY` | Pass the product id to roll back: fluid rollback . | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_TABLE_INVALID` | Name the table as catalog.schema.table, e.g. --results-table ops.dq.fluid_test_results. Each part may hold letters, digits and underscores only. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_TABLE_IN_PRODUCT_SCHEMA` | Put the results table in a schema of its own: every product writes to it, and fluid apply owns each product's schema. Create one with CREATE SCHEMA .dq, then pass --results-table .dq.fluid_test_results. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_TABLE_MISMATCH` | The table exists without the fluid test results columns, so nothing was written. Point --results-table at a new table name; forge creates it with the right columns. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_WRITE_FAILED` | Check DATABRICKS_HOST, the token or OAuth client, and DATABRICKS_WAREHOUSE_ID. The schema must exist and this identity needs USE CATALOG, USE SCHEMA, CREATE TABLE and MODIFY on it. The test report above is unaffected: rerun fluid test once the write can succeed. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_DASHBOARD_NEEDS_TABLE` | --results-dashboard draws the --results-table table: pass both. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_TABLE_NEEDS_NATIVE_ENGINE` | --results-table records the native engine's per-rule results; drop --engine soda. Or keep --engine soda and its JUnit or JSON output, without --results-table. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_DASHBOARD_INVALID` | Give a workspace path whose last part is the name, e.g. --results-dashboard '/Workspace/Shared/fluid/Data quality'. The path must start with / and have no empty, . or .. parts. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_DASHBOARD_PATH_TAKEN` | Something other than a dashboard already sits at that workspace path. Choose another --results-dashboard path, or move that object away. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_DASHBOARD_FAILED` | The results were written; only the dashboard step failed. This identity needs CAN MANAGE on the dashboard (or write access to its folder) and CAN USE on DATABRICKS_WAREHOUSE_ID. | advanced/production-troubleshooting.html | | `ERR_ICEBERG_CATALOG_MOVE_BLOCKED` | Run the printed `tofu state rm` commands: they release the resources from this contract's OpenTofu state and touch nothing in the cloud. Then re-run fluid apply. The released Glue database/table or Snowflake EXTERNAL VOLUME stays in place; delete it by hand only if nothing else uses it. | cli/apply.html#iceberg-catalog-move-guard | diff --git a/fluid_build/cli/_test_cases.py b/fluid_build/cli/_test_cases.py new file mode 100644 index 00000000..974fa9a6 --- /dev/null +++ b/fluid_build/cli/_test_cases.py @@ -0,0 +1,63 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""The test cases one ``fluid test`` run reports, in one place. + +``--output junit`` and ``--results-table`` both report a run as the same +cases: one per executed data-quality rule, then one per structural check +category. Reading both from here keeps Jenkins' test list and the results +table from ever disagreeing about what ran. +""" + +from __future__ import annotations + +from typing import Dict, List, Tuple + +#: The structural check categories, in report order. ``quality`` stands for +#: the data-quality rules only when no rule ran on its own (then there is no +#: per-rule case to carry them). +STRUCTURAL_CATEGORIES: Tuple[str, ...] = ( + "schema", + "connection", + "binding", + "missing_resource", + "missing_field", + "type_mismatch", + "mode_mismatch", + "extra_field", + "empty_table", + "row_count_below_threshold", + "quality", + "metadata", + "drift", +) + + +def rule_checks(report) -> List: + """The data-quality rules that ran, one ``CheckOutcome`` each.""" + return [c for c in getattr(report, "checks", []) if c.category == "quality"] + + +def structural_categories(report) -> Tuple[List[str], List[str], Dict[str, List]]: + """``(built-in, extra, issues by category)`` for the structural cases. + + ``built-in`` is :data:`STRUCTURAL_CATEGORIES` (without ``quality`` when a + rule ran on its own); ``extra`` is every other category the run raised an + issue in, in first-seen order. Each case passes unless it has an + error-severity issue. + """ + by_category: Dict[str, List] = {} + for issue in report.issues: + by_category.setdefault(issue.category, []).append(issue) + builtin = list(STRUCTURAL_CATEGORIES) + if rule_checks(report): + # The per-rule cases already carry every rule outcome; a roll-up + # ``quality`` case on top would double-count them. + builtin.remove("quality") + extra = [c for c in by_category if c not in builtin] + if rule_checks(report) and "quality" in extra: + extra.remove("quality") + return builtin, extra, by_category + + +__all__ = ["STRUCTURAL_CATEGORIES", "rule_checks", "structural_categories"] diff --git a/fluid_build/cli/_test_dashboard_databricks.py b/fluid_build/cli/_test_dashboard_databricks.py new file mode 100644 index 00000000..6968ab19 --- /dev/null +++ b/fluid_build/cli/_test_dashboard_databricks.py @@ -0,0 +1,493 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``fluid test --results-dashboard``: an AI/BI (Lakeview) dashboard over the results table. + +The dashboard is found by its workspace path, ``/.lvdash.json`` +(``GET /api/2.0/workspace/get-status``, whose ``resource_id`` is the dashboard +id), so every run, on any machine, finds the same one and no state file is +kept. The lookup is the one databricks-solutions/ai-dev-kit uses; Databricks +Labs DQX records the id in its install state instead, which an ephemeral CI +agent would lose. + +- Missing: the folder is made (``workspace/mkdirs``), the dashboard created + (``POST /api/2.0/lakeview/dashboards``) and published. +- Present with this definition: left alone (published if it never was). +- Present with another definition (another table, or an older forge): replaced + with ``PATCH``, then published again. + +It is published without embedded credentials: each viewer reads the results +table as themself, so they need ``SELECT`` on it and access to the warehouse. + +The widget shapes (counter v2, bar and line v3, table v1, text box) follow the +``.lvdash.json`` files Databricks Labs DQX ships; nothing is copied from them. +""" + +from __future__ import annotations + +import json +import sys +from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple +from urllib.parse import quote + +from ._common import CLIError + +DASHBOARDS_PATH = "/api/2.0/lakeview/dashboards" +GET_STATUS_PATH = "/api/2.0/workspace/get-status" +MKDIRS_PATH = "/api/2.0/workspace/mkdirs" +_SUFFIX = ".lvdash.json" + +#: Rows that are fluid test's own structural checks, not data-quality rules. +_RULES_ONLY = "COALESCE(rule_type, '') <> 'structure'" + + +def _status(message: str) -> None: + sys.stderr.write(message + "\n") + + +def parse_dashboard_path(value: Any) -> Tuple[str, str]: + """``(folder, name)`` of ``--results-dashboard /Folder/Name``.""" + text = str(value or "").strip() + if text.endswith(_SUFFIX): + text = text[: -len(_SUFFIX)] + segments = text.split("/") + problems = ( + not text.startswith("/") + or len(segments) < 3 + or any(not s.strip() or s in (".", "..") for s in segments[1:]) + or any(ord(ch) < 32 or ch in "\\\x7f" for ch in text) + ) + if problems: + raise CLIError( + 2, + "test_results_dashboard_invalid", + { + "results_dashboard": str(value), + "expected": "a workspace path: /Workspace/Shared/fluid/Data quality " + "(the last part is the dashboard's name)", + }, + ) + return "/".join(segments[:-1]), segments[-1] + + +def _sql_lines(sql: str) -> List[str]: + lines = sql.strip().splitlines() + return [line + "\n" for line in lines[:-1]] + lines[-1:] + + +def _table_column(name: str, kind: str, title: str, order: int) -> Dict[str, Any]: + """One column of a table widget (spec version 1), with every key the editor + writes: Lakeview refuses a column without them ("Invalid widget definition").""" + column: Dict[str, Any] = { + "fieldName": name, + "booleanValues": ["false", "true"], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": True, + "type": kind, + "displayAs": kind, + "visible": True, + "order": order, + "title": title, + "allowSearch": kind == "string", + "alignContent": "left" if kind == "string" else "right", + "allowHTML": False, + "highlightLinks": False, + "useMonospaceFont": False, + "preserveWhitespace": False, + } + if kind == "datetime": + column["dateTimeFormat"] = "YYYY-MM-DD HH:mm:ss" + return column + + +def dashboard_definition(table: Tuple[str, str, str]) -> Dict[str, Any]: + """The ``serialized_dashboard`` for a results table (validated name parts).""" + ref = ".".join(f"`{p}`" for p in table) + latest = ( + f"WITH latest AS (\n SELECT * FROM {ref}\n" + " QUALIFY run_time = MAX(run_time) OVER (PARTITION BY product_id)\n)\n" + ) + pass_pct = "ROUND(100 * AVG(CASE WHEN passed THEN 1.0 ELSE 0.0 END), 1)" + datasets = [ + { + "name": "fluid_dq_overall", + "displayName": "Latest run, all products", + "queryLines": _sql_lines( + f"{latest}SELECT {pass_pct} AS pass_rate_pct,\n" + " COUNT(DISTINCT product_id) AS products,\n" + " SUM(CASE WHEN passed THEN 0 ELSE 1 END) AS failing_rules\n" + f"FROM latest WHERE {_RULES_ONLY}" + ), + }, + { + "name": "fluid_dq_products", + "displayName": "Latest run per product", + "queryLines": _sql_lines( + f"{latest}SELECT product_id, COALESCE(layer, 'unset') AS layer,\n" + " COUNT(*) AS rules,\n" + " SUM(CASE WHEN passed THEN 0 ELSE 1 END) AS failing_rules,\n" + f" {pass_pct} AS pass_rate_pct,\n" + " MAX(run_time) AS last_run\n" + f"FROM latest WHERE {_RULES_ONLY}\n" + "GROUP BY product_id, COALESCE(layer, 'unset')" + ), + }, + { + "name": "fluid_dq_trend", + "displayName": "Daily pass rate per product", + "queryLines": _sql_lines( + "SELECT DATE(run_time) AS run_date, product_id,\n" + f" {pass_pct} AS pass_rate_pct\n" + f"FROM {ref} WHERE {_RULES_ONLY}\n" + "GROUP BY DATE(run_time), product_id" + ), + }, + { + "name": "fluid_dq_failing", + "displayName": "Failing checks, latest run", + "queryLines": _sql_lines( + f"{latest}SELECT product_id, layer, rule_id, rule_type, severity,\n" + " observed_value, threshold, message, run_time\n" + "FROM latest WHERE NOT passed\n" + "ORDER BY product_id, rule_id" + ), + }, + ] + + def counter(name: str, title: str, field: str, x: int) -> Dict[str, Any]: + agg = f"sum({field})" + return { + "widget": { + "name": name, + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "fluid_dq_overall", + "fields": [{"name": agg, "expression": f"SUM(`{field}`)"}], + "disaggregated": False, + }, + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": {"value": {"fieldName": agg, "displayName": title}}, + "frame": {"title": title, "showTitle": True}, + }, + }, + "position": {"x": x, "y": 1, "width": 2, "height": 2}, + } + + def fields(*names: str) -> List[Dict[str, str]]: + return [{"name": n, "expression": f"`{n}`"} for n in names] + + columns = ( + ("product_id", "string", "Product"), + ("layer", "string", "Layer"), + ("rule_id", "string", "Rule"), + ("rule_type", "string", "Type"), + ("severity", "string", "Severity"), + ("observed_value", "string", "Observed"), + ("threshold", "string", "Threshold"), + ("message", "string", "Message"), + ("run_time", "datetime", "Run time"), + ) + layout = [ + { + "widget": { + "name": "fluid_dq_title", + "multilineTextboxSpec": { + "lines": [ + "## Data quality by product\n", + f"Every `fluid test --results-table {'.'.join(table)}` run: each " + "product's data-quality rules, passed or failed, for Bronze products " + "and dbt-built ones alike.", + ] + }, + }, + "position": {"x": 0, "y": 0, "width": 6, "height": 1}, + }, + counter("fluid_dq_pass_rate", "Rules passed, latest runs (%)", "pass_rate_pct", 0), + counter("fluid_dq_product_count", "Products tested", "products", 2), + counter("fluid_dq_failing_count", "Failing rules, latest runs", "failing_rules", 4), + { + "widget": { + "name": "fluid_dq_pass_rate_by_product", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "fluid_dq_products", + "fields": fields("product_id", "layer", "pass_rate_pct"), + "disaggregated": True, + }, + } + ], + "spec": { + "version": 3, + "widgetType": "bar", + "encodings": { + "x": { + "fieldName": "product_id", + "displayName": "Product", + "scale": {"type": "categorical"}, + }, + "y": { + "fieldName": "pass_rate_pct", + "displayName": "Rules passed (%)", + "scale": {"type": "quantitative"}, + }, + "color": { + "fieldName": "layer", + "displayName": "Layer", + "scale": {"type": "categorical"}, + }, + "label": {"show": True}, + }, + "frame": {"title": "Pass rate per product, latest run", "showTitle": True}, + }, + }, + "position": {"x": 0, "y": 3, "width": 6, "height": 6}, + }, + { + "widget": { + "name": "fluid_dq_trend", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "fluid_dq_trend", + "fields": fields("run_date", "product_id", "pass_rate_pct"), + "disaggregated": True, + }, + } + ], + "spec": { + "version": 3, + "widgetType": "line", + "encodings": { + "x": { + "fieldName": "run_date", + "displayName": "Day", + "scale": {"type": "temporal"}, + }, + "y": { + "fieldName": "pass_rate_pct", + "displayName": "Rules passed (%)", + "scale": {"type": "quantitative"}, + }, + "color": { + "fieldName": "product_id", + "displayName": "Product", + "scale": {"type": "categorical"}, + }, + }, + "frame": {"title": "Pass rate over time", "showTitle": True}, + }, + }, + "position": {"x": 0, "y": 9, "width": 6, "height": 6}, + }, + { + "widget": { + "name": "fluid_dq_failing", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "fluid_dq_failing", + "fields": fields(*(c[0] for c in columns)), + "disaggregated": True, + }, + } + ], + "spec": { + "version": 1, + "widgetType": "table", + "encodings": { + "columns": [ + _table_column(name, kind, title, 100000 + i) + for i, (name, kind, title) in enumerate(columns) + ] + }, + "invisibleColumns": [], + "allowHTMLByDefault": False, + "itemsPerPage": 25, + "paginationSize": "default", + "condensed": True, + "withRowNumber": False, + "frame": {"title": "Failing checks, latest run", "showTitle": True}, + }, + }, + "position": {"x": 0, "y": 15, "width": 6, "height": 8}, + }, + ] + return { + "datasets": datasets, + "pages": [ + { + "name": "fluid_dq", + "displayName": "Data quality", + "pageType": "PAGE_TYPE_CANVAS", + "layout": layout, + } + ], + } + + +def _status_code_is(exc: BaseException, *codes: str) -> bool: + text = f"{getattr(exc, 'error_code', '')} {exc}".upper() + return any(code in text for code in codes) + + +def _is_not_found(exc: BaseException) -> bool: + for attr in ("status_code", "status", "http_status"): + if getattr(exc, attr, None) == 404: + return True + return type(exc).__name__ in {"NotFound", "ResourceDoesNotExist"} or _status_code_is( + exc, "RESOURCE_DOES_NOT_EXIST", "NOT_FOUND" + ) + + +def _same(theirs: Any, ours: Mapping[str, Any]) -> bool: + """Whether a stored ``serialized_dashboard`` holds our datasets and pages.""" + try: + stored = json.loads(theirs) if isinstance(theirs, str) else theirs + except ValueError: + return False + if not isinstance(stored, dict): + return False + return all( + json.dumps(stored.get(key), sort_keys=True) == json.dumps(ours.get(key), sort_keys=True) + for key in ("datasets", "pages") + ) + + +class _Lakeview: + """The few REST calls this needs, through the SDK's ``api_client.do``.""" + + def __init__(self, client: Any) -> None: + self._client = client + + def do( + self, + method: str, + path: str, + *, + query: Optional[Mapping[str, Any]] = None, + body: Optional[Mapping[str, Any]] = None, + ) -> Dict[str, Any]: + out = self._client.api_client.do( + method, path, query=dict(query) if query else None, body=dict(body) if body else None + ) + return out if isinstance(out, dict) else {} + + def find(self, folder: str, name: str) -> Optional[str]: + path = f"{folder}/{name}{_SUFFIX}" + try: + found = self.do("GET", GET_STATUS_PATH, query={"path": path}) + except Exception as exc: # noqa: BLE001 + if _is_not_found(exc): + return None + raise + if str(found.get("object_type") or "").upper() != "DASHBOARD": + raise CLIError( + 1, + "test_results_dashboard_path_taken", + {"results_dashboard": f"{folder}/{name}", "found": found.get("object_type")}, + ) + return str(found.get("resource_id") or "") or None + + +def ensure_dashboard( + dashboard_path: str, + table: Tuple[str, str, str], + *, + client_factory: Optional[Callable[[], Any]] = None, + warehouse_id: Optional[str] = None, +) -> Dict[str, Any]: + """Create, update or leave the dashboard; return what happened. + + Raises :class:`CLIError` with what to do when the dashboard could not be + made or published. + """ + from fluid_build.providers.databricks.statements import ( + default_client, + scrub, + warehouse_id_from_env, + ) + + folder, name = parse_dashboard_path(dashboard_path) + definition = dashboard_definition(table) + serialized = json.dumps(definition, separators=(",", ":")) + try: + client = (client_factory or default_client)() + warehouse = warehouse_id or warehouse_id_from_env() + api = _Lakeview(client) + dashboard_id = api.find(folder, name) + action = "unchanged" + if dashboard_id is None: + api.do("POST", MKDIRS_PATH, body={"path": folder}) + body = { + "display_name": name, + "parent_path": folder, + "serialized_dashboard": serialized, + "warehouse_id": warehouse, + } + try: + dashboard_id = str(api.do("POST", DASHBOARDS_PATH, body=body)["dashboard_id"]) + action = "created" + except Exception as exc: # noqa: BLE001 + # Another run created it between the lookup and here. + if not _status_code_is(exc, "ALREADY_EXISTS", "ALREADY EXISTS"): + raise + dashboard_id = api.find(folder, name) + if dashboard_id is None: + raise + if action != "created": + current = api.do("GET", f"{DASHBOARDS_PATH}/{quote(dashboard_id, safe='')}") + if not _same(current.get("serialized_dashboard"), definition) or ( + current.get("warehouse_id") != warehouse + ): + patch = {"serialized_dashboard": serialized, "warehouse_id": warehouse} + if current.get("etag"): + patch["etag"] = current["etag"] + api.do("PATCH", f"{DASHBOARDS_PATH}/{quote(dashboard_id, safe='')}", body=patch) + action = "updated" + published_path = f"{DASHBOARDS_PATH}/{quote(dashboard_id, safe='')}/published" + publish = action != "unchanged" + if not publish: + try: + api.do("GET", published_path) + except Exception as exc: # noqa: BLE001 + if not _is_not_found(exc): + raise + publish = True + if publish: + api.do( + "POST", published_path, body={"embed_credentials": False, "warehouse_id": warehouse} + ) + except CLIError: + raise + except Exception as exc: # noqa: BLE001 - every failure becomes one safe message + raise CLIError( + 1, + "test_results_dashboard_failed", + { + "results_dashboard": f"{folder}/{name}", + "error": f"{type(exc).__name__}: {scrub(exc)}", + }, + ) from None + host = str(getattr(getattr(client, "config", None), "host", "") or "").rstrip("/") + url = f"{host}/dashboardsv3/{dashboard_id}/published" if host else None + _status(f"Dashboard {action}: {folder}/{name}" + (f" ({url})" if url else "")) + return {"action": action, "dashboard_id": dashboard_id, "url": url} + + +__all__ = ["dashboard_definition", "ensure_dashboard", "parse_dashboard_path"] diff --git a/fluid_build/cli/_test_results_databricks.py b/fluid_build/cli/_test_results_databricks.py new file mode 100644 index 00000000..dfee2457 --- /dev/null +++ b/fluid_build/cli/_test_results_databricks.py @@ -0,0 +1,363 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``fluid test --results-table``: append one run's results to a Unity Catalog table. + +One row per test case of the run, the same cases ``--output junit`` reports +(``_test_cases.py``): each data-quality rule that ran, then each structural +check category (``rule_type = 'structure'``). The table is shared by every +product, so it lives in a schema the operator names, never in a product's own +schema (``fluid apply`` owns those). + +The shape follows the one-row-per-check-per-run results tables of Databricks +Labs DQX (its summary-metrics table: run id, run time, metric, value) and of +Elementary's ``elementary_test_results``; the pattern is adapted, no code is +taken (DQX is under the Databricks License). + +What is created: the table, with ``CREATE TABLE IF NOT EXISTS``, owned by the +identity that first runs this. The catalog and the schema are the operator's +and are never created. An existing table must have every column below, or +nothing is written: a name that points at some other table writes nothing +into it. + +Every value is bound as a ``:name`` parameter; the three name parts are +validated identifiers, backtick-quoted. +""" + +from __future__ import annotations + +import sys +import uuid +from datetime import datetime, timezone +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple +from urllib.parse import quote + +from fluid_build.providers._sql_safety import validate_ident + +from ._common import CLIError +from ._test_cases import rule_checks, structural_categories + +#: ``(column, type, comment)`` in table order. +RESULT_COLUMNS: Tuple[Tuple[str, str, str], ...] = ( + ("run_id", "STRING NOT NULL", "One fluid test invocation"), + ("run_time", "TIMESTAMP NOT NULL", "When the checks ran (UTC)"), + ("product_id", "STRING NOT NULL", "The contract id"), + ("product_version", "STRING", "The contract version"), + ("layer", "STRING", "metadata.layer: Bronze, Silver or Gold"), + ("expose_id", "STRING", "The expose the check ran against"), + ("table_name", "STRING", "catalog.schema.table of that expose"), + ("rule_id", "STRING NOT NULL", "dq.rules[].id, or the name of the structural check"), + ("rule_type", "STRING", "dq.rules[].type, or structure for the checks of fluid test itself"), + ("severity", "STRING", "The declared severity of the rule"), + ("passed", "BOOLEAN NOT NULL", "Whether the check passed"), + ("observed_value", "STRING", "What was measured"), + ("threshold", "STRING", "What it had to be"), + ("message", "STRING", "What happened"), + ("fluid_version", "STRING", "The data-product-forge version that ran the checks"), +) + +#: ``rule_type`` of the rows for ``fluid test``'s own structural checks. +STRUCTURE_RULE_TYPE = "structure" + +_CASTS = {"run_time": "TIMESTAMP", "passed": "BOOLEAN"} +_TABLE_COMMENT = "fluid test results: one row per check per run (data-product-forge)" +_BATCH_ROWS = 25 +_MAX_TEXT = 2000 +TABLES_PATH = "/api/2.1/unity-catalog/tables" + + +def _status(message: str) -> None: + """One status line on stderr, so ``--output json`` keeps stdout clean.""" + sys.stderr.write(message + "\n") + + +def parse_results_table(value: Any) -> Tuple[str, str, str]: + """``catalog.schema.table`` of ``--results-table``, each part validated.""" + parts = str(value or "").split(".") + try: + if len(parts) != 3: + raise ValueError(value) + return validate_ident(parts[0]), validate_ident(parts[1]), validate_ident(parts[2]) + except ValueError: + raise CLIError( + 2, + "test_results_table_invalid", + { + "results_table": str(value), + "expected": "catalog.schema.table, each part letters, digits and underscores", + }, + ) from None + + +def _expose_tables(contract: Mapping[str, Any]) -> Dict[str, Tuple[str, str, str]]: + """``expose id -> (catalog, schema, table)`` for each expose bound to a table.""" + out: Dict[str, Tuple[str, str, str]] = {} + for idx, expose in enumerate(contract.get("exposes") or []): + if not isinstance(expose, dict): + continue + binding = expose.get("binding") if isinstance(expose.get("binding"), dict) else {} + location = binding.get("location") if isinstance(binding.get("location"), dict) else {} + catalog = location.get("database") or location.get("catalog") + schema, table = location.get("schema"), location.get("table") + if catalog and schema and table: + key = str(expose.get("id") or expose.get("exposeId") or f"expose_{idx}") + out[key] = (str(catalog), str(schema), str(table)) + return out + + +def _expose_versions(contract: Mapping[str, Any]) -> Dict[str, str]: + """``expose id -> exposes[].version`` (the version ``fluid apply`` tags the table with).""" + out: Dict[str, str] = {} + for idx, expose in enumerate(contract.get("exposes") or []): + if isinstance(expose, dict) and expose.get("version") is not None: + key = str(expose.get("id") or expose.get("exposeId") or f"expose_{idx}") + out[key] = str(expose["version"]) + return out + + +def refuse_product_schema(table: Tuple[str, str, str], contract: Mapping[str, Any]) -> None: + """A results table in one of the tested product's own schemas is refused.""" + for catalog, schema, _ in _expose_tables(contract).values(): + if (catalog.lower(), schema.lower()) == (table[0].lower(), table[1].lower()): + raise CLIError( + 2, + "test_results_table_in_product_schema", + { + "results_table": ".".join(table), + "product_schema": f"{catalog}.{schema}", + "why": "the results table is shared by every product, and fluid apply " + "owns this product's schema", + "fix": f"name a schema of its own, e.g. {table[0]}.dq.{table[2]}", + }, + ) + + +def _text(value: Any) -> Optional[str]: + if value is None: + return None + text = str(value) + return text if len(text) <= _MAX_TEXT else text[: _MAX_TEXT - 3] + "..." + + +def _severity(value: Any) -> Optional[str]: + text = str(value or "").strip().lower() + return text or None + + +def results_rows( + report: Any, + contract: Mapping[str, Any], + *, + run_id: str, + fluid_version: Optional[str], +) -> List[Dict[str, Any]]: + """The rows of one run: each rule that ran, then each structural check.""" + tables = _expose_tables(contract) + versions = _expose_versions(contract) + # The contract's own version when it has one; the report's "0.0.0" is a + # placeholder for "none declared", never a version. + product_version = contract.get("version") + product_version = None if product_version is None else str(product_version) + metadata = contract.get("metadata") if isinstance(contract.get("metadata"), dict) else {} + run_time = report.validation_time.astimezone(timezone.utc) + only_table = next(iter(tables.items())) if len(tables) == 1 else None + common = { + "run_id": run_id, + "run_time": run_time, + "product_id": report.contract_id, + "layer": _text(metadata.get("layer")), + "fluid_version": fluid_version, + } + rows: List[Dict[str, Any]] = [] + for check in rule_checks(report): + expose_id = check.expose_id or (check.name.rsplit(".", 1)[0] if "." in check.name else None) + table = tables.get(expose_id or "") + rows.append( + { + **common, + "product_version": _text(versions.get(expose_id or "", product_version)), + "expose_id": expose_id, + "table_name": ".".join(table) if table else None, + "rule_id": check.rule_id or check.name.rsplit(".", 1)[-1], + "rule_type": check.rule_type, + "severity": _severity(check.severity), + "passed": bool(check.passed), + "observed_value": _text(check.actual), + "threshold": _text(check.expected), + "message": _text(check.message), + } + ) + builtin, extra, by_category = structural_categories(report) + from fluid_build.severity import is_error, is_warning + + for category in builtin + extra: + issues = by_category.get(category, []) + errors = [i for i in issues if is_error(i.severity)] + warnings = [i for i in issues if is_warning(i.severity)] + shown = errors + warnings + single = shown[0] if len(shown) == 1 else None + if single is not None: + observed, threshold = single.actual, single.expected + elif shown: + observed, threshold = f"{len(errors)} error(s), {len(warnings)} warning(s)", None + else: + observed, threshold = None, None + only = only_table[0] if only_table else None + rows.append( + { + **common, + "product_version": _text(versions.get(only or "", product_version)), + "expose_id": only, + "table_name": ".".join(only_table[1]) if only_table else None, + "rule_id": category, + "rule_type": STRUCTURE_RULE_TYPE, + "severity": "error" if errors else ("warning" if warnings else None), + "passed": not errors, + "observed_value": _text(observed), + "threshold": _text(threshold), + "message": _text("; ".join(i.message for i in shown) or "OK"), + } + ) + return rows + + +def _create_table_sql(ref: str) -> str: + from fluid_build.providers._sql_safety import quote_string_literal_for + + columns = ",\n ".join( + f"{name} {kind} COMMENT {quote_string_literal_for('databricks', note)}" + for name, kind, note in RESULT_COLUMNS + ) + comment = quote_string_literal_for("databricks", _TABLE_COMMENT) + return f"CREATE TABLE IF NOT EXISTS {ref} (\n {columns}\n) USING DELTA\nCOMMENT {comment}" + + +def _value(name: str, value: Any) -> Any: + if value is None: + return None + if name == "passed": + return "true" if value else "false" + if name == "run_time" and isinstance(value, datetime): + return value.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + return value + + +def insert_statements(ref: str, rows: Sequence[Mapping[str, Any]]) -> List[Tuple[str, List]]: + """``(statement, parameters)`` for ``rows``, at most ``_BATCH_ROWS`` rows each.""" + from fluid_build.providers.databricks.statements import param + + names = [name for name, _, _ in RESULT_COLUMNS] + out: List[Tuple[str, List]] = [] + for start in range(0, len(rows), _BATCH_ROWS): + batch = rows[start : start + _BATCH_ROWS] + tuples: List[str] = [] + params: List[Dict[str, Any]] = [] + for i, row in enumerate(batch): + cells = [] + for j, name in enumerate(names): + marker = f"r{i}c{j}" + cells.append( + f"CAST(:{marker} AS {_CASTS[name]})" if name in _CASTS else f":{marker}" + ) + params.append(param(marker, _value(name, row.get(name)))) + tuples.append("(" + ", ".join(cells) + ")") + statement = f"INSERT INTO {ref} ({', '.join(names)}) VALUES\n" + ",\n".join(tuples) + out.append((statement, params)) + return out + + +def _fix_for(error: str, table: Tuple[str, str, str]) -> Optional[str]: + """The one action that fixes a failed write, when the error says which.""" + catalog, schema, _ = table + upper = error.upper() + if "SCHEMA_NOT_FOUND" in upper or "NO_SUCH_SCHEMA" in upper: + return f"Create the schema first (forge never creates it): CREATE SCHEMA `{catalog}`.`{schema}`" + if "CATALOG_NOT_FOUND" in upper or "NO_SUCH_CATALOG" in upper: + return f"Catalog `{catalog}` does not exist; name an existing catalog" + if "PERMISSION" in upper or "INSUFFICIENT_PRIVILEGES" in upper or "NOT AUTHORIZED" in upper: + return ( + f"Grant this identity USE CATALOG on `{catalog}`, USE SCHEMA and CREATE TABLE on " + f"`{catalog}`.`{schema}`, and MODIFY on the table once it exists" + ) + return None + + +def _check_columns(client: Any, full_name: str) -> None: + """Refuse a table that lacks one of :data:`RESULT_COLUMNS`.""" + table = client.api_client.do("GET", f"{TABLES_PATH}/{quote(full_name, safe='')}") + have = { + str(c.get("name", "")).lower() + for c in (table.get("columns") or [] if isinstance(table, dict) else []) + if isinstance(c, dict) + } + missing = [name for name, _, _ in RESULT_COLUMNS if name not in have] + if missing: + raise CLIError( + 1, + "test_results_table_mismatch", + { + "results_table": full_name, + "missing_columns": ", ".join(missing), + "why": "the table exists and is not a fluid test results table, so nothing " + "was written to it", + }, + ) + + +def write_results( + report: Any, + contract: Mapping[str, Any], + results_table: str, + *, + client_factory: Optional[Callable[[], Any]] = None, + warehouse_id: Optional[str] = None, + run_id: Optional[str] = None, +) -> Tuple[str, int]: + """Append the run's rows; return ``(table, rows written)``. + + Raises :class:`CLIError` with what to do when nothing could be written. + """ + from fluid_build import __version__ + from fluid_build.providers.databricks.statements import ( + StatementRunner, + default_client, + scrub, + warehouse_id_from_env, + ) + + table = parse_results_table(results_table) + refuse_product_schema(table, contract) + full_name = ".".join(table) + ref = ".".join(f"`{p}`" for p in table) + rows = results_rows( + report, contract, run_id=run_id or uuid.uuid4().hex, fluid_version=__version__ + ) + try: + client = (client_factory or default_client)() + runner = StatementRunner(client, warehouse_id or warehouse_id_from_env()) + runner.run(_create_table_sql(ref)) + _check_columns(client, full_name) + for statement, params in insert_statements(ref, rows): + runner.run(statement, params) + except CLIError: + raise + except Exception as exc: # noqa: BLE001 - every failure becomes one safe message + message = f"{type(exc).__name__}: {scrub(exc)}" + context: Dict[str, Any] = {"results_table": full_name, "error": message} + fix = _fix_for(message, table) + if fix: + context["fix"] = fix + raise CLIError(1, "test_results_write_failed", context) from None + _status(f"Results: {len(rows)} row(s) appended to {full_name}") + return full_name, len(rows) + + +__all__ = [ + "RESULT_COLUMNS", + "STRUCTURE_RULE_TYPE", + "insert_statements", + "parse_results_table", + "refuse_product_schema", + "results_rows", + "write_results", +] diff --git a/fluid_build/cli/test.py b/fluid_build/cli/test.py index 14cefc7a..ad9fa0b1 100644 --- a/fluid_build/cli/test.py +++ b/fluid_build/cli/test.py @@ -33,7 +33,7 @@ import sys import xml.etree.ElementTree as ET from pathlib import Path -from typing import Dict, List, Optional +from typing import List, Optional from fluid_build.cli.console import cprint, cprint_literal, literal_text, success, warning from fluid_build.cli.console import error as console_error @@ -116,6 +116,12 @@ def register(subparsers: argparse._SubParsersAction) -> None: # Enable drift detection fluid test contract.fluid.yaml --check-drift + + # Databricks: JUnit for Jenkins, plus one row per check in a Unity Catalog + # table and an AI/BI dashboard over it + fluid test contract.fluid.yaml --env databricks --output junit \ + --output-file results.xml --results-table ops.dq.fluid_test_results \ + --results-dashboard "/Workspace/Shared/fluid/Data quality" """, formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -220,9 +226,78 @@ def register(subparsers: argparse._SubParsersAction) -> None: ), ) + # --- results table + dashboard (Databricks Unity Catalog) --- + p.add_argument( + "--results-table", + metavar="CATALOG.SCHEMA.TABLE", + help=( + "Append this run's results to a Unity Catalog table, one row per check: each " + "data-quality rule plus the schema, row-count and metadata checks (the same " + "cases as --output junit, which still works beside it). The table is created " + "if missing, in a schema you already have: forge never creates the catalog or " + "the schema, and refuses the product's own schema. Signs in like `fluid apply` " + "(DATABRICKS_HOST and a token or an OAuth client) and writes on " + "DATABRICKS_WAREHOUSE_ID." + ), + ) + p.add_argument( + "--results-dashboard", + metavar="WORKSPACE_PATH", + help=( + "With --results-table: keep an AI/BI dashboard over that table at this " + "workspace path, e.g. '/Workspace/Shared/fluid/Data quality' (the last part is " + "its name). It shows each product's pass rate, the failing rules and the trend. " + "Created on first use, found by its path afterwards and updated only when its " + "definition changed (an edit made in the dashboard editor is replaced then); " + "published without embedded credentials, so viewers need SELECT on the table." + ), + ) + p.set_defaults(cmd=COMMAND, func=run) +def _write_results(report, contract, results_table: str, results_dashboard) -> None: + """Append the run to the results table, then keep the dashboard over it.""" + from ._test_results_databricks import parse_results_table, write_results + + write_results(report, contract, results_table) + if results_dashboard: + from ._test_dashboard_databricks import ensure_dashboard + + ensure_dashboard(results_dashboard, parse_results_table(results_table)) + + +def _results_options(args: argparse.Namespace): + """``(table, dashboard path)`` of the results flags, checked before any test runs.""" + table = getattr(args, "results_table", None) + dashboard = getattr(args, "results_dashboard", None) + if dashboard and not table: + raise CLIError( + 2, + "test_results_dashboard_needs_table", + {"results_dashboard": dashboard, "fix": "add --results-table CATALOG.SCHEMA.TABLE"}, + ) + if table and getattr(args, "engine", "native") == "soda": + raise CLIError( + 2, + "test_results_table_needs_native_engine", + { + "results_table": table, + "why": "--results-table records the native engine's per-rule results", + "fix": "drop --engine soda, or drop --results-table", + }, + ) + if table: + from ._test_results_databricks import parse_results_table + + parse_results_table(table) + if dashboard: + from ._test_dashboard_databricks import parse_dashboard_path + + parse_dashboard_path(dashboard) + return table, dashboard + + # ====================================================================== # Entry point # ====================================================================== @@ -237,6 +312,9 @@ def run(args: argparse.Namespace, logger: logging.Logger) -> int: # file isn't there" rather than one slug per command. raise CLIError(1, "contract_file_not_found", {"path": str(contract_path)}) + # A malformed results flag fails before any check runs (or any compute starts). + results_table, results_dashboard = _results_options(args) + # Soda engine dispatch — generates SodaCL from the contract's quality # block and shells out to the user's installed `soda` binary. Skipped # when --engine native (the default). @@ -298,6 +376,11 @@ def run(args: argparse.Namespace, logger: logging.Logger) -> int: if publish_url: _publish_results(report, publish_url, logger) + # --- results table + dashboard (Unity Catalog). After the report is out, + # so a JUnit file exists even when the write fails. --- + if results_table: + _write_results(report, validator.contract or {}, results_table, results_dashboard) + # --- exit code --- if not report.is_valid(): return 1 @@ -647,6 +730,9 @@ def _output_json(report, output_file: Optional[str] = None) -> None: "message": c.message, "expected": c.expected, "actual": c.actual, + "rule_id": getattr(c, "rule_id", None), + "rule_type": getattr(c, "rule_type", None), + "expose_id": getattr(c, "expose_id", None), } for c in getattr(report, "checks", []) ], @@ -681,12 +767,15 @@ def _output_junit(report, output_file: Optional[str] = None) -> None: ts.set("time", f"{report.duration:.3f}") ts.set("timestamp", report.validation_time.isoformat()) + from ._test_cases import rule_checks as _rule_checks + from ._test_cases import structural_categories as _structural_categories + classname = f"fluid.test.{report.contract_id}" total = 0 failures = 0 # ── one testcase per executed data-quality rule ── - rule_checks = [c for c in getattr(report, "checks", []) if c.category == "quality"] + rule_checks = _rule_checks(report) for check in rule_checks: total += 1 tc = ET.SubElement(ts, "testcase") @@ -707,33 +796,7 @@ def _output_junit(report, output_file: Optional[str] = None) -> None: fail.text = "\n".join(body) # ── one testcase per structural check category ── - categories_seen: Dict[str, List] = {} - for issue in report.issues: - categories_seen.setdefault(issue.category, []).append(issue) - - structural_categories = [ - "schema", - "connection", - "binding", - "missing_resource", - "missing_field", - "type_mismatch", - "mode_mismatch", - "extra_field", - "empty_table", - "row_count_below_threshold", - "quality", - "metadata", - "drift", - ] - if rule_checks: - # The per-rule cases above already carry every rule outcome; a - # roll-up ``quality`` case on top would double-count them. - structural_categories.remove("quality") - - extra_categories = [c for c in categories_seen if c not in structural_categories] - if rule_checks and "quality" in extra_categories: - extra_categories.remove("quality") + structural_categories, extra_categories, categories_seen = _structural_categories(report) for cat in structural_categories + extra_categories: total += 1 diff --git a/tests/cli/test_fluid_test_dashboard.py b/tests/cli/test_fluid_test_dashboard.py new file mode 100644 index 00000000..f4cb5081 --- /dev/null +++ b/tests/cli/test_fluid_test_dashboard.py @@ -0,0 +1,371 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``fluid test --results-dashboard``: path parsing, the dashboard definition's structural +invariants, and ensure_dashboard's create/update/unchanged flows against a fake client.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest + +from fluid_build.cli import _test_dashboard_databricks as dash +from fluid_build.cli._common import CLIError + +TABLE = ("main", "dq", "results") +REF = "`main`.`dq`.`results`" +PATH = "/Workspace/Shared/fluid/Data quality" +FAKE_TOKEN = "fake-test-token-0000" # pragma: allowlist secret + + +class NotFound(Exception): + status_code = 404 + + +# ------------------------------------------------------------ path parsing + + +def test_parse_dashboard_path_splits_folder_and_name(): + assert dash.parse_dashboard_path(PATH) == ("/Workspace/Shared/fluid", "Data quality") + + +def test_parse_dashboard_path_strips_lvdash_suffix(): + assert dash.parse_dashboard_path(PATH + ".lvdash.json") == ( + "/Workspace/Shared/fluid", + "Data quality", + ) + + +def test_parse_dashboard_path_accepts_a_name_directly_under_workspace(): + # Pins current behaviour: the leading empty segment counts toward the minimum of 3. + assert dash.parse_dashboard_path("/Workspace/x") == ("/Workspace", "x") + + +@pytest.mark.parametrize( + "bad", + [ + "Workspace/Shared/x", + "/Workspace/../x", + "/Workspace//x", + "/Workspace/Shared/", + "/Workspace/./x", + "/Workspace/Shared/a\x00b", + "/Workspace/Shared/a\nb", + "/x", + "", + None, + ], +) +def test_parse_dashboard_path_rejects(bad): + with pytest.raises(CLIError) as ei: + dash.parse_dashboard_path(bad) + assert ei.value.exit_code == 2 and ei.value.event == "test_results_dashboard_invalid" + + +# ------------------------------------------------------ definition shape + + +@pytest.fixture(scope="module") +def definition(): + return dash.dashboard_definition(TABLE) + + +def widgets(defn): + return [item for page in defn["pages"] for item in page["layout"]] + + +def field_names(widget): + return {f["name"] for q in widget.get("queries", []) for f in q["query"]["fields"]} + + +def encoded_fields(node): + found = [] + if isinstance(node, dict): + if "fieldName" in node: + found.append(node["fieldName"]) + for v in node.values(): + found.extend(encoded_fields(v)) + elif isinstance(node, list): + for v in node: + found.extend(encoded_fields(v)) + return found + + +def test_definition_is_json_serialisable_and_deterministic(definition): + assert json.dumps(definition) == json.dumps(dash.dashboard_definition(TABLE)) + + +def test_every_query_points_at_a_dataset(definition): + names = {d["name"] for d in definition["datasets"]} + used = { + q["query"]["datasetName"] + for w in widgets(definition) + for q in w["widget"].get("queries", []) + } + assert used and used <= names + + +def test_every_encoding_field_is_a_query_field(definition): + for item in widgets(definition): + w = item["widget"] + if "queries" not in w: + continue + assert set(encoded_fields(w["spec"]["encodings"])) <= field_names(w), w["name"] + + +def test_positions_fit_six_column_grid_without_overlap(definition): + boxes = [i["position"] for i in widgets(definition)] + for p in boxes: + assert p["x"] >= 0 and p["x"] + p["width"] <= 6 + + def overlap(a, b): + return ( + a["x"] < b["x"] + b["width"] + and b["x"] < a["x"] + a["width"] + and a["y"] < b["y"] + b["height"] + and b["y"] < a["y"] + a["height"] + ) + + for i, a in enumerate(boxes): + for b in boxes[i + 1 :]: + assert not overlap(a, b), (a, b) + + +def test_spec_versions_by_widget_type(definition): + seen = {} + for item in widgets(definition): + spec = item["widget"].get("spec") + if spec: + seen.setdefault(spec["widgetType"], set()).add(spec["version"]) + assert seen == {"counter": {2}, "bar": {3}, "line": {3}, "table": {1}} + + +def test_datasets_read_the_backticked_table(definition): + for d in definition["datasets"]: + assert REF in "".join(d["queryLines"]), d["name"] + + +def test_structure_filter_on_rule_datasets_but_not_on_failing_list(definition): + sql = {d["name"]: "".join(d["queryLines"]) for d in definition["datasets"]} + for name in ("fluid_dq_overall", "fluid_dq_products", "fluid_dq_trend"): + assert "COALESCE(rule_type, '') <> 'structure'" in sql[name] + assert "structure" not in sql["fluid_dq_failing"] + assert "WHERE NOT passed" in sql["fluid_dq_failing"] + + +def test_definition_names_the_table_in_the_text_box(definition): + box = widgets(definition)[0]["widget"]["multilineTextboxSpec"]["lines"] + assert "main.dq.results" in "".join(box) + + +# ---------------------------------------------------------- ensure_dashboard + + +class FakeApi: + def __init__(self, status=None, dashboard=None, published=None, create=None): + self.status = status if status is not None else NotFound("nope") + self.dashboard = dashboard + self.published = published if published is not None else {"revision_create_time": "t"} + self.create = create if create is not None else {"dashboard_id": "dash-1"} + self.calls = [] + + def do(self, method, path, query=None, body=None, **kw): + self.calls.append((method, path, query, body)) + if (method, path) == ("GET", dash.GET_STATUS_PATH): + result = self.status + elif (method, path) == ("POST", dash.MKDIRS_PATH): + result = {} + elif (method, path) == ("POST", dash.DASHBOARDS_PATH): + result = self.create + elif method == "GET" and path.endswith("/published"): + result = self.published + elif method == "GET" and path.startswith(dash.DASHBOARDS_PATH + "/"): + result = self.dashboard + elif method in ("PATCH", "POST"): + result = {} + else: + raise AssertionError(f"unexpected {method} {path}") + if isinstance(result, Exception): + raise result + return result + + def verbs(self): + return [(c[0], c[1]) for c in self.calls] + + +class FakeClient: + def __init__(self, api, host="https://adb-1.azuredatabricks.net/"): + self.api_client = api + self.config = SimpleNamespace(host=host) + + +def ensure(api, **kw): + return dash.ensure_dashboard( + PATH, TABLE, client_factory=lambda: FakeClient(api), warehouse_id="wh", **kw + ) + + +def existing_status(object_type="DASHBOARD"): + return { + "object_type": object_type, + "path": "/Workspace/Shared/fluid/Data quality.lvdash.json", + "resource_id": "dash-1", + } + + +def live_dashboard(serialized=None, warehouse="wh"): + return { + "create_time": "t", + "dashboard_id": "dash-1", + "display_name": "Data quality", + "etag": "etag-7", + "lifecycle_state": "ACTIVE", + "parent_path": "/Workspace/Shared/fluid", + "path": "/Workspace/Shared/fluid/Data quality.lvdash.json", + "serialized_dashboard": ( + serialized if serialized is not None else json.dumps(dash.dashboard_definition(TABLE)) + ), + "update_time": "t", + "warehouse_id": warehouse, + } + + +def test_missing_dashboard_is_created_and_published(capsys): + api = FakeApi() + result = ensure(api) + assert result["action"] == "created" and result["dashboard_id"] == "dash-1" + assert api.verbs() == [ + ("GET", dash.GET_STATUS_PATH), + ("POST", dash.MKDIRS_PATH), + ("POST", dash.DASHBOARDS_PATH), + ("POST", f"{dash.DASHBOARDS_PATH}/dash-1/published"), + ] + assert api.calls[0][2] == {"path": "/Workspace/Shared/fluid/Data quality.lvdash.json"} + assert api.calls[1][3] == {"path": "/Workspace/Shared/fluid"} + create = api.calls[2][3] + assert create["display_name"] == "Data quality" + assert create["parent_path"] == "/Workspace/Shared/fluid" and create["warehouse_id"] == "wh" + assert json.loads(create["serialized_dashboard"]) == dash.dashboard_definition(TABLE) + assert api.calls[3][3] == {"embed_credentials": False, "warehouse_id": "wh"} + err = capsys.readouterr() + assert err.out == "" and "Dashboard created" in err.err + assert result["url"] == "https://adb-1.azuredatabricks.net/dashboardsv3/dash-1/published" + + +def test_same_definition_and_published_is_left_alone(): + api = FakeApi(status=existing_status(), dashboard=live_dashboard()) + result = ensure(api) + assert result["action"] == "unchanged" + assert [v[0] for v in api.verbs()] == ["GET", "GET", "GET"] + assert not any(v[0] in ("PATCH", "POST") for v in api.verbs()) + + +def test_unchanged_but_never_published_publishes_only(): + api = FakeApi(status=existing_status(), dashboard=live_dashboard(), published=NotFound("x")) + result = ensure(api) + assert result["action"] == "unchanged" + assert api.verbs()[-1] == ("POST", f"{dash.DASHBOARDS_PATH}/dash-1/published") + assert "PATCH" not in [v[0] for v in api.verbs()] + + +def test_different_definition_is_patched_with_etag_then_published(): + api = FakeApi( + status=existing_status(), + dashboard=live_dashboard(serialized='{"datasets": [], "pages": []}'), + ) + result = ensure(api) + assert result["action"] == "updated" + verbs = api.verbs() + assert verbs[-2:] == [ + ("PATCH", f"{dash.DASHBOARDS_PATH}/dash-1"), + ("POST", f"{dash.DASHBOARDS_PATH}/dash-1/published"), + ] + patch = [c for c in api.calls if c[0] == "PATCH"][0][3] + assert patch["etag"] == "etag-7" and patch["warehouse_id"] == "wh" + assert json.loads(patch["serialized_dashboard"]) == dash.dashboard_definition(TABLE) + + +def test_different_warehouse_counts_as_changed(): + api = FakeApi(status=existing_status(), dashboard=live_dashboard(warehouse="other")) + assert ensure(api)["action"] == "updated" + + +def test_unparseable_stored_definition_is_replaced(): + api = FakeApi(status=existing_status(), dashboard=live_dashboard(serialized="not json")) + assert ensure(api)["action"] == "updated" + + +def test_create_race_already_exists_is_treated_as_existing(): + class Api(FakeApi): + def __init__(self): + super().__init__(dashboard=live_dashboard()) + self.lookups = 0 + + def do(self, method, path, query=None, body=None, **kw): + if (method, path) == ("GET", dash.GET_STATUS_PATH): + self.lookups += 1 + self.calls.append((method, path, query, body)) + if self.lookups == 1: + raise NotFound("missing") + return existing_status() + if (method, path) == ("POST", dash.DASHBOARDS_PATH): + self.calls.append((method, path, query, body)) + raise RuntimeError("RESOURCE_ALREADY_EXISTS: dashboard exists") + return super().do(method, path, query=query, body=body) + + api = Api() + result = ensure(api) + assert api.lookups == 2 and result["dashboard_id"] == "dash-1" + assert result["action"] == "unchanged" + + +def test_already_exists_but_refind_fails_surfaces_error(): + api = FakeApi(create=RuntimeError("RESOURCE_ALREADY_EXISTS: x")) + with pytest.raises(CLIError) as ei: + ensure(api) + assert ei.value.event == "test_results_dashboard_failed" + + +def test_notebook_at_the_path_is_refused(): + api = FakeApi(status=existing_status("NOTEBOOK")) + with pytest.raises(CLIError) as ei: + ensure(api) + assert ei.value.event == "test_results_dashboard_path_taken" + assert ei.value.context["found"] == "NOTEBOOK" + assert [v[0] for v in api.verbs()] == ["GET"] + + +def test_other_failure_is_exit_1_with_scrubbed_message(monkeypatch): + monkeypatch.setenv("DATABRICKS_TOKEN", FAKE_TOKEN) + api = FakeApi(create=PermissionError(f"denied for token {FAKE_TOKEN}")) + with pytest.raises(CLIError) as ei: + ensure(api) + assert ei.value.exit_code == 1 and ei.value.event == "test_results_dashboard_failed" + error = ei.value.context["error"] + assert error.startswith("PermissionError:") and FAKE_TOKEN not in error + + +def test_get_status_non_404_failure_is_reported(): + api = FakeApi(status=RuntimeError("503 unavailable")) + with pytest.raises(CLIError) as ei: + ensure(api) + assert ei.value.event == "test_results_dashboard_failed" and "503" in ei.value.context["error"] + + +def test_invalid_path_is_rejected_before_any_call(): + api = FakeApi() + with pytest.raises(CLIError) as ei: + dash.ensure_dashboard( + "relative", TABLE, client_factory=lambda: FakeClient(api), warehouse_id="w" + ) + assert ei.value.exit_code == 2 and api.calls == [] + + +def test_no_host_means_no_url(): + api = FakeApi() + result = dash.ensure_dashboard( + PATH, TABLE, client_factory=lambda: FakeClient(api, host=""), warehouse_id="wh" + ) + assert result["url"] is None diff --git a/tests/cli/test_fluid_test_results_table.py b/tests/cli/test_fluid_test_results_table.py new file mode 100644 index 00000000..83a92279 --- /dev/null +++ b/tests/cli/test_fluid_test_results_table.py @@ -0,0 +1,438 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""``fluid test``: JUnit golden output, the results-table flags and the Unity Catalog +results writer (rows, insert statements, write_results) against a fake client.""" + +from __future__ import annotations + +import argparse +import logging +import xml.etree.ElementTree as ET +from datetime import datetime, timedelta, timezone + +import pytest + +from fluid_build.cli import _test_results_databricks as rt +from fluid_build.cli import test as fluid_test +from fluid_build.cli._common import CLIError +from fluid_build.cli.contract_validation import CheckOutcome, ValidationReport + +# ---------------------------------------------------------------- helpers + + +def make_report(*, version="0.0.0", when=None): + return ValidationReport( + contract_path="c.yaml", + contract_id="prod", + contract_version=version, + validation_time=when or datetime(2026, 11, 12, 10, 0, 0, tzinfo=timezone.utc), + duration=1.2, + ) + + +def rule_check(name, passed, **kw): + expose, _, rid = name.rpartition(".") + base = dict( + name=name, + category="quality", + passed=passed, + severity="error", + message="msg", + expected="1.0", + actual="0.5", + rule_id=rid, + rule_type="completeness", + expose_id=expose, + ) + base.update(kw) + return CheckOutcome(**base) + + +def contract(**extra): + base = { + "version": "2.0.0", + "metadata": {"layer": "Silver"}, + "exposes": [ + { + "id": "e1", + "version": "1.4.0", + "binding": { + "platform": "databricks", + "location": {"database": "main", "schema": "energy", "table": "meters"}, + }, + } + ], + } + base.update(extra) + return base + + +# ------------------------------------------------------------ JUnit golden + + +def test_junit_golden_testcase_names_order_and_failure_counts(tmp_path): + report = make_report() + report.record_check(rule_check("e1.c1", True)) + report.record_check(rule_check("e1.u1", False, rule_type="uniqueness")) + report.add_issue("error", "quality", "u1 failed", "exposes[0].dq.rules.u1") + report.add_issue("error", "missing_field", "column b missing", "fields.b") + report.add_issue("warning", "mode_mismatch", "b is nullable", "fields.b.mode") + report.add_issue("warning", "overlay", "overlay hit another entry", "x") + out = tmp_path / "j.xml" + fluid_test._output_junit(report, str(out)) + suite = ET.parse(out).getroot() + cases = [(c.get("classname"), c.get("name")) for c in suite.findall("testcase")] + assert cases == [ + ("fluid.test.prod.quality", "e1.c1"), + ("fluid.test.prod.quality", "e1.u1"), + *[ + ("fluid.test.prod", n) + for n in ( + "schema", + "connection", + "binding", + "missing_resource", + "missing_field", + "type_mismatch", + "mode_mismatch", + "extra_field", + "empty_table", + "row_count_below_threshold", + "metadata", + "drift", + "overlay", + ) + ], + ] + assert suite.get("tests") == "15" and suite.get("failures") == "2" + failing = {c.get("name") for c in suite.findall("testcase") if c.find("failure") is not None} + assert failing == {"e1.u1", "missing_field"} + warned = {c.get("name") for c in suite.findall("testcase") if c.find("system-out") is not None} + assert warned == {"mode_mismatch", "overlay"} + rule_fail = suite.findall("testcase")[1].find("failure") + assert rule_fail.get("type") == "DataQualityFailure[error]" + + +def test_junit_without_rule_checks_keeps_quality_rollup_case(tmp_path): + report = make_report() + out = tmp_path / "j.xml" + fluid_test._output_junit(report, str(out)) + names = [c.get("name") for c in ET.parse(out).getroot().findall("testcase")] + assert names == list(fluid_test_cases().STRUCTURAL_CATEGORIES) + + +def fluid_test_cases(): + from fluid_build.cli import _test_cases + + return _test_cases + + +# --------------------------------------------------------------- flags + + +def args_for(path, **kw): + ns = dict(contract=str(path), engine="native", results_table=None, results_dashboard=None) + ns.update(kw) + return argparse.Namespace(**ns) + + +@pytest.fixture +def contract_file(tmp_path): + p = tmp_path / "contract.fluid.yaml" + p.write_text("fluidVersion: 0.7.3\nid: x\n", encoding="utf-8") + return p + + +@pytest.fixture +def no_validator(monkeypatch): + def boom(*a, **k): + raise AssertionError("validation must not start") + + monkeypatch.setattr("fluid_build.cli.contract_validation.ContractValidator", boom) + + +LOG = logging.getLogger("t") + + +def test_dashboard_without_table_is_exit_2(contract_file, no_validator): + with pytest.raises(CLIError) as ei: + fluid_test.run(args_for(contract_file, results_dashboard="/Workspace/Shared/a/b"), LOG) + assert ei.value.exit_code == 2 and ei.value.event == "test_results_dashboard_needs_table" + + +def test_results_table_with_soda_engine_is_exit_2(contract_file, no_validator): + with pytest.raises(CLIError) as ei: + fluid_test.run(args_for(contract_file, results_table="c.s.t", engine="soda"), LOG) + assert ei.value.exit_code == 2 and ei.value.event == "test_results_table_needs_native_engine" + + +@pytest.mark.parametrize("bad", ["a.b", "a.b.c.d", "bad-name.s.t", "a..t", "a.b.t`x"]) +def test_malformed_results_table_fails_before_validation(contract_file, no_validator, bad): + with pytest.raises(CLIError) as ei: + fluid_test.run(args_for(contract_file, results_table=bad), LOG) + assert ei.value.exit_code == 2 and ei.value.event == "test_results_table_invalid" + + +def test_malformed_dashboard_path_fails_before_validation(contract_file, no_validator): + with pytest.raises(CLIError) as ei: + fluid_test.run( + args_for(contract_file, results_table="c.s.t", results_dashboard="relative/x"), LOG + ) + assert ei.value.exit_code == 2 + + +# ------------------------------------------------------ parse / refuse + + +def test_parse_results_table_returns_parts(): + assert rt.parse_results_table("main.dq.results") == ("main", "dq", "results") + + +def test_refuse_product_schema_is_case_insensitive(): + with pytest.raises(CLIError) as ei: + rt.refuse_product_schema(("MAIN", "Energy", "results"), contract()) + assert ei.value.exit_code == 2 and ei.value.event == "test_results_table_in_product_schema" + + +def test_refuse_product_schema_allows_other_schema(): + rt.refuse_product_schema(("main", "dq", "results"), contract()) + + +def test_refuse_product_schema_accepts_catalog_key(): + c = contract() + loc = c["exposes"][0]["binding"]["location"] + loc["catalog"] = loc.pop("database") + with pytest.raises(CLIError): + rt.refuse_product_schema(("main", "energy", "r"), c) + + +# ---------------------------------------------------------------- rows + + +def rows_for(report, c=None): + return rt.results_rows(report, c or contract(), run_id="run-1", fluid_version="9.9") + + +def test_rule_rows_then_one_structural_row_per_category(): + report = make_report() + report.record_check(rule_check("e1.c1", True)) + report.add_issue("warning", "overlay", "o", "p") + rows = rows_for(report) + builtin, extra, _ = fluid_test_cases().structural_categories(report) + assert [r["rule_id"] for r in rows] == ["c1", *builtin, *extra] + assert rows[0]["rule_type"] == "completeness" + assert {r["rule_type"] for r in rows[1:]} == {"structure"} + + +def test_rule_row_fields_come_from_check_and_expose(): + report = make_report() + report.record_check(rule_check("e1.c1", False, severity="Warning")) + row = rows_for(report)[0] + assert row["product_id"] == "prod" and row["layer"] == "Silver" + assert row["expose_id"] == "e1" and row["table_name"] == "main.energy.meters" + assert row["severity"] == "warning" and row["passed"] is False + assert (row["observed_value"], row["threshold"]) == ("0.5", "1.0") + assert row["fluid_version"] == "9.9" and row["run_id"] == "run-1" + + +def test_product_version_prefers_expose_version(): + report = make_report(version="0.0.0") + report.record_check(rule_check("e1.c1", True)) + assert rows_for(report)[0]["product_version"] == "1.4.0" + + +def test_product_version_falls_back_to_contract_version_never_report_placeholder(): + c = contract() + del c["exposes"][0]["version"] + report = make_report(version="0.0.0") + report.record_check(rule_check("e1.c1", True)) + assert {r["product_version"] for r in rows_for(report, c)} == {"2.0.0"} + + +def test_product_version_none_when_nothing_declared(): + c = contract() + del c["exposes"][0]["version"], c["version"] + report = make_report(version="0.0.0") + report.record_check(rule_check("e1.c1", True)) + assert rows_for(report, c)[0]["product_version"] is None + + +def test_run_time_is_converted_to_utc(): + tz = timezone(timedelta(hours=5)) + report = make_report(when=datetime(2026, 11, 12, 15, 0, 0, tzinfo=tz)) + row = rows_for(report)[0] + assert row["run_time"].utcoffset() == timedelta(0) and row["run_time"].hour == 10 + + +def test_message_is_truncated_to_2000(): + report = make_report() + report.record_check(rule_check("e1.c1", False, message="x" * 5000)) + msg = rows_for(report)[0]["message"] + assert len(msg) == 2000 and msg.endswith("...") + + +def test_structural_row_for_error_and_clean_category(): + report = make_report() + report.add_issue("error", "missing_field", "b missing", "p", expected="b", actual=None) + rows = {r["rule_id"]: r for r in rows_for(report)} + bad, ok = rows["missing_field"], rows["schema"] + assert bad["passed"] is False and bad["severity"] == "error" and bad["threshold"] == "b" + assert bad["message"] == "b missing" and bad["table_name"] == "main.energy.meters" + assert ok["passed"] is True and ok["message"] == "OK" and ok["severity"] is None + + +def test_structural_warning_does_not_fail_the_row(): + report = make_report() + report.add_issue("warning", "mode_mismatch", "nullable", "p") + row = {r["rule_id"]: r for r in rows_for(report)}["mode_mismatch"] + assert row["passed"] is True and row["severity"] == "warning" + + +# --------------------------------------------------- insert statements + + +def row_dict(i): + return { + "run_id": "r", + "run_time": datetime(2026, 1, 2, 3, 4, 5, 6, tzinfo=timezone.utc), + "product_id": "p", + "rule_id": f"rule{i}", + "passed": i % 2 == 0, + } + + +def col(name): + return [n for n, _, _ in rt.RESULT_COLUMNS].index(name) + + +def test_insert_statements_batches_25_rows(): + stmts = rt.insert_statements("`c`.`s`.`t`", [row_dict(i) for i in range(60)]) + assert [len(p) // len(rt.RESULT_COLUMNS) for _, p in stmts] == [25, 25, 10] + + +def test_insert_statement_markers_casts_and_param_values(): + ((sql, params),) = rt.insert_statements("`c`.`s`.`t`", [row_dict(0), row_dict(1)]) + n = len(rt.RESULT_COLUMNS) + assert sql.startswith("INSERT INTO `c`.`s`.`t` (run_id, run_time,") + assert f"CAST(:r0c{col('run_time')} AS TIMESTAMP)" in sql + assert f"CAST(:r1c{col('passed')} AS BOOLEAN)" in sql + assert f":r0c{col('run_id')}," in sql and "CAST(:r0c0" not in sql + assert len(params) == 2 * n + by_name = {p["name"]: p for p in params} + assert by_name[f"r0c{col('passed')}"]["value"] == "true" + assert by_name[f"r1c{col('passed')}"]["value"] == "false" + assert by_name[f"r0c{col('run_time')}"]["value"] == "2026-01-02T03:04:05.000006Z" + assert "value" not in by_name[f"r0c{col('layer')}"] + + +def test_insert_statement_never_inlines_values(): + row = row_dict(0) + row["message"] = "x'); DROP TABLE t; --" + ((sql, _),) = rt.insert_statements("`c`.`s`.`t`", [row]) + assert "DROP TABLE" not in sql + + +# ---------------------------------------------------------- write_results + + +class FakeApi: + def __init__(self, columns=None, fail_on=None, fail_message=""): + self.calls = [] + self.columns = columns if columns is not None else [n for n, _, _ in rt.RESULT_COLUMNS] + self.fail_on = fail_on + self.fail_message = fail_message + + def do(self, method, path, query=None, body=None, **kw): + self.calls.append((method, path, body)) + if method == "GET": + return {"name": "t", "columns": [{"name": c} for c in self.columns]} + statement = body["statement"] + if self.fail_on and statement.startswith(self.fail_on): + return { + "statement_id": "s", + "status": {"state": "FAILED", "error": {"message": self.fail_message}}, + } + return {"statement_id": "s", "status": {"state": "SUCCEEDED"}, "result": {}} + + def statements(self): + return [c[2]["statement"] for c in self.calls if c[0] == "POST"] + + +class FakeClient: + def __init__(self, api): + self.api_client = api + + +def write(api, table="main.dq.results", c=None, report=None): + report = report or make_report() + if not report.checks: + report.record_check(rule_check("e1.c1", True)) + return rt.write_results( + report, + c or contract(), + table, + client_factory=lambda: FakeClient(api), + warehouse_id="wh", + run_id="run-1", + ) + + +def test_write_results_creates_checks_columns_then_inserts(capsys): + api = FakeApi() + name, count = write(api) + sql = api.statements() + assert name == "main.dq.results" and count == 1 + 12 + assert sql[0].startswith("CREATE TABLE IF NOT EXISTS `main`.`dq`.`results` (") + assert "COMMENT 'One fluid test invocation'" in sql[0] and "USING DELTA" in sql[0] + assert [c[0] for c in api.calls[:3]] == ["POST", "GET", "POST"] + assert api.calls[1][1] == "/api/2.1/unity-catalog/tables/main.dq.results" + assert sql[1].startswith("INSERT INTO `main`.`dq`.`results`") + out = capsys.readouterr() + assert out.out == "" and "13 row(s) appended to main.dq.results" in out.err + + +def test_write_results_mismatched_table_writes_nothing(): + api = FakeApi(columns=["run_id", "other"]) + with pytest.raises(CLIError) as ei: + write(api) + assert ei.value.event == "test_results_table_mismatch" and ei.value.exit_code == 1 + assert "run_time" in ei.value.context["missing_columns"] + assert not any(s.startswith("INSERT") for s in api.statements()) + + +def test_write_results_missing_schema_has_create_schema_fix(): + api = FakeApi( + fail_on="CREATE TABLE", fail_message="[SCHEMA_NOT_FOUND] schema main.dq not found" + ) + with pytest.raises(CLIError) as ei: + write(api) + assert ei.value.exit_code == 1 and ei.value.event == "test_results_write_failed" + assert "CREATE SCHEMA `main`.`dq`" in ei.value.context["fix"] + + +def test_write_results_permission_failure_names_grants(): + api = FakeApi(fail_on="INSERT", fail_message="PERMISSION_DENIED: no") + with pytest.raises(CLIError) as ei: + write(api) + assert "MODIFY" in ei.value.context["fix"] + + +def test_write_results_unknown_failure_has_no_fix(): + api = FakeApi(fail_on="CREATE", fail_message="boom") + with pytest.raises(CLIError) as ei: + write(api) + assert "fix" not in ei.value.context and "boom" in ei.value.context["error"] + + +def test_write_results_refuses_product_schema_before_any_call(): + api = FakeApi() + with pytest.raises(CLIError) as ei: + write(api, table="main.energy.results") + assert ei.value.exit_code == 2 and api.calls == [] + + +def test_write_results_status_goes_to_stderr_only(capsys): + write(FakeApi()) + captured = capsys.readouterr() + assert captured.out == "" and captured.err.startswith("Results:") From c6a6e9fca9165d6ed537ea9762b60a2cebf57548 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:51:45 +0100 Subject: [PATCH 3/6] docs: changelog, AGENTS.md and HONESTLY_TESTED for fluid test results on Databricks --- AGENTS.md | 23 ++++++++++++++++++++++ CHANGELOG.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++ HONESTLY_TESTED.md | 2 ++ 3 files changed, 73 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index d2102957..00eb61eb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -768,6 +768,29 @@ onprem = "my_pkg.iac:OnPremIacPlugin" - **Tests.** `tests/iac/test_iac_entrypoint_lazy_plugin.py` builds a real throwaway `.dist-info` and is the pattern to copy. +## `fluid test` on Databricks: results table and dashboard (2026-11-12) + +- **Live checks.** `providers/databricks_validation.py` reads columns from the Unity + Catalog tables API and runs the row count and the rules on `DATABRICKS_WAREHOUSE_ID` + through `providers/databricks/statements.py` (Statement Execution API, `:name` + parameters, credential-scrubbed errors). Contract types go through + `engines/dbt/_types.sql_type(..., "databricks")` before they are compared. +- **One translation of the rules.** `providers/quality_engine.py` is the native engine; + the SodaCL exporter and the dbt test mapping mirror it. It quotes identifiers per + dialect (backticks on Databricks and BigQuery, where `"x"` is a string) and string + literals through `_sql_safety.quote_string_literal_for`. A `valid_values` rule's values + come from the rule, else from its column's `enum` / `regex` validationRule, else from + its `labels.references` (resolved by the provider's `resolve_reference`; Databricks finds + the table by its `fluid_product_id` tag). `schema` rules are decided in + `cli/contract_validation.py::_decide_schema_rules` from the live column comparison. +- **Same cases everywhere.** `cli/_test_cases.py` lists the run's test cases (each rule, then + each structural category). `--output junit` and `--results-table` + (`cli/_test_results_databricks.py`) both read it, so Jenkins and the table agree. +- **Ownership.** The results table is created if missing; the catalog and schema never are, + a product's own schema is refused, and a table without the result columns is never written. + The dashboard (`cli/_test_dashboard_databricks.py`) is found by its workspace path, created, + PATCHed only when its definition differs, and published without embedded credentials. + ## Links - **Documentation**: [https://agenticstiger.github.io/forge_docs/](https://agenticstiger.github.io/forge_docs/) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b3820c2..a9352ec0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 script needs a SQL warehouse that runs SQL scripting. A dbt build that `--mode amend-and-build` runs locally is followed by the same re-declaration. +- **`fluid test` checks a Databricks product live, and can keep its results in Unity Catalog.** + `fluid test` used to answer "Live validation against Unity Catalog is not available" for + `platform: databricks`. It now reads the table's columns from the Unity Catalog tables API + (no compute), and runs the row count and every `dq.rules[]` entry as SQL on + `DATABRICKS_WAREHOUSE_ID`, signing in as `fluid apply` and `fluid verify` do (the + `[databricks]` extra). Bronze products count like dbt-built ones. Two new flags: + - `--results-table CATALOG.SCHEMA.TABLE` appends one row per check of the run (each rule, then + each structural check, the same cases `--output junit` reports, which still works beside + it): run id and time, product id, version and layer, expose and table, rule id, type and + severity, passed, observed value, threshold, message, forge version. The table is created if + missing in a schema the operator already has; forge never creates the catalog or the schema, + refuses the product's own schema, and writes nothing into an existing table without these + columns. Every value is a bound parameter. + - `--results-dashboard WORKSPACE_PATH` keeps an AI/BI (Lakeview) dashboard over that table: + the pass rate of the latest runs, the pass rate per product (coloured by layer), the failing + checks and the daily trend. It is found by its workspace path (no state file, so ephemeral CI + agents never duplicate it), created on first use, updated only when its definition changed, + and published without embedded credentials. + The row shape follows Databricks Labs DQX's summary-metrics table and Elementary's test + results; the dashboard lookup follows databricks-solutions/ai-dev-kit. No code is taken from + either. + ### Changed - **`fluid apply --mode amend-and-build` runs a Databricks dbt build now.** A dbt build with @@ -78,9 +100,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 path exits 1 where it used to write a template instead. A `fluid ai setup` choice and a single keyring key still come first, and `GEMINI_API_KEY` with `GOOGLE_API_KEY` is one provider. Pick one with `--llm-provider` or `FLUID_LLM_PROVIDER`. +- **The native quality engine runs the rules a contract already declares in full.** A + `valid_values` rule that names no values takes them from its column: its `enum` + validationRule (`"A | B"` or `"A,B"`), its `regex` validationRule, its `range` + validationRule (`">= 0"`, `">= 0 and <= 100"`, `"between 0 and 100"`), or its + `labels.references: .` (on Databricks, the referenced product's table is + found by the `fluid_product_id` tag `fluid apply` puts on it, the tested table's catalog + first, and every non-null value must be in its column). A `completeness`, `accuracy` or + `valid_values` rule may list several comma-separated columns, each of which must hold (a + `valid_values` column against its own declaration); a `uniqueness` or `freshness` rule over + several columns is reported as not run, never guessed. A `schema` rule passes when the live + table has every contract column with its type, and fails, at the rule's severity, when it + does not or could not be read. Every rule's result, passed or not, now carries what was + measured (`--output json` adds `rule_id`, `rule_type` and `expose_id` to each check). On + the tower-management industry pack `fluid test` ran 38 of its 168 rules before; it now runs + every rule of every deployed product. ### Fixed +- **Quality checks measured a constant on BigQuery (and would have on Databricks).** The + completeness, uniqueness, accuracy and valid_values checks quoted their column the ANSI way, + `"col"`, which BigQuery and Spark SQL read as a string literal, and cast ratios to + `DOUBLE PRECISION`, which neither has. Columns are now backtick-quoted and ratios cast to + `FLOAT64` / `DOUBLE` on those two dialects; ANSI, DuckDB and Snowflake SQL is unchanged. +- **A valid value containing a quote never matched on BigQuery or Databricks.** String literals + doubled the quote (`'it''s'`), which Databricks reads as the adjacent literals `'it'` and `'s'` + (checked on a SQL warehouse: the value becomes `its`) and BigQuery rejects. The new + `_sql_safety.quote_string_literal_for(dialect, value)` escapes it as `\'` there, after + doubling every backslash, so no value can end the literal early. + - **`fluid ai models --provider` works, and takes the names `fluid forge --llm-provider` takes.** The flag stored its value where the global infrastructure `--provider` keeps its own, so the global check read `gemini` as an infrastructure provider: every diff --git a/HONESTLY_TESTED.md b/HONESTLY_TESTED.md index 55ce77ba..f9b04d81 100644 --- a/HONESTLY_TESTED.md +++ b/HONESTLY_TESTED.md @@ -98,6 +98,8 @@ Markers declared in `pyproject.toml`. | Stage 1 real tofu, stand-in workspace: Genie space | `tests/iac/test_iac_databricks_genie_fake_e2e.py` | ✅ 3. The real `fluid` CLI and `tofu`: validate, dry-run, apply (the space after the tables, shared with the readers), a second apply that changes nothing, the block removed (trashed); a Genie failure fails the apply after the tables with its fix and the next apply converges; an invalid block refused before `tofu` | The stand-in is modelled on the measured API, not the service itself | | Manual live, Free Edition, Genie space, 2026-11-12 | the real `fluid` CLI, no test file: branch `feat/databricks-genie-space`, databricks-sdk 0.162.0, a copy of industry-models tower-management `dp019_site_360` in the scratch catalog `fluid_iactest_genie` (no dbt build, no MENA sovereignty block: the workspace is in us-west-2), 6 synthetic rows | ✅ dry-run (would create, would share), apply `+73` then the space created and shared `CAN_RUN` with two persona service principals, a second apply `+0 ~0 -0` and "no change", the question "Which on-air sites in Amman are colocation-ready with hybrid power and no standing risk?" answered through the Genie Conversation API with the 2 expected sites (its SQL took the latest `snapshot_date` and the inverse of the `at_risk_site_count` filter from the instruction), a metric view created by hand picked up ("updated: metric views", then "no change"), the block removed (trashed). Two defects found and fixed in the run: the metric-view read failed on a schema not created yet, and the service keeps a metric view among the tables. A re-check after the security review (the plugin's own hook over `samples.nyctaxi.trips`, nothing created in Unity Catalog) found the list shows a new space only ~9 s later (a run straight after made a second space) and that a taken title is suffixed on create and 409 on PATCH; fixed, then create, an immediate second run ("no change", same space), and the block removed (trashed, via the folder check) passed live. Swept: the space trashed, `DROP CATALOG fluid_iactest_genie CASCADE`, 0 left | A reader without `SELECT` asking (no persona credential here; the access model is Genie's documented one); a service principal as the deploying identity (its folder is `/Users/`, the one Databricks Asset Bundles build from `current_user.userName`; a create that lands anywhere else is trashed and stops the apply); a space over several products | | Stage 2 emulator (nightly, non-gating) | `tests/iac/test_iac_databricks_minilake.py` (job `nightly-databricks-minilake`) | ⚠️ 1 passed, 4 strict xfails (manual dispatch, run 42267640334, 2026-11-06, minilake v1.9.0). Schema, table and grant creates work, the replace gate holds, adding a column plans no replace, and a grant revoke is classified not gated. The four tests about those resources apply the module without its tag assignments (the `untagged` fixture). An xfail is satisfied only by its own gap's failure (`MinilakeGap`), and a pass fails the run, so an image that closes a gap turns the job red until the mark comes off. Earlier: 1/4 from the first run (2026-10-30) to the 2026-11-04 nightly, then 0/4 after #831, every apply stopping at the first tag. | A second plan is not empty and no in-place table update applies: minilake drops a CREATE TABLE's `TBLPROPERTIES` and `COMMENT`, and DuckDB cannot parse the `ALTER TABLE ... SET TBLPROPERTIES` the provider sends on every update of a table that sets properties. No tag is applied: minilake has no entity-tag-assignments API (it answers 501). v1.9.3 (2026-11-05) has both gaps, so the pin stays; an upstream issue for each is drafted, not filed. Jobs and workspace files are not in this flow yet. | +| Stage 1 unit: `fluid test` on Databricks, results table, dashboard | `tests/providers/test_databricks_validation.py`, `tests/providers/test_databricks_statements.py`, `tests/providers/test_quality_engine_column_rules.py`, `tests/cli/test_contract_validation_schema_rules.py`, `tests/cli/test_fluid_test_results_table.py`, `tests/cli/test_fluid_test_dashboard.py` | ✅ Against a stubbed `api_client.do`: the Unity Catalog columns and the contract's types through the emitter's type table (`VARCHAR` is `string`), a missing table (404 → `missing_resource`, and every rule reported as not run with no statement), the row count, the warehouse check; the statements over `databricks_enforce.SqlRunner` with `:name` parameters, polling, timeout cancel, chunks, type conversion and scrubbed errors; every rule kind in the Databricks dialect (backticked identifiers, `\'` literals, `timestampdiff` freshness, `RLIKE`), comma-listed columns, values from the column's enum, regex or range rule, references resolved by the `fluid_product_id` tag (same catalog first, ambiguity refused, cached); `schema` rules from the column comparison; the result rows (one per JUnit case, `rule_type` `structure` for the structural ones, expose version, UTC run time), the batched parameterised `INSERT`, the refusals (bad name, a product's own schema, a table without the columns), the fix-its; the dashboard definition's structure and the create, unchanged, re-publish, update-with-etag and create-race paths. | What Databricks accepts: the manual live row below. | +| Manual live, Free Edition (founder-approved), 2026-11-12: `fluid test` results | the real `fluid` CLI, no test file: `fluid test --env databricks --output junit --results-table fluid_iactest_dqr.dq.fluid_test_results --results-dashboard '/Workspace/Shared/fluid_iactest_dqr/Data quality'` on the 19 tower-management products in `northwind_towers` (read only: `SELECT` and metadata reads), contracts derived from industry-models main | ✅ 19 runs, 17-34 s each, 18 exit 0; `dp019_site_360` exits 1 because its table is not deployed in the demo workspace (`missing_resource`; its rules are reported as not run, and no statement is sent). 168 rule cases: the 155 on deployed tables all ran (146 pass, 9 freshness rules fail at `warn`: the bronze data is hours old against a 15-minute window), among them 57 enum, 10 regex and 24 reference rules (3 of them bronze references outside `consumes[]`, resolved by the `fluid_product_id` tag), one range rule over five columns (industry-models#4's form), 19 `schema` and 19 comma-listed completeness rules. 396 rows appended, one per JUnit case; the dashboard was created by the first run, found unchanged by every later one, and updated (PATCH, re-publish) once when its definition changed; in a browser it shows the counters (88.1% of rules passing, 19 products, 20 failing rules at that point), the per-product bars by layer, the trend and the failing-checks table. Afterwards the catalog `fluid_iactest_dqr` was dropped CASCADE and the dashboard deleted. | Lakeview validates widget specs only when the dashboard renders, so a widget shape Databricks changes would show as "Invalid widget definition" without failing the command (the table widget did, until it carried every column key). | | Stage 3 live | — | ❌ not automated: manual maintainer runs only, never CI. Two have run: the manual rows above (2026-11-01 and 2026-11-03). | Everything the rows above leave to the live tier. | ## What's NOT tested (deliberately deferred) From 0bc6413c0f1245cc77fe3a7d8ea45fbc41e9f19e Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:53:57 +0100 Subject: [PATCH 4/6] chore(secrets): refresh .secrets.baseline line numbers --- .secrets.baseline | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index b7a2d390..52b748b2 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -404,14 +404,14 @@ "filename": "fluid_build/providers/_sql_safety.py", "hashed_secret": "fb2620e2ed7e530fcb1ea0a2b2456751057a5178", "is_verified": false, - "line_number": 237 + "line_number": 259 }, { "type": "Secret Keyword", "filename": "fluid_build/providers/_sql_safety.py", "hashed_secret": "6d613a1ee01eec4c0f8ca66df0db71dca0c6e1cf", "is_verified": false, - "line_number": 238 + "line_number": 260 } ], "fluid_build/providers/datamesh_manager/README.md": [ @@ -1710,7 +1710,7 @@ "filename": "tests/test_contract_validation_ext2.py", "hashed_secret": "1a91d62f7ca67399625a4368a6ab5d4a3baa6073", "is_verified": false, - "line_number": 577 + "line_number": 592 } ], "tests/test_credential_adapters.py": [ @@ -2317,5 +2317,5 @@ } ] }, - "generated_at": "2026-11-12T21:55:04Z" + "generated_at": "2026-11-12T22:46:31Z" } From 2b4eb5bb35101a7b379048143a3ac0e1c0899aa8 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:32:44 +0100 Subject: [PATCH 5/6] fix(test): review round for the Databricks results table and live checks From the branch's security, inspection and defect-scan reviews: - A schema rule on BigQuery no longer passes when the table was never read: the BigQuery check now says whether it compared, as the generic one does. - Two rules sharing an id: each outcome pairs with its own result, so one rule's pass cannot mask the other's failure. - A quality error no rule case carries (an expose whose rules could not run) keeps its JUnit and results-table case. - valid_values checks every declaration its column makes (enum, regex and range in one statement, and a reference besides them), not only the first. - An accuracy rule with no threshold is reported as not run; it passed whenever the column held a value. - Databricks schemas and row counts are always read live, never from the hour-long validation cache. - A run is appended in one INSERT, its rows bound as one JSON parameter, so a failure never leaves half a run that the dashboard would take as the latest. - The dashboard's latest run is the latest one that tested rules, and a draft a failed publish left behind is published on the next run. - --results-table refuses --no-data, and refuses the product's own schema before any check runs; a missing databricks-sdk names the extra to install. - A warehouse that stops answering stops the remaining statements instead of each waiting out its timeout; a failed table lookup is reported once. - The ambiguous-reference message no longer names other tables: it is written to the shared results table. - --engine soda renders the same column declarations as the native engine (valid values, valid regex, valid min / max, one check per listed column); a reference or a strict bound stays unmapped. --- AGENTS.md | 18 ++- CHANGELOG.md | 17 +- fluid_build/_error_catalog.py | 7 + .../references/error-codes.md | 1 + fluid_build/cli/_test_cases.py | 20 ++- fluid_build/cli/_test_dashboard_databricks.py | 53 +++++-- fluid_build/cli/_test_results_databricks.py | 51 +++--- fluid_build/cli/contract_validation.py | 41 +++-- fluid_build/cli/test.py | 28 +++- fluid_build/exporters/sodacl.py | 76 +++++++-- .../providers/databricks/statements.py | 17 +- .../providers/databricks_validation.py | 16 +- fluid_build/providers/quality_engine.py | 145 +++++++++++++----- .../test_contract_validation_schema_rules.py | 125 +++++++++++++++ tests/cli/test_fluid_test_dashboard.py | 56 ++++++- tests/cli/test_fluid_test_results_table.py | 134 +++++++++++++--- tests/exporters/__init__.py | 9 ++ tests/exporters/test_sodacl_column_values.py | 90 +++++++++++ tests/providers/test_databricks_statements.py | 37 +++++ tests/providers/test_databricks_validation.py | 24 +++ .../test_quality_engine_column_rules.py | 69 ++++++++- 21 files changed, 882 insertions(+), 152 deletions(-) create mode 100644 tests/exporters/__init__.py create mode 100644 tests/exporters/test_sodacl_column_values.py diff --git a/AGENTS.md b/AGENTS.md index 00eb61eb..7d8d0ea0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -775,19 +775,23 @@ onprem = "my_pkg.iac:OnPremIacPlugin" through `providers/databricks/statements.py` (Statement Execution API, `:name` parameters, credential-scrubbed errors). Contract types go through `engines/dbt/_types.sql_type(..., "databricks")` before they are compared. -- **One translation of the rules.** `providers/quality_engine.py` is the native engine; - the SodaCL exporter and the dbt test mapping mirror it. It quotes identifiers per - dialect (backticks on Databricks and BigQuery, where `"x"` is a string) and string - literals through `_sql_safety.quote_string_literal_for`. A `valid_values` rule's values - come from the rule, else from its column's `enum` / `regex` validationRule, else from - its `labels.references` (resolved by the provider's `resolve_reference`; Databricks finds - the table by its `fluid_product_id` tag). `schema` rules are decided in +- **One reading of the rules.** `providers/quality_engine.py` is the native engine, and + `quality_engine.value_domain` is what a `valid_values` rule allows: the rule's own list, + else every declaration its column makes (`enum`, `regex`, `range` validationRules, and + `labels.references`, resolved by the provider's `resolve_reference`; Databricks finds the + table by its `fluid_product_id` tag). The SodaCL exporter renders the same reading (a + reference or a strict bound stays unmapped); the dbt test mapping + (`engines/dbt/_test_mapping.py`) still has its own. Identifiers are quoted per dialect + (backticks on Databricks and BigQuery, where `"x"` is a string), string literals through + `_sql_safety.quote_string_literal_for`. `schema` rules are decided in `cli/contract_validation.py::_decide_schema_rules` from the live column comparison. - **Same cases everywhere.** `cli/_test_cases.py` lists the run's test cases (each rule, then each structural category). `--output junit` and `--results-table` (`cli/_test_results_databricks.py`) both read it, so Jenkins and the table agree. - **Ownership.** The results table is created if missing; the catalog and schema never are, a product's own schema is refused, and a table without the result columns is never written. + A run is appended in one statement (its rows bound as one JSON parameter), so it is never + half-written. Databricks schemas are never served from the validation cache. The dashboard (`cli/_test_dashboard_databricks.py`) is found by its workspace path, created, PATCHed only when its definition differs, and published without embedded credentials. diff --git a/CHANGELOG.md b/CHANGELOG.md index a9352ec0..ca202674 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,8 +62,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 it): run id and time, product id, version and layer, expose and table, rule id, type and severity, passed, observed value, threshold, message, forge version. The table is created if missing in a schema the operator already has; forge never creates the catalog or the schema, - refuses the product's own schema, and writes nothing into an existing table without these - columns. Every value is a bound parameter. + refuses the product's own schema (before any check runs), and writes nothing into an + existing table without these columns. A run is appended in one statement, so it is never + half-written, and every value is a bound parameter. It refuses `--no-data`, which runs no + live check to record; Databricks schemas are read live, never from the validation cache. - `--results-dashboard WORKSPACE_PATH` keeps an AI/BI (Lakeview) dashboard over that table: the pass rate of the latest runs, the pass rate per product (coloured by layer), the failing checks and the daily trend. It is found by its workspace path (no state file, so ephemeral CI @@ -111,10 +113,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `valid_values` column against its own declaration); a `uniqueness` or `freshness` rule over several columns is reported as not run, never guessed. A `schema` rule passes when the live table has every contract column with its type, and fails, at the rule's severity, when it - does not or could not be read. Every rule's result, passed or not, now carries what was - measured (`--output json` adds `rule_id`, `rule_type` and `expose_id` to each check). On - the tower-management industry pack `fluid test` ran 38 of its 168 rules before; it now runs - every rule of every deployed product. + does not or could not be read. An `accuracy` rule with no threshold is reported as not run + (it used to pass whenever the column held a value). Every rule's result, passed or not, now + carries what was measured (`--output json` adds `rule_id`, `rule_type` and `expose_id` to + each check). `--engine soda` renders the same column declarations (`valid values`, + `valid regex`, `valid min` / `valid max`, one check per listed column); a reference or a + strict bound stays unmapped there. On the tower-management industry pack `fluid test` ran + 38 of its 168 rules before; it now runs every rule of every deployed product. ### Fixed diff --git a/fluid_build/_error_catalog.py b/fluid_build/_error_catalog.py index 0c2d340c..deefc5a5 100644 --- a/fluid_build/_error_catalog.py +++ b/fluid_build/_error_catalog.py @@ -684,6 +684,13 @@ def slug_for(event: str) -> str: ], None, ), + "test_results_table_needs_live_checks": ( + [ + "--results-table records the live checks of a run, and --no-data runs none", + "Drop --no-data to test and record, or drop --results-table", + ], + None, + ), "test_results_dashboard_invalid": ( [ "Give a workspace path whose last part is the name, e.g. " diff --git a/fluid_build/agent_skills/skills/troubleshooting-fluid-errors/references/error-codes.md b/fluid_build/agent_skills/skills/troubleshooting-fluid-errors/references/error-codes.md index 6a08fd48..219bf921 100644 --- a/fluid_build/agent_skills/skills/troubleshooting-fluid-errors/references/error-codes.md +++ b/fluid_build/agent_skills/skills/troubleshooting-fluid-errors/references/error-codes.md @@ -97,6 +97,7 @@ Docs pages are relative to https://agenticstiger.github.io/forge_docs/ | `ERR_TEST_RESULTS_WRITE_FAILED` | Check DATABRICKS_HOST, the token or OAuth client, and DATABRICKS_WAREHOUSE_ID. The schema must exist and this identity needs USE CATALOG, USE SCHEMA, CREATE TABLE and MODIFY on it. The test report above is unaffected: rerun fluid test once the write can succeed. | advanced/production-troubleshooting.html | | `ERR_TEST_RESULTS_DASHBOARD_NEEDS_TABLE` | --results-dashboard draws the --results-table table: pass both. | advanced/production-troubleshooting.html | | `ERR_TEST_RESULTS_TABLE_NEEDS_NATIVE_ENGINE` | --results-table records the native engine's per-rule results; drop --engine soda. Or keep --engine soda and its JUnit or JSON output, without --results-table. | advanced/production-troubleshooting.html | +| `ERR_TEST_RESULTS_TABLE_NEEDS_LIVE_CHECKS` | --results-table records the live checks of a run, and --no-data runs none. Drop --no-data to test and record, or drop --results-table. | advanced/production-troubleshooting.html | | `ERR_TEST_RESULTS_DASHBOARD_INVALID` | Give a workspace path whose last part is the name, e.g. --results-dashboard '/Workspace/Shared/fluid/Data quality'. The path must start with / and have no empty, . or .. parts. | advanced/production-troubleshooting.html | | `ERR_TEST_RESULTS_DASHBOARD_PATH_TAKEN` | Something other than a dashboard already sits at that workspace path. Choose another --results-dashboard path, or move that object away. | advanced/production-troubleshooting.html | | `ERR_TEST_RESULTS_DASHBOARD_FAILED` | The results were written; only the dashboard step failed. This identity needs CAN MANAGE on the dashboard (or write access to its folder) and CAN USE on DATABRICKS_WAREHOUSE_ID. | advanced/production-troubleshooting.html | diff --git a/fluid_build/cli/_test_cases.py b/fluid_build/cli/_test_cases.py index 974fa9a6..f1f78879 100644 --- a/fluid_build/cli/_test_cases.py +++ b/fluid_build/cli/_test_cases.py @@ -11,6 +11,7 @@ from __future__ import annotations +import re from typing import Dict, List, Tuple #: The structural check categories, in report order. ``quality`` stands for @@ -33,6 +34,10 @@ ) +#: The path of an issue raised for one rule: ``exposes[0].contract.dq.rules.``. +_RULE_PATH = re.compile(r"\.rules\.[^.]+$") + + def rule_checks(report) -> List: """The data-quality rules that ran, one ``CheckOutcome`` each.""" return [c for c in getattr(report, "checks", []) if c.category == "quality"] @@ -52,11 +57,18 @@ def structural_categories(report) -> Tuple[List[str], List[str], Dict[str, List] builtin = list(STRUCTURAL_CATEGORIES) if rule_checks(report): # The per-rule cases already carry every rule outcome; a roll-up - # ``quality`` case on top would double-count them. - builtin.remove("quality") + # ``quality`` case on top would double-count them. A quality issue no + # rule case carries (an expose whose rules could not run at all) still + # needs a case, so the roll-up stays, with only those issues. + unattributed = [ + i for i in by_category.get("quality", []) if not _RULE_PATH.search(i.path or "") + ] + if unattributed: + by_category["quality"] = unattributed + else: + builtin.remove("quality") + by_category.pop("quality", None) extra = [c for c in by_category if c not in builtin] - if rule_checks(report) and "quality" in extra: - extra.remove("quality") return builtin, extra, by_category diff --git a/fluid_build/cli/_test_dashboard_databricks.py b/fluid_build/cli/_test_dashboard_databricks.py index 6968ab19..d3e61ff3 100644 --- a/fluid_build/cli/_test_dashboard_databricks.py +++ b/fluid_build/cli/_test_dashboard_databricks.py @@ -109,9 +109,20 @@ def _table_column(name: str, kind: str, title: str, order: int) -> Dict[str, Any def dashboard_definition(table: Tuple[str, str, str]) -> Dict[str, Any]: """The ``serialized_dashboard`` for a results table (validated name parts).""" ref = ".".join(f"`{p}`" for p in table) + # Each product's latest run is the latest one that tested rules (a run that + # tested none, say a product that declares none, counts only when no other + # did), so the rule widgets always show a product's last real result. latest = ( - f"WITH latest AS (\n SELECT * FROM {ref}\n" - " QUALIFY run_time = MAX(run_time) OVER (PARTITION BY product_id)\n)\n" + "WITH last_run AS (\n" + " SELECT product_id, COALESCE(\n" + f" MAX(CASE WHEN {_RULES_ONLY} THEN run_time END), MAX(run_time)\n" + " ) AS run_time\n" + f" FROM {ref} GROUP BY product_id\n" + "),\n" + "latest AS (\n" + f" SELECT t.* FROM {ref} AS t JOIN last_run AS l\n" + " ON t.product_id = l.product_id AND t.run_time = l.run_time\n" + ")\n" ) pass_pct = "ROUND(100 * AVG(CASE WHEN passed THEN 1.0 ELSE 0.0 END), 1)" datasets = [ @@ -355,6 +366,21 @@ def _is_not_found(exc: BaseException) -> bool: ) +def _later(first: Any, second: Any) -> bool: + """Whether timestamp ``first`` is after ``second`` (``False`` when either is unreadable).""" + from datetime import datetime + + try: + a = datetime.fromisoformat(str(first).replace("Z", "+00:00")) + b = datetime.fromisoformat(str(second).replace("Z", "+00:00")) + except (TypeError, ValueError): + return False + try: + return a > b + except TypeError: # one naive, one aware + return False + + def _same(theirs: Any, ours: Mapping[str, Any]) -> bool: """Whether a stored ``serialized_dashboard`` holds our datasets and pages.""" try: @@ -463,12 +489,16 @@ def ensure_dashboard( published_path = f"{DASHBOARDS_PATH}/{quote(dashboard_id, safe='')}/published" publish = action != "unchanged" if not publish: + # Unchanged here, but a run whose PATCH landed and whose publish failed + # left the published revision behind the draft: publish it then. try: - api.do("GET", published_path) + published = api.do("GET", published_path) except Exception as exc: # noqa: BLE001 if not _is_not_found(exc): raise publish = True + else: + publish = _later(current.get("update_time"), published.get("revision_create_time")) if publish: api.do( "POST", published_path, body={"embed_credentials": False, "warehouse_id": warehouse} @@ -476,14 +506,15 @@ def ensure_dashboard( except CLIError: raise except Exception as exc: # noqa: BLE001 - every failure becomes one safe message - raise CLIError( - 1, - "test_results_dashboard_failed", - { - "results_dashboard": f"{folder}/{name}", - "error": f"{type(exc).__name__}: {scrub(exc)}", - }, - ) from None + context = { + "results_dashboard": f"{folder}/{name}", + "error": f"{type(exc).__name__}: {scrub(exc)}", + } + if isinstance(exc, ImportError): + from fluid_build.providers.databricks.statements import DATABRICKS_EXTRA_INSTALL + + context["fix"] = f"Install the Databricks extra: {DATABRICKS_EXTRA_INSTALL}" + raise CLIError(1, "test_results_dashboard_failed", context) from None host = str(getattr(getattr(client, "config", None), "host", "") or "").rstrip("/") url = f"{host}/dashboardsv3/{dashboard_id}/published" if host else None _status(f"Dashboard {action}: {folder}/{name}" + (f" ({url})" if url else "")) diff --git a/fluid_build/cli/_test_results_databricks.py b/fluid_build/cli/_test_results_databricks.py index dfee2457..db5e42f0 100644 --- a/fluid_build/cli/_test_results_databricks.py +++ b/fluid_build/cli/_test_results_databricks.py @@ -26,6 +26,7 @@ from __future__ import annotations +import json import sys import uuid from datetime import datetime, timezone @@ -61,7 +62,6 @@ _CASTS = {"run_time": "TIMESTAMP", "passed": "BOOLEAN"} _TABLE_COMMENT = "fluid test results: one row per check per run (data-product-forge)" -_BATCH_ROWS = 25 _MAX_TEXT = 2000 TABLES_PATH = "/api/2.1/unity-catalog/tables" @@ -242,34 +242,39 @@ def _value(name: str, value: Any) -> Any: return value -def insert_statements(ref: str, rows: Sequence[Mapping[str, Any]]) -> List[Tuple[str, List]]: - """``(statement, parameters)`` for ``rows``, at most ``_BATCH_ROWS`` rows each.""" +def insert_statement(ref: str, rows: Sequence[Mapping[str, Any]]) -> Tuple[str, List]: + """One ``INSERT`` for every row of the run, and its one parameter. + + The rows travel as a single JSON array bound to ``:rows`` and are unpacked in + SQL (``from_json`` + ``explode``), so the run is appended in one statement: + all of it or none of it, whatever its size. A run split over several + statements could be left half-written, and the dashboard would take that + half for the latest run. + """ from fluid_build.providers.databricks.statements import param names = [name for name, _, _ in RESULT_COLUMNS] - out: List[Tuple[str, List]] = [] - for start in range(0, len(rows), _BATCH_ROWS): - batch = rows[start : start + _BATCH_ROWS] - tuples: List[str] = [] - params: List[Dict[str, Any]] = [] - for i, row in enumerate(batch): - cells = [] - for j, name in enumerate(names): - marker = f"r{i}c{j}" - cells.append( - f"CAST(:{marker} AS {_CASTS[name]})" if name in _CASTS else f":{marker}" - ) - params.append(param(marker, _value(name, row.get(name)))) - tuples.append("(" + ", ".join(cells) + ")") - statement = f"INSERT INTO {ref} ({', '.join(names)}) VALUES\n" + ",\n".join(tuples) - out.append((statement, params)) - return out + struct = ", ".join(f"{name}: STRING" for name in names) + select = ", ".join( + f"CAST(r.{name} AS {_CASTS[name]})" if name in _CASTS else f"r.{name}" for name in names + ) + statement = ( + f"INSERT INTO {ref} ({', '.join(names)})\n" + f"SELECT {select}\n" + f"FROM (SELECT explode(from_json(:rows, 'ARRAY>')) AS r)" + ) + payload = [{name: _value(name, row.get(name)) for name in names} for row in rows] + return statement, [param("rows", json.dumps(payload))] def _fix_for(error: str, table: Tuple[str, str, str]) -> Optional[str]: """The one action that fixes a failed write, when the error says which.""" catalog, schema, _ = table upper = error.upper() + if "NO MODULE NAMED 'DATABRICKS" in upper: + from fluid_build.providers.databricks.statements import DATABRICKS_EXTRA_INSTALL + + return f"Install the Databricks extra: {DATABRICKS_EXTRA_INSTALL}" if "SCHEMA_NOT_FOUND" in upper or "NO_SUCH_SCHEMA" in upper: return f"Create the schema first (forge never creates it): CREATE SCHEMA `{catalog}`.`{schema}`" if "CATALOG_NOT_FOUND" in upper or "NO_SUCH_CATALOG" in upper: @@ -337,8 +342,8 @@ def write_results( runner = StatementRunner(client, warehouse_id or warehouse_id_from_env()) runner.run(_create_table_sql(ref)) _check_columns(client, full_name) - for statement, params in insert_statements(ref, rows): - runner.run(statement, params) + statement, params = insert_statement(ref, rows) + runner.run(statement, params) except CLIError: raise except Exception as exc: # noqa: BLE001 - every failure becomes one safe message @@ -355,7 +360,7 @@ def write_results( __all__ = [ "RESULT_COLUMNS", "STRUCTURE_RULE_TYPE", - "insert_statements", + "insert_statement", "parse_results_table", "refuse_product_schema", "results_rows", diff --git a/fluid_build/cli/contract_validation.py b/fluid_build/cli/contract_validation.py index db219e45..57dd4b90 100644 --- a/fluid_build/cli/contract_validation.py +++ b/fluid_build/cli/contract_validation.py @@ -1023,13 +1023,19 @@ def _record_result_outcomes( ) -> None: """One :class:`CheckOutcome` per rule from the provider's full results, so a passing rule carries what was measured too.""" - by_id = {getattr(r, "rule_id", None): r for r in results} + # Results come back in rule order; a queue per id pairs the n-th rule + # with an id with the n-th result for it, so two rules sharing an id + # cannot hide one's failure behind the other's pass. + queues: Dict[Any, List[Any]] = {} + for result in results: + queues.setdefault(getattr(result, "rule_id", None), []).append(result) for rule in rules: if not isinstance(rule, dict) or not rule.get("id"): continue - result = by_id.get(rule["id"]) - if result is None: + queue = queues.get(rule["id"]) + if not queue: continue + result = queue.pop(0) self.report.record_check( CheckOutcome( name=f"{expose_id}.{rule['id']}", @@ -1118,8 +1124,7 @@ def _validate_against_actual_resource(self, expose: Dict[str, Any], path: str) - binding = expose.get("binding", {}) location = binding.get("location", {}) props = location.get("properties", {}) - self._validate_bigquery_resource(expose, path, props) - return True + return self._validate_bigquery_resource(expose, path, props) elif self.provider_name == "aws" and self._iceberg_outside_glue(expose, path): return False elif self.provider_name in ("snowflake", "aws", "local", "databricks"): @@ -1130,8 +1135,11 @@ def _validate_against_actual_resource(self, expose: Dict[str, Any], path: str) - def _validate_bigquery_resource( self, expose: Dict[str, Any], path: str, props: Dict[str, Any] - ) -> None: - """Validate against actual BigQuery table using provider abstraction.""" + ) -> bool: + """Validate against actual BigQuery table using provider abstraction. + + Returns whether the live columns were compared with the contract. + """ if not self.validation_provider: self.report.add_issue( "warning", @@ -1139,7 +1147,7 @@ def _validate_bigquery_resource( "No validation provider initialized, skipping resource validation", path, ) - return + return False project = props.get("project", self.project) dataset = props.get("dataset") @@ -1162,7 +1170,7 @@ def _validate_bigquery_resource( suggestion=f"Add the following properties to binding.location.properties: {', '.join(missing)}", documentation_url="https://agenticstiger.github.io/forge_docs/providers/gcp.html", ) - return + return False table_fqn = f"{project}.{dataset}.{table}" @@ -1248,6 +1256,7 @@ def _validate_bigquery_resource( LOG.info(f"✅ Resource '{table_fqn}' validated successfully") else: LOG.warning(f"⚠️ Resource '{table_fqn}' validation found issues") + return True except Exception as e: self.report.add_issue( @@ -1258,6 +1267,7 @@ def _validate_bigquery_resource( suggestion="Check provider credentials and network connectivity", documentation_url="https://agenticstiger.github.io/forge_docs/advanced/production-troubleshooting.html", ) + return False def _iceberg_outside_glue(self, expose: Dict[str, Any], path: str) -> bool: """Report an Iceberg table cataloged outside Glue as not checked. @@ -1300,19 +1310,22 @@ def _validate_generic_resource(self, expose: Dict[str, Any], path: str) -> bool: expose_id = expose.get("id") or expose.get("exposeId") or "unknown" try: - # Cache lookup + # Cache lookup. Databricks is always read live: its columns are a free + # metadata read, and a cached row count would report an hour-old table + # (and write it to a results table) after a build changed it. + cache = None if self.provider_name == "databricks" else self.cache actual_schema = None cache_key = self._resource_cache_key(expose, expose_id) - if self.cache: - actual_schema = self.cache.get_schema(cache_key, self.provider_name) + if cache: + actual_schema = cache.get_schema(cache_key, self.provider_name) if actual_schema: LOG.info("Using cached schema for %s", cache_key) # Fetch from provider if actual_schema is None: actual_schema = self.validation_provider.get_resource_schema(expose) - if self.cache and actual_schema: - self.cache.set_schema(cache_key, self.provider_name, actual_schema) + if cache and actual_schema: + cache.set_schema(cache_key, self.provider_name, actual_schema) # Validate result = self.validation_provider.validate_resource(expose, actual_schema) diff --git a/fluid_build/cli/test.py b/fluid_build/cli/test.py index ad9fa0b1..8b849446 100644 --- a/fluid_build/cli/test.py +++ b/fluid_build/cli/test.py @@ -267,7 +267,7 @@ def _write_results(report, contract, results_table: str, results_dashboard) -> N ensure_dashboard(results_dashboard, parse_results_table(results_table)) -def _results_options(args: argparse.Namespace): +def _results_options(args: argparse.Namespace, contract_path: Path): """``(table, dashboard path)`` of the results flags, checked before any test runs.""" table = getattr(args, "results_table", None) dashboard = getattr(args, "results_dashboard", None) @@ -287,10 +287,30 @@ def _results_options(args: argparse.Namespace): "fix": "drop --engine soda, or drop --results-table", }, ) + if table and getattr(args, "no_data", False): + raise CLIError( + 2, + "test_results_table_needs_live_checks", + { + "results_table": table, + "why": "--no-data runs no live check, so there would be no result to record", + "fix": "drop --no-data, or drop --results-table", + }, + ) if table: - from ._test_results_databricks import parse_results_table + from ._test_results_databricks import parse_results_table, refuse_product_schema + + parsed = parse_results_table(table) + # The product's own schema is refused here, before any statement runs, from + # the contract as written; write_results checks again with the overlay applied. + try: + import yaml - parse_results_table(table) + raw = yaml.safe_load(Path(contract_path).read_text(encoding="utf-8")) + except Exception: # noqa: BLE001 - an unreadable contract is the run's to report + raw = None + if isinstance(raw, dict): + refuse_product_schema(parsed, raw) if dashboard: from ._test_dashboard_databricks import parse_dashboard_path @@ -313,7 +333,7 @@ def run(args: argparse.Namespace, logger: logging.Logger) -> int: raise CLIError(1, "contract_file_not_found", {"path": str(contract_path)}) # A malformed results flag fails before any check runs (or any compute starts). - results_table, results_dashboard = _results_options(args) + results_table, results_dashboard = _results_options(args, contract_path) # Soda engine dispatch — generates SodaCL from the contract's quality # block and shells out to the user's installed `soda` binary. Skipped diff --git a/fluid_build/exporters/sodacl.py b/fluid_build/exporters/sodacl.py index fa615be0..27dfe19a 100644 --- a/fluid_build/exporters/sodacl.py +++ b/fluid_build/exporters/sodacl.py @@ -289,11 +289,14 @@ def _dq_rules_to_checks( unmapped: list[UnmappedRule], ) -> list[Any]: """Convert ``contract.dq.rules[]`` to SodaCL checks, accounting for each.""" + from fluid_build.providers.quality_engine import expose_columns + checks: list[Any] = [] + columns = expose_columns(dict(expose)) for rule in _dq_rules(expose): rule_id = str(rule.get("id") or "unnamed") rule_type = str(rule.get("type") or "").strip().lower() - emitted, reason = _convert_dq_rule(rule, rule_type) + emitted, reason = _convert_dq_rule(rule, rule_type, columns) if emitted: checks.extend(emitted) mapped.append(f"{expose_id}.{rule_id}") @@ -309,11 +312,15 @@ def _dq_rules_to_checks( return checks -def _convert_dq_rule(rule: Mapping[str, Any], rule_type: str) -> tuple[list[Any], str]: +def _convert_dq_rule( + rule: Mapping[str, Any], rule_type: str, columns: Any = None +) -> tuple[list[Any], str]: """Map one ``$defs.dqRule`` to SodaCL checks, or explain why we can't. Returns ``(checks, reason)``. ``checks`` empty ⇒ unmapped, and ``reason`` - says why in terms the contract author can act on. + says why in terms the contract author can act on. ``columns`` is the + expose's ``contract.schema``: a valid_values rule reads its column's + declarations from it, as the native engine does. """ selector = rule.get("selector") selector = selector.strip() if isinstance(selector, str) else "" @@ -337,6 +344,20 @@ def _convert_dq_rule(rule: Mapping[str, Any], rule_type: str) -> tuple[list[Any] if not selector: return [], "rule declares no 'selector' (column name)" + from fluid_build.providers.quality_engine import rule_columns + + listed = rule_columns(selector) + if len(listed) > 1: + # As the native engine reads it: every listed column must hold. + if rule_type not in ("completeness", "accuracy", "anomaly_detection") + _VALIDITY: + return [], f"a {rule_type} rule takes one column, not {len(listed)}" + per_column: list[Any] = [] + for column in listed: + emitted, reason = _convert_dq_rule({**rule, "selector": column}, rule_type, columns) + if not emitted: + return [], f"column {column!r}: {reason}" + per_column.extend(emitted) + return per_column, "" if not _SAFE_COLUMN.match(selector): # SodaCL check expressions are parsed by an ANTLR grammar; an # exotic identifier would produce a confusing parse error deep @@ -351,8 +372,8 @@ def _convert_dq_rule(rule: Mapping[str, Any], rule_type: str) -> tuple[list[Any] # Column-scoped anomaly_detection delegates to the same bound logic # the native engine uses (``_check_anomaly_detection`` → ``_check_accuracy``). return _bound_checks(selector, threshold, operator) - if rule_type in ("validity", "valid_values"): - return _valid_values_checks(selector, rule) + if rule_type in _VALIDITY: + return _valid_values_checks(selector, rule, columns) if rule_type == "freshness": return _freshness_checks(selector, rule) @@ -363,6 +384,8 @@ def _convert_dq_rule(rule: Mapping[str, Any], rule_type: str) -> tuple[list[Any] ) +_VALIDITY = ("validity", "valid_values") + # SodaCL threshold comparators (soda-core 3.x supports =, !=, <, >, <=, >=). _SODACL_OPERATORS = { ">=": ">=", @@ -472,18 +495,45 @@ def _bound_checks(selector: str, threshold, operator: str) -> tuple[list[Any], s ) -def _valid_values_checks(selector: str, rule: Mapping[str, Any]) -> tuple[list[Any], str]: - """validity / valid_values ⇒ ``invalid_count(col) = 0`` + ``valid values``.""" - from fluid_build.providers.quality_engine import extract_valid_values +def _valid_values_checks( + selector: str, rule: Mapping[str, Any], columns: Any = None +) -> tuple[list[Any], str]: + """validity / valid_values ⇒ ``invalid_count(col) = 0`` with the column's + ``valid values`` / ``valid regex`` / ``valid min`` / ``valid max``. - values = extract_valid_values(rule) - if not values: + The allowed values come from the native engine's own reading + (``quality_engine.value_domain``): the rule's list, else every declaration + the column makes. One SodaCL cannot express (a strict bound, a reference to + another product's table) leaves the rule unmapped rather than half-checked. + """ + from fluid_build.providers.quality_engine import value_domain + + domain = value_domain(dict(rule), columns, selector) + config: dict[str, Any] = {} + if domain.values: + config["valid values"] = list(domain.values) + if domain.pattern: + config["valid regex"] = domain.pattern + for op, bound in domain.bounds: + if op not in (">=", "<="): + return [], ( + f"range bound {op} {bound:g} has no SodaCL equivalent (valid min and " + "valid max are inclusive)" + ) + config["valid min" if op == ">=" else "valid max"] = _num_value(bound) + if domain.reference is not None: + product, column = domain.reference + return [], ( + f"the column references {product}.{column}, another product's table, " + "which SodaCL cannot look up" + ) + if not config: return [], ( "valid_values rule declares no allowed values — declare them in " - 'the description ("COLUMN valid values: A, B, C.") or as a ' - "'validValues' list" + 'the description ("COLUMN valid values: A, B, C."), as an enum, regex or ' + "range validationRule on the column, or as a 'validValues' list" ) - return [{f"invalid_count({selector}) = 0": {"valid values": list(values)}}], "" + return [{f"invalid_count({selector}) = 0": config}], "" def _freshness_checks(selector: str, rule: Mapping[str, Any]) -> tuple[list[Any], str]: diff --git a/fluid_build/providers/databricks/statements.py b/fluid_build/providers/databricks/statements.py index 5c206473..dc260075 100644 --- a/fluid_build/providers/databricks/statements.py +++ b/fluid_build/providers/databricks/statements.py @@ -95,12 +95,25 @@ def __init__( self._sql = SqlRunner( client, warehouse_id, timeout=timeout_seconds, poll=poll_seconds, sleep=sleep ) + self._unreachable: Optional[str] = None def run( self, sql: str, parameters: Optional[Sequence[Mapping[str, Any]]] = None ) -> List[tuple]: - """Run ``sql`` (``:name`` markers bound from ``parameters``) and return its rows.""" - doc = self._sql.execute(sql, parameters) + """Run ``sql`` (``:name`` markers bound from ``parameters``) and return its rows. + + A statement that failed because the warehouse did not answer (a timeout, a + transport error, a cancelled statement), rather than because of its SQL, stops + the ones after it: each would wait out the same timeout. + """ + if self._unreachable: + raise StatementError(f"not run: {self._unreachable}") + try: + doc = self._sql.execute(sql, parameters) + except StatementError as exc: + if not str(exc).startswith("FAILED"): + self._unreachable = f"an earlier statement did not complete ({exc})" + raise columns = ((doc.get("manifest") or {}).get("schema") or {}).get("columns") or [] types = [str(c.get("type_name") or "").upper() for c in columns] statement_id = str(doc.get("statement_id") or "") diff --git a/fluid_build/providers/databricks_validation.py b/fluid_build/providers/databricks_validation.py index efc3fbfa..6ee83a8f 100644 --- a/fluid_build/providers/databricks_validation.py +++ b/fluid_build/providers/databricks_validation.py @@ -354,7 +354,15 @@ def _reference_table( """ key = (product_id, column.lower(), (prefer_catalog or "").lower()) if key not in self._references: - self._references[key] = self._find_reference(product_id, column, prefer_catalog) + try: + self._references[key] = self._find_reference(product_id, column, prefer_catalog) + except Exception as exc: # noqa: BLE001 - one failed lookup, said once per rule + from fluid_build.providers.databricks.statements import scrub + + self._references[key] = LookupError( + f"the lookup of the table tagged {PRODUCT_TAG}={product_id} failed " + f"({type(exc).__name__}: {scrub(exc)})" + ) found = self._references[key] if isinstance(found, LookupError): raise found @@ -390,10 +398,12 @@ def _find_reference( "visible to this identity (has that product been applied?)" ) if len(having) > 1: - listed = ", ".join(".".join(p) for p in having[:3]) return LookupError( + # The tables are not named: this message is written to the shared + # results table, whose readers may not see them. f"{len(having)} tables tagged {PRODUCT_TAG}={product_id} have a column " - f"{column!r} ({listed}), so which one it references is ambiguous" + f"{column!r}, so which one it references is ambiguous; keep the tag on the " + "product's own table only" ) try: return sql_table_ref(having[0]) diff --git a/fluid_build/providers/quality_engine.py b/fluid_build/providers/quality_engine.py index e30ee386..df9811ce 100644 --- a/fluid_build/providers/quality_engine.py +++ b/fluid_build/providers/quality_engine.py @@ -152,9 +152,9 @@ def parse_range(constraint: Any) -> List[Tuple[str, float]]: @dataclass(frozen=True) class ValueDomain: - """What a valid value of one column is: the first declaration found of - the rule's own values, the column's ``enum``, ``regex`` or ``range`` - validationRule, or its ``labels.references``.""" + """What a valid value of one column is: the rule's own value list, or else + every declaration the column makes (its ``enum``, ``regex`` and ``range`` + validationRules and its ``labels.references``), all of which must hold.""" values: Tuple[str, ...] = () pattern: Optional[str] = None @@ -165,16 +165,17 @@ class ValueDomain: def value_domain(rule: Dict[str, Any], columns: Any = None, column: Optional[str] = None): """The :class:`ValueDomain` of ``column`` (default: the rule's selector).""" sub = rule if column is None else {**rule, "selector": column} - values = extract_valid_values(sub, columns) - if values: - return ValueDomain(values=tuple(values)) - pattern = extract_value_pattern(sub, columns) - if pattern: - return ValueDomain(pattern=pattern) - bounds = parse_range(_column_constraint(columns, sub.get("selector") or "", "range")) - if bounds: - return ValueDomain(bounds=tuple(bounds)) - return ValueDomain(reference=extract_reference(sub, columns)) + listed = extract_valid_values(sub) + if listed: + return ValueDomain(values=tuple(listed)) + name = sub.get("selector") or "" + enum = _column_constraint(columns, name, "enum") + return ValueDomain( + values=tuple(v.strip() for v in re.split(r"[|,]", enum) if v.strip()) if enum else (), + pattern=extract_value_pattern(sub, columns), + bounds=tuple(parse_range(_column_constraint(columns, name, "range"))), + reference=extract_reference(sub, columns), + ) #: ``resolve_reference(product id, column) -> table reference for SQL``; raises @@ -304,33 +305,41 @@ def _validity_pattern_sql(table_ref: str, column: str, pattern: str, dialect: st anchored pattern (``^[A-Z]{2}$``) means the same on all of them. """ col = _qi(column, dialect) - literal = quote_string_literal_for(dialect, pattern) - name = _dialect(dialect) - if name == "databricks": - mismatch = f"NOT ({col} RLIKE {literal})" - elif name == "bigquery": - mismatch = f"NOT REGEXP_CONTAINS({col}, {literal})" - elif name == "snowflake": - mismatch = f"REGEXP_INSTR({col}, {literal}) = 0" - elif name in ("ansi", "duckdb"): - mismatch = f"NOT regexp_matches({col}, {literal})" - else: - raise ValueError(f"no regular-expression test for SQL dialect {dialect!r}") return ( f"SELECT COUNT(*) AS invalid_count FROM {table_ref} " - f"WHERE {col} IS NOT NULL AND {mismatch}" + f"WHERE {col} IS NOT NULL AND {_pattern_mismatch(col, pattern, dialect)}" ) +def _pattern_mismatch(col: str, pattern: str, dialect: str) -> str: + """The predicate "the (quoted) column ``col`` does not match ``pattern``".""" + literal = quote_string_literal_for(dialect, pattern) + name = _dialect(dialect) + if name == "databricks": + return f"NOT ({col} RLIKE {literal})" + if name == "bigquery": + return f"NOT REGEXP_CONTAINS({col}, {literal})" + if name == "snowflake": + return f"REGEXP_INSTR({col}, {literal}) = 0" + if name in ("ansi", "duckdb"): + return f"NOT regexp_matches({col}, {literal})" + raise ValueError(f"no regular-expression test for SQL dialect {dialect!r}") + + +def _range_violation(col: str, bounds: Tuple[Tuple[str, float], ...]) -> str: + """The predicate "the (quoted) column ``col`` is outside ``bounds``".""" + inside = " AND ".join(f"{col} {op} {float(n)!r}" for op, n in bounds if op in _OPERATORS) + return f"NOT ({inside})" + + def _validity_range_sql( table_ref: str, column: str, bounds: Tuple[Tuple[str, float], ...], dialect: str ) -> str: """Count non-null values of ``column`` outside the declared bounds.""" col = _qi(column, dialect) - inside = " AND ".join(f"{col} {op} {float(n)!r}" for op, n in bounds if op in _OPERATORS) return ( f"SELECT COUNT(*) AS invalid_count FROM {table_ref} " - f"WHERE {col} IS NOT NULL AND NOT ({inside})" + f"WHERE {col} IS NOT NULL AND {_range_violation(col, bounds)}" ) @@ -935,6 +944,22 @@ def _check_accuracy( execute_fn, dialect: str = "ansi", ) -> QualityCheckResult: + if threshold is None: + # A bound with nothing to compare against checks nothing; reading it as + # passed whenever the column holds a value made a gate read green. + return QualityCheckResult( + rule_id=rule_id, + rule_type="accuracy", + selector=selector, + passed=False, + severity=severity, + message=( + f"Rule '{rule_id}' is an accuracy rule with no threshold, so nothing was " + "checked. Declare a threshold and an operator (>=, >, <=, <, == or !=)." + ), + expected="a threshold to compare with", + actual="not executed", + ) columns = rule_columns(selector) if len(columns) > 1: return _check_accuracy_columns( @@ -1100,24 +1125,23 @@ def _check_validity( rule_type: str = "validity", resolve_reference: Optional[ReferenceResolver] = None, ) -> QualityCheckResult: + col = _qi(selector, dialect) + violations: List[str] = [] + kinds: List[str] = [] if domain.values: - col = _qi(selector, dialect) # The literals are contract-supplied, so the quoting must hold on the # *target* dialect (a value ending in a backslash must not end the # literal early and append live SQL to this predicate). escaped = ", ".join(quote_string_literal_for(dialect, str(v)) for v in domain.values) - sql = ( - f"SELECT COUNT(*) AS invalid_count FROM {table_ref} " - f"WHERE {col} IS NOT NULL AND {col} NOT IN ({escaped})" - ) - expected = "0 invalid values" - elif domain.pattern: - sql = _validity_pattern_sql(table_ref, selector, domain.pattern, dialect) - expected = f"0 values not matching {domain.pattern}" - elif domain.bounds: - sql = _validity_range_sql(table_ref, selector, domain.bounds, dialect) - expected = "0 values outside " + " and ".join(f"{op} {n:g}" for op, n in domain.bounds) - elif domain.reference is not None: + violations.append(f"{col} NOT IN ({escaped})") + kinds.append("invalid values") + if domain.pattern: + violations.append(_pattern_mismatch(col, domain.pattern, dialect)) + kinds.append(f"values not matching {domain.pattern}") + if domain.bounds: + violations.append(_range_violation(col, domain.bounds)) + kinds.append("values outside " + " and ".join(f"{op} {n:g}" for op, n in domain.bounds)) + if not violations and domain.reference is not None: return _check_reference( rule_id, selector, @@ -1130,6 +1154,13 @@ def _check_validity( dialect, rule_type, ) + if violations: + invalid = violations[0] if len(violations) == 1 else f"({' OR '.join(violations)})" + sql = ( + f"SELECT COUNT(*) AS invalid_count FROM {table_ref} " + f"WHERE {col} IS NOT NULL AND {invalid}" + ) + expected = "0 " + kinds[0] if len(kinds) == 1 else "0 " + "; 0 ".join(kinds) else: # A misconfigured gate is a failed gate. Forcing 'warning' here # overrode the author's declared severity, so a rule declared @@ -1153,7 +1184,7 @@ def _check_validity( rows = execute_fn(sql) invalid_count = rows[0][0] if rows and rows[0][0] is not None else 0 passed = int(invalid_count) == 0 - return QualityCheckResult( + result = QualityCheckResult( rule_id=rule_id, rule_type=rule_type, selector=selector, @@ -1167,6 +1198,36 @@ def _check_validity( expected=expected, actual=f"{invalid_count} invalid value(s)", ) + if domain.reference is None: + return result + # The column also references another product: that must hold too. + reference = _check_reference( + rule_id, + selector, + severity, + description, + domain.reference, + resolve_reference, + table_ref, + execute_fn, + dialect, + rule_type, + ) + failed = [r.message for r in (result, reference) if not r.passed] + return QualityCheckResult( + rule_id=rule_id, + rule_type=rule_type, + selector=selector, + passed=not failed, + severity=severity, + message="; ".join(failed) if failed else result.message, + expected=f"{result.expected}; {reference.expected}", + actual=( + "not executed" + if reference.actual == "not executed" + else f"{result.actual}; {reference.actual}" + ), + ) _NOT_RUN = ("not executed", "no allowed values declared") diff --git a/tests/cli/test_contract_validation_schema_rules.py b/tests/cli/test_contract_validation_schema_rules.py index 6af4e4c4..629acd2c 100644 --- a/tests/cli/test_contract_validation_schema_rules.py +++ b/tests/cli/test_contract_validation_schema_rules.py @@ -283,3 +283,128 @@ def run_quality_results(self, resource_spec, rules): def test_rule_kind_normalises_case_and_space(value): assert cv._rule_kind({"type": value}) == "schema" assert cv._rule_kind("nope") == "" and cv._rule_kind({}) == "" + + +# ------------------------------------------------ gcp: a failed read is not a pass + + +def gcp_expose(rules, **props): + location = {"properties": {"project": "p", "dataset": "d", "table": "t", **props}} + return {**expose(rules), "binding": {"platform": "gcp", "location": location}} + + +def gcp_validator(provider, tmp_path): + v = make_validator(provider, tmp_path) + v.provider_name = "gcp" + return v + + +def test_bigquery_read_failure_leaves_the_schema_rule_not_executed(tmp_path): + p = FakeProvider(schema_error=RuntimeError("bigquery down")) + v = gcp_validator(p, tmp_path) + v._validate_single_expose(gcp_expose([SCHEMA_RULE]), 0) + c = check(v, "e1.conforms") + assert not c.passed and c.actual == "not executed" + assert "Schema conforms" not in (c.message or "") + + +def test_bigquery_resource_returns_false_on_each_early_exit(tmp_path): + v = gcp_validator(FakeProvider(live=live(("a", "string"))), tmp_path) + assert v._validate_bigquery_resource(gcp_expose([]), "p", {"project": "p"}) is False + v.validation_provider = None + full = {"project": "p", "dataset": "d", "table": "t"} + assert v._validate_bigquery_resource(gcp_expose([]), "p", full) is False + assert v._validate_against_actual_resource(gcp_expose([]), "p") is False + + +def test_bigquery_resource_returns_true_when_columns_were_compared(tmp_path): + v = gcp_validator(FakeProvider(live=live(("a", "string"), ("b", "int"))), tmp_path) + assert v._validate_against_actual_resource(gcp_expose([]), "p") is True + + +# ------------------------------------------------------- duplicate rule ids + + +def test_two_rules_sharing_an_id_are_paired_with_their_own_results_in_order(tmp_path): + results = [ + QualityCheckResult("d", "completeness", "a", False, "error", "bad", "1.0", "0.5"), + QualityCheckResult("d", "completeness", "b", True, "error", "ok", "1.0", "1.0"), + ] + rules = [ + {"id": "d", "type": "completeness", "selector": "a"}, + {"id": "d", "type": "completeness", "selector": "b"}, + ] + p = FakeProvider(live=live(("a", "string"), ("b", "int")), results=results) + v = make_validator(p, tmp_path) + v._validate_single_expose(expose(rules), 0) + outcomes = [c for c in v.report.checks if c.name == "e1.d"] + assert [c.passed for c in outcomes] == [False, True] + assert [c.actual for c in outcomes] == ["0.5", "1.0"] + + +# --------------------------------------------- the schema cache and Databricks + + +class CountingProvider(FakeProvider): + """Row count 0 on the first read, 5 on the second; empty tables are an issue.""" + + def __init__(self): + super().__init__() + self.reads = 0 + + def get_resource_schema(self, resource_spec): + self.reads += 1 + count = 0 if self.reads == 1 else 5 + return ResourceSchema( + ResourceType.TABLE, + "c.s.t", + [FieldSchema("a", "string", "NULLABLE"), FieldSchema("b", "int", "NULLABLE")], + row_count=count, + ) + + def validate_resource(self, contract_spec, actual_schema): + from fluid_build.providers.validation_provider import ValidationIssue + + issues = [] + if actual_schema.row_count == 0: + issues.append( + ValidationIssue( + severity="warning", category="empty_table", message="empty", path="p" + ) + ) + return ValidationResult("x", True, issues, actual_schema) + + +def cached_validator(provider_name, tmp_path, monkeypatch): + from fluid_build.providers.validation_cache import ValidationCache + + monkeypatch.setenv("HOME", str(tmp_path / "home")) + provider = CountingProvider() + v = make_validator(provider, tmp_path) + v.provider_name = provider_name + v.use_cache = True + v.cache = ValidationCache(cache_dir=tmp_path / "cache", ttl=3600) + return v, provider + + +def empty_table_issues(v): + return [i for i in v.report.issues if i.category == "empty_table"] + + +def test_databricks_never_reads_the_schema_cache(tmp_path, monkeypatch): + v, provider = cached_validator("databricks", tmp_path, monkeypatch) + e = expose([]) + v._validate_generic_resource(e, "p") + assert len(empty_table_issues(v)) == 1 + v._validate_generic_resource(e, "p") + assert provider.reads == 2 + assert len(empty_table_issues(v)) == 1 # the second read saw 5 rows + assert list((tmp_path / "cache").glob("*.json")) == [] + + +def test_snowflake_still_uses_the_schema_cache(tmp_path, monkeypatch): + v, provider = cached_validator("snowflake", tmp_path, monkeypatch) + e = expose([]) + v._validate_generic_resource(e, "p") + v._validate_generic_resource(e, "p") + assert provider.reads == 1 diff --git a/tests/cli/test_fluid_test_dashboard.py b/tests/cli/test_fluid_test_dashboard.py index f4cb5081..563c55cf 100644 --- a/tests/cli/test_fluid_test_dashboard.py +++ b/tests/cli/test_fluid_test_dashboard.py @@ -151,7 +151,8 @@ def test_structure_filter_on_rule_datasets_but_not_on_failing_list(definition): sql = {d["name"]: "".join(d["queryLines"]) for d in definition["datasets"]} for name in ("fluid_dq_overall", "fluid_dq_products", "fluid_dq_trend"): assert "COALESCE(rule_type, '') <> 'structure'" in sql[name] - assert "structure" not in sql["fluid_dq_failing"] + # last_run's CTE names 'structure'; the rows listed (after FROM latest) are not filtered. + assert "structure" not in sql["fluid_dq_failing"].split("FROM latest", 1)[1] assert "WHERE NOT passed" in sql["fluid_dq_failing"] @@ -369,3 +370,56 @@ def test_no_host_means_no_url(): PATH, TABLE, client_factory=lambda: FakeClient(api, host=""), warehouse_id="wh" ) assert result["url"] is None + + +def test_dashboard_without_the_sdk_names_the_extra(): + def no_sdk(): + raise ModuleNotFoundError("No module named 'databricks'") + + with pytest.raises(CLIError) as ei: + dash.ensure_dashboard(PATH, TABLE, client_factory=no_sdk, warehouse_id="wh") + assert ei.value.event == "test_results_dashboard_failed" + assert "data-product-forge[databricks]" in ei.value.context["fix"] + + +def test_latest_run_is_the_latest_one_that_tested_rules(definition): + sql = {d["name"]: "".join(d["queryLines"]) for d in definition["datasets"]} + latest_users = ("fluid_dq_overall", "fluid_dq_products", "fluid_dq_failing") + for name in latest_users: + assert "WITH last_run AS" in sql[name], name + assert "MAX(CASE WHEN" in sql[name] and "JOIN last_run" in sql[name], name + assert "COALESCE(" in sql[name] and "MAX(run_time)" in sql[name], name + assert not any("QUALIFY" in s for s in sql.values()) + + +def stale_published(update_time, revision_create_time): + dashboard = live_dashboard() + dashboard["update_time"] = update_time + return FakeApi( + status=existing_status(), + dashboard=dashboard, + published={"revision_create_time": revision_create_time}, + ) + + +PUBLISH_CALL = ("POST", f"{dash.DASHBOARDS_PATH}/dash-1/published") + + +def test_unchanged_draft_newer_than_published_revision_is_published(): + api = stale_published("2026-11-12T10:05:00Z", "2026-11-12T10:00:00Z") + result = ensure(api) + assert result["action"] == "unchanged" + assert PUBLISH_CALL in api.verbs() + assert "PATCH" not in [v[0] for v in api.verbs()] + + +def test_unchanged_draft_older_than_published_revision_is_not_republished(): + api = stale_published("2026-11-12T10:00:00Z", "2026-11-12T10:05:00Z") + assert ensure(api)["action"] == "unchanged" + assert PUBLISH_CALL not in api.verbs() + + +def test_unparseable_timestamps_do_not_republish(): + api = stale_published("garbage", "also garbage") + assert ensure(api)["action"] == "unchanged" + assert PUBLISH_CALL not in api.verbs() diff --git a/tests/cli/test_fluid_test_results_table.py b/tests/cli/test_fluid_test_results_table.py index 83a92279..de8963f7 100644 --- a/tests/cli/test_fluid_test_results_table.py +++ b/tests/cli/test_fluid_test_results_table.py @@ -7,6 +7,7 @@ from __future__ import annotations import argparse +import json import logging import xml.etree.ElementTree as ET from datetime import datetime, timedelta, timezone @@ -306,31 +307,40 @@ def col(name): return [n for n, _, _ in rt.RESULT_COLUMNS].index(name) -def test_insert_statements_batches_25_rows(): - stmts = rt.insert_statements("`c`.`s`.`t`", [row_dict(i) for i in range(60)]) - assert [len(p) // len(rt.RESULT_COLUMNS) for _, p in stmts] == [25, 25, 10] +@pytest.mark.parametrize("count", [1, 60]) +def test_insert_is_one_statement_with_one_rows_parameter(count): + sql, params = rt.insert_statement("`c`.`s`.`t`", [row_dict(i) for i in range(count)]) + assert isinstance(sql, str) + assert [p["name"] for p in params] == ["rows"] + payload = json.loads(params[0]["value"]) + assert isinstance(payload, list) and len(payload) == count + names = {n for n, _, _ in rt.RESULT_COLUMNS} + assert all(set(item) == names for item in payload) + assert [item["rule_id"] for item in payload] == [f"rule{i}" for i in range(count)] -def test_insert_statement_markers_casts_and_param_values(): - ((sql, params),) = rt.insert_statements("`c`.`s`.`t`", [row_dict(0), row_dict(1)]) - n = len(rt.RESULT_COLUMNS) +def test_insert_statement_unpacks_the_json_array_in_sql(): + sql, _ = rt.insert_statement("`c`.`s`.`t`", [row_dict(0)]) assert sql.startswith("INSERT INTO `c`.`s`.`t` (run_id, run_time,") - assert f"CAST(:r0c{col('run_time')} AS TIMESTAMP)" in sql - assert f"CAST(:r1c{col('passed')} AS BOOLEAN)" in sql - assert f":r0c{col('run_id')}," in sql and "CAST(:r0c0" not in sql - assert len(params) == 2 * n - by_name = {p["name"]: p for p in params} - assert by_name[f"r0c{col('passed')}"]["value"] == "true" - assert by_name[f"r1c{col('passed')}"]["value"] == "false" - assert by_name[f"r0c{col('run_time')}"]["value"] == "2026-01-02T03:04:05.000006Z" - assert "value" not in by_name[f"r0c{col('layer')}"] + assert "from_json(:rows, 'ARRAY.py +""" diff --git a/tests/exporters/test_sodacl_column_values.py b/tests/exporters/test_sodacl_column_values.py new file mode 100644 index 00000000..405bb626 --- /dev/null +++ b/tests/exporters/test_sodacl_column_values.py @@ -0,0 +1,90 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""SodaCL export of ``valid_values`` rules from the column's own declarations, and of +multi-column selectors.""" + +from __future__ import annotations + +import yaml + +from fluid_build.exporters.sodacl import render_sodacl_document + +COLUMNS = [ + {"name": "status", "validationRules": [{"type": "enum", "constraint": "A | B"}]}, + {"name": "code", "validationRules": [{"type": "regex", "constraint": "^x$"}]}, + {"name": "pct", "validationRules": [{"type": "range", "constraint": ">= 0 and <= 100"}]}, + {"name": "pos", "validationRules": [{"type": "range", "constraint": "> 0"}]}, + {"name": "site", "labels": {"references": "acme.sites.site_id"}}, + {"name": "tier", "validationRules": [{"type": "enum", "constraint": "P | Q"}]}, +] + + +def render(rules): + contract = { + "exposes": [ + { + "id": "e1", + "binding": { + "platform": "databricks", + "location": {"database": "c", "schema": "s", "table": "t"}, + }, + "contract": {"schema": COLUMNS, "dq": {"rules": rules}}, + } + ] + } + result = render_sodacl_document(contract) + checks = (yaml.safe_load(result.text) or {}).get("checks for t", []) + return result, checks + + +def valid_values(rule_id, selector, **extra): + return {"id": rule_id, "type": "valid_values", "selector": selector, **extra} + + +def test_enum_becomes_valid_values(): + result, checks = render([valid_values("r", "status")]) + assert checks == [{"invalid_count(status) = 0": {"valid values": ["A", "B"]}}] + assert result.mapped == ["e1.r"] + + +def test_regex_becomes_valid_regex(): + _, checks = render([valid_values("r", "code")]) + assert checks == [{"invalid_count(code) = 0": {"valid regex": "^x$"}}] + + +def test_inclusive_range_becomes_valid_min_and_max(): + _, checks = render([valid_values("r", "pct")]) + assert checks == [{"invalid_count(pct) = 0": {"valid min": 0, "valid max": 100}}] + + +def test_strict_bound_is_unmapped_naming_the_bound(): + result, checks = render([valid_values("r", "pos")]) + assert checks == [] and result.mapped == [] + (rule,) = result.unmapped + assert rule.rule_id == "r" and "> 0" in rule.reason + + +def test_reference_column_is_unmapped_naming_the_product(): + result, checks = render([valid_values("r", "site")]) + assert checks == [] + (rule,) = result.unmapped + assert "acme.sites" in rule.reason + + +def test_rule_description_list_wins_over_the_column_enum(): + _, checks = render([valid_values("r", "tier", description="tier valid values: X, Y.")]) + assert checks == [{"invalid_count(tier) = 0": {"valid values": ["X", "Y"]}}] + + +def test_multi_column_completeness_is_one_check_per_column_and_mapped(): + result, checks = render([{"id": "c", "type": "completeness", "selector": "a,b"}]) + assert checks == ["missing_count(a) = 0", "missing_count(b) = 0"] + assert result.mapped == ["e1.c"] and result.unmapped == [] + + +def test_multi_column_uniqueness_is_unmapped(): + result, checks = render([{"id": "u", "type": "uniqueness", "selector": "a,b"}]) + assert checks == [] and result.mapped == [] + (rule,) = result.unmapped + assert "takes one column" in rule.reason diff --git a/tests/providers/test_databricks_statements.py b/tests/providers/test_databricks_statements.py index d8649dc8..fd841f08 100644 --- a/tests/providers/test_databricks_statements.py +++ b/tests/providers/test_databricks_statements.py @@ -180,3 +180,40 @@ def test_warehouse_id_from_env_trims_and_reads_os_environ(monkeypatch): assert warehouse_id_from_env({"DATABRICKS_WAREHOUSE_ID": " wh "}) == "wh" monkeypatch.setenv("DATABRICKS_WAREHOUSE_ID", "from-env") assert warehouse_id_from_env() == "from-env" + + +# ------------------------------------------------------- circuit breaker + + +def test_timeout_stops_the_statements_after_it_without_an_api_call(): + r, api = runner([running("PENDING")] * 50 + [{}], timeout_seconds=0) + with pytest.raises(StatementError, match="did not finish"): + r.run("q1") + calls = len(api.calls) + with pytest.raises(StatementError) as exc: + r.run("q2") + assert str(exc.value).startswith("not run:") + assert len(api.calls) == calls + + +def test_transport_failure_stops_the_statements_after_it_without_an_api_call(): + r, api = runner([RuntimeError("connection reset")]) + with pytest.raises(StatementError, match="RuntimeError"): + r.run("q1") + with pytest.raises(StatementError) as exc: + r.run("q2") + assert str(exc.value).startswith("not run:") + assert len(api.calls) == 1 + + +def test_a_sql_error_does_not_stop_the_statements_after_it(): + failed = { + "statement_id": "sid-1", + "status": {"state": "FAILED", "error": {"message": "[TABLE_OR_VIEW_NOT_FOUND] t"}}, + } + r, api = runner([failed, ok([["7"]], [("n", "LONG")])]) + with pytest.raises(StatementError) as first: + r.run("q1") + assert not str(first.value).startswith("not run:") + assert r.run("q2") == [(7,)] + assert len([c for c in api.calls if c[0] == "POST"]) == 2 diff --git a/tests/providers/test_databricks_validation.py b/tests/providers/test_databricks_validation.py index fb4bd8c0..f1ceb73d 100644 --- a/tests/providers/test_databricks_validation.py +++ b/tests/providers/test_databricks_validation.py @@ -372,6 +372,8 @@ def test_reference_ambiguous_is_not_executed(provider, live): ) (res,) = provider.run_quality_results(REF_EXPOSE, [REF_RULE]) assert res.actual == "not executed" and "ambiguous" in res.message + # The message lands in the shared results table: other tables are not named. + assert "main.a.sites" not in res.message and "main.b.sites" not in res.message def test_reference_lookup_cached_per_product_and_column(provider, live): @@ -427,3 +429,25 @@ def test_reference_two_candidates_elsewhere_stay_ambiguous(provider, live): ) (res,) = provider.run_quality_results(REF_EXPOSE, [REF_RULE]) assert res.actual == "not executed" and "ambiguous" in res.message + + +def test_failed_tag_lookup_is_cached_as_a_lookup_error(provider, live, monkeypatch): + answer = live._answer + + def failing_tags(body): + if "table_tags" in body["statement"]: + return { + "statement_id": "s", + "status": {"state": "FAILED", "error": {"message": "[INSUFFICIENT_PERMISSIONS]"}}, + } + return answer(body) + + monkeypatch.setattr(live, "_answer", failing_tags) + rules = [REF_RULE, {**REF_RULE, "id": "fk2"}] + results = provider.run_quality_results(REF_EXPOSE, rules) + tag_queries = [b for b in live.sql if "table_tags" in b["statement"]] + assert len(tag_queries) == 1 + assert len(results) == 2 + for res in results: + assert not res.passed and res.actual == "not executed" + assert "lookup" in res.message and "failed" in res.message diff --git a/tests/providers/test_quality_engine_column_rules.py b/tests/providers/test_quality_engine_column_rules.py index b713b4d2..19f6dfb0 100644 --- a/tests/providers/test_quality_engine_column_rules.py +++ b/tests/providers/test_quality_engine_column_rules.py @@ -401,7 +401,7 @@ def test_range_rule_counts_values_outside_the_bounds_on_databricks(): rule = {"id": "r", "type": "valid_values", "selector": "grid_kwh", "severity": "error"} (res,) = execute_quality_checks([rule], "`c`.`s`.`t`", execute, "databricks", columns=ENERGY) assert res.passed and res.expected == "0 values outside >= 0" - assert "WHERE `grid_kwh` IS NOT NULL AND NOT (`grid_kwh` >= 0.0)" in seen[0] + assert seen[0].endswith("WHERE `grid_kwh` IS NOT NULL AND NOT (`grid_kwh` >= 0.0)") def test_valid_values_over_columns_checks_each_against_its_own_domain(): @@ -437,7 +437,7 @@ def test_valid_values_over_columns_with_an_undeclared_column_is_not_executed(): assert "nothing checked for 'free_text'" in res.message -def test_value_domain_order_rule_list_then_enum_regex_range_reference(): +def test_value_domain_is_the_rule_list_else_every_column_declaration(): columns = [ { "name": "a", @@ -460,8 +460,69 @@ def test_value_domain_order_rule_list_then_enum_regex_range_reference(): ] listed = {"id": "r", "selector": "a", "description": "a valid values: K, L."} assert value_domain(listed, columns) == ValueDomain(values=("K", "L")) - assert value_domain({"selector": "a"}, columns) == ValueDomain(values=("X", "Y")) - assert value_domain({"selector": "b"}, columns) == ValueDomain(pattern="^x$") + assert value_domain({"selector": "a"}, columns) == ValueDomain( + values=("X", "Y"), pattern="^x$", bounds=((">=", 0.0),), reference=("p.q", "col") + ) + assert value_domain({"selector": "b"}, columns) == ValueDomain( + pattern="^x$", bounds=((">=", 0.0),) + ) assert value_domain({"selector": "c"}, columns) == ValueDomain(bounds=((">=", 0.0),)) assert value_domain({"selector": "d"}, columns) == ValueDomain(reference=("p.q", "col")) assert value_domain({"selector": "zz"}, columns) == ValueDomain() + + +def test_every_column_declaration_is_checked_in_one_statement(): + # A column with an enum, a regex and a range: a value that breaks any of them is + # invalid (the range used to be dropped when a regex came first). + columns = [ + { + "name": "pct", + "validationRules": [ + {"type": "regex", "constraint": "^[0-9]+$"}, + {"type": "range", "constraint": ">= 0 and <= 100"}, + {"type": "enum", "constraint": "0,50,100,150"}, + ], + } + ] + seen, execute = _recording([1]) + rule = {"id": "p", "type": "valid_values", "selector": "pct", "severity": "error"} + (res,) = execute_quality_checks([rule], "t", execute, "databricks", columns=columns) + assert len(seen) == 1 + where = seen[0].split("WHERE", 1)[1] + assert "NOT IN ('0', '50', '100', '150')" in where + assert "NOT (`pct` RLIKE '^[0-9]+$')" in where + assert "NOT (`pct` >= 0.0 AND `pct` <= 100.0)" in where + assert where.count(" OR ") == 2 + assert not res.passed and res.actual == "1 invalid value(s)" + + +def test_a_column_with_values_and_a_reference_checks_both(): + columns = [ + { + "name": "site_id", + "validationRules": [{"type": "regex", "constraint": "^S[0-9]+$"}], + "labels": {"references": "acme.sites.site_id"}, + } + ] + seen, execute = _recording([0, 2]) + rule = {"id": "ref", "type": "valid_values", "selector": "site_id", "severity": "warn"} + (res,) = execute_quality_checks( + [rule], + "t", + execute, + "databricks", + columns=columns, + resolve_reference=lambda product, column: "`m`.`s`.`sites`", + ) + assert len(seen) == 2 and "RLIKE" in seen[0] and "NOT EXISTS" in seen[1] + assert not res.passed and "2 value(s) of 'site_id' are not in acme.sites.site_id" in res.message + assert res.actual == "0 invalid value(s); 2 value(s) missing" + + +def test_accuracy_without_a_threshold_is_not_run(): + seen, execute = _recording([]) + for selector in ("a", "a,b"): + rule = {"id": "acc", "type": "accuracy", "selector": selector, "severity": "error"} + (res,) = execute_quality_checks([rule], "t", execute, "databricks") + assert not res.passed and res.actual == "not executed" and "no threshold" in res.message + assert seen == [] From b6e8eb9c3ed6eb605cd8fdf0e9e972dbdeb8c69a Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:46:00 +0100 Subject: [PATCH 6/6] fix(test): low review notes on the Databricks results checks - The early product-schema check reads the contract through fluid_build.loader.load_contract, as the no-raw-readers rule asks, and a malformed `exposes` is left to the validator instead of raising. - The dashboard's publish-time comparison reads timestamps with any fraction length (Python 3.10's fromisoformat takes only 3 or 6 digits). - A range bound too large for a float is no bound (it reached SQL as inf). - The SodaCL reason for a bound with no threshold no longer says the native engine passes it. - HONESTLY_TESTED: the reference count is the 23 that ran on deployed tables. --- HONESTLY_TESTED.md | 2 +- fluid_build/cli/_test_dashboard_databricks.py | 11 +++++++++-- fluid_build/cli/_test_results_databricks.py | 4 +++- fluid_build/cli/test.py | 10 +++++----- fluid_build/exporters/sodacl.py | 2 +- fluid_build/providers/quality_engine.py | 19 +++++++++++-------- tests/cli/test_fluid_test_dashboard.py | 13 +++++++++++++ tests/cli/test_fluid_test_results_table.py | 5 +++++ .../test_quality_engine_column_rules.py | 5 +++++ 9 files changed, 53 insertions(+), 18 deletions(-) diff --git a/HONESTLY_TESTED.md b/HONESTLY_TESTED.md index f9b04d81..d0d5d52f 100644 --- a/HONESTLY_TESTED.md +++ b/HONESTLY_TESTED.md @@ -99,7 +99,7 @@ Markers declared in `pyproject.toml`. | Manual live, Free Edition, Genie space, 2026-11-12 | the real `fluid` CLI, no test file: branch `feat/databricks-genie-space`, databricks-sdk 0.162.0, a copy of industry-models tower-management `dp019_site_360` in the scratch catalog `fluid_iactest_genie` (no dbt build, no MENA sovereignty block: the workspace is in us-west-2), 6 synthetic rows | ✅ dry-run (would create, would share), apply `+73` then the space created and shared `CAN_RUN` with two persona service principals, a second apply `+0 ~0 -0` and "no change", the question "Which on-air sites in Amman are colocation-ready with hybrid power and no standing risk?" answered through the Genie Conversation API with the 2 expected sites (its SQL took the latest `snapshot_date` and the inverse of the `at_risk_site_count` filter from the instruction), a metric view created by hand picked up ("updated: metric views", then "no change"), the block removed (trashed). Two defects found and fixed in the run: the metric-view read failed on a schema not created yet, and the service keeps a metric view among the tables. A re-check after the security review (the plugin's own hook over `samples.nyctaxi.trips`, nothing created in Unity Catalog) found the list shows a new space only ~9 s later (a run straight after made a second space) and that a taken title is suffixed on create and 409 on PATCH; fixed, then create, an immediate second run ("no change", same space), and the block removed (trashed, via the folder check) passed live. Swept: the space trashed, `DROP CATALOG fluid_iactest_genie CASCADE`, 0 left | A reader without `SELECT` asking (no persona credential here; the access model is Genie's documented one); a service principal as the deploying identity (its folder is `/Users/`, the one Databricks Asset Bundles build from `current_user.userName`; a create that lands anywhere else is trashed and stops the apply); a space over several products | | Stage 2 emulator (nightly, non-gating) | `tests/iac/test_iac_databricks_minilake.py` (job `nightly-databricks-minilake`) | ⚠️ 1 passed, 4 strict xfails (manual dispatch, run 42267640334, 2026-11-06, minilake v1.9.0). Schema, table and grant creates work, the replace gate holds, adding a column plans no replace, and a grant revoke is classified not gated. The four tests about those resources apply the module without its tag assignments (the `untagged` fixture). An xfail is satisfied only by its own gap's failure (`MinilakeGap`), and a pass fails the run, so an image that closes a gap turns the job red until the mark comes off. Earlier: 1/4 from the first run (2026-10-30) to the 2026-11-04 nightly, then 0/4 after #831, every apply stopping at the first tag. | A second plan is not empty and no in-place table update applies: minilake drops a CREATE TABLE's `TBLPROPERTIES` and `COMMENT`, and DuckDB cannot parse the `ALTER TABLE ... SET TBLPROPERTIES` the provider sends on every update of a table that sets properties. No tag is applied: minilake has no entity-tag-assignments API (it answers 501). v1.9.3 (2026-11-05) has both gaps, so the pin stays; an upstream issue for each is drafted, not filed. Jobs and workspace files are not in this flow yet. | | Stage 1 unit: `fluid test` on Databricks, results table, dashboard | `tests/providers/test_databricks_validation.py`, `tests/providers/test_databricks_statements.py`, `tests/providers/test_quality_engine_column_rules.py`, `tests/cli/test_contract_validation_schema_rules.py`, `tests/cli/test_fluid_test_results_table.py`, `tests/cli/test_fluid_test_dashboard.py` | ✅ Against a stubbed `api_client.do`: the Unity Catalog columns and the contract's types through the emitter's type table (`VARCHAR` is `string`), a missing table (404 → `missing_resource`, and every rule reported as not run with no statement), the row count, the warehouse check; the statements over `databricks_enforce.SqlRunner` with `:name` parameters, polling, timeout cancel, chunks, type conversion and scrubbed errors; every rule kind in the Databricks dialect (backticked identifiers, `\'` literals, `timestampdiff` freshness, `RLIKE`), comma-listed columns, values from the column's enum, regex or range rule, references resolved by the `fluid_product_id` tag (same catalog first, ambiguity refused, cached); `schema` rules from the column comparison; the result rows (one per JUnit case, `rule_type` `structure` for the structural ones, expose version, UTC run time), the batched parameterised `INSERT`, the refusals (bad name, a product's own schema, a table without the columns), the fix-its; the dashboard definition's structure and the create, unchanged, re-publish, update-with-etag and create-race paths. | What Databricks accepts: the manual live row below. | -| Manual live, Free Edition (founder-approved), 2026-11-12: `fluid test` results | the real `fluid` CLI, no test file: `fluid test --env databricks --output junit --results-table fluid_iactest_dqr.dq.fluid_test_results --results-dashboard '/Workspace/Shared/fluid_iactest_dqr/Data quality'` on the 19 tower-management products in `northwind_towers` (read only: `SELECT` and metadata reads), contracts derived from industry-models main | ✅ 19 runs, 17-34 s each, 18 exit 0; `dp019_site_360` exits 1 because its table is not deployed in the demo workspace (`missing_resource`; its rules are reported as not run, and no statement is sent). 168 rule cases: the 155 on deployed tables all ran (146 pass, 9 freshness rules fail at `warn`: the bronze data is hours old against a 15-minute window), among them 57 enum, 10 regex and 24 reference rules (3 of them bronze references outside `consumes[]`, resolved by the `fluid_product_id` tag), one range rule over five columns (industry-models#4's form), 19 `schema` and 19 comma-listed completeness rules. 396 rows appended, one per JUnit case; the dashboard was created by the first run, found unchanged by every later one, and updated (PATCH, re-publish) once when its definition changed; in a browser it shows the counters (88.1% of rules passing, 19 products, 20 failing rules at that point), the per-product bars by layer, the trend and the failing-checks table. Afterwards the catalog `fluid_iactest_dqr` was dropped CASCADE and the dashboard deleted. | Lakeview validates widget specs only when the dashboard renders, so a widget shape Databricks changes would show as "Invalid widget definition" without failing the command (the table widget did, until it carried every column key). | +| Manual live, Free Edition (founder-approved), 2026-11-12: `fluid test` results | the real `fluid` CLI, no test file: `fluid test --env databricks --output junit --results-table fluid_iactest_dqr.dq.fluid_test_results --results-dashboard '/Workspace/Shared/fluid_iactest_dqr/Data quality'` on the 19 tower-management products in `northwind_towers` (read only: `SELECT` and metadata reads), contracts derived from industry-models main | ✅ 19 runs, 17-34 s each, 18 exit 0; `dp019_site_360` exits 1 because its table is not deployed in the demo workspace (`missing_resource`; its rules are reported as not run, and no statement is sent). 168 rule cases: the 155 on deployed tables all ran (146 pass, 9 freshness rules fail at `warn`: the bronze data is hours old against a 15-minute window), every rule kind the pack uses among them: enum, regex and range values (#4's range rule over five columns included), 23 references (3 of them bronze references outside `consumes[]`, resolved by the `fluid_product_id` tag), `schema` rules and comma-listed completeness rules. 396 rows appended, one per JUnit case; the dashboard was created by the first run, found unchanged by every later one, and updated (PATCH, re-publish) once when its definition changed; in a browser it shows the counters (88.1% of rules passing, 19 products, 20 failing rules at that point), the per-product bars by layer, the trend and the failing-checks table. Afterwards the catalog `fluid_iactest_dqr` was dropped CASCADE and the dashboard deleted. | Lakeview validates widget specs only when the dashboard renders, so a widget shape Databricks changes would show as "Invalid widget definition" without failing the command (the table widget did, until it carried every column key). | | Stage 3 live | — | ❌ not automated: manual maintainer runs only, never CI. Two have run: the manual rows above (2026-11-01 and 2026-11-03). | Everything the rows above leave to the live tier. | ## What's NOT tested (deliberately deferred) diff --git a/fluid_build/cli/_test_dashboard_databricks.py b/fluid_build/cli/_test_dashboard_databricks.py index d3e61ff3..2f697f63 100644 --- a/fluid_build/cli/_test_dashboard_databricks.py +++ b/fluid_build/cli/_test_dashboard_databricks.py @@ -368,11 +368,18 @@ def _is_not_found(exc: BaseException) -> bool: def _later(first: Any, second: Any) -> bool: """Whether timestamp ``first`` is after ``second`` (``False`` when either is unreadable).""" + import re from datetime import datetime + def parse(value: Any) -> datetime: + text = str(value).replace("Z", "+00:00") + # Python 3.10's fromisoformat takes only 3 or 6 fraction digits; the API + # may send 1 to 9. Pad or cut the fraction to 6. + text = re.sub(r"\.(\d+)", lambda m: "." + (m.group(1) + "000000")[:6], text, count=1) + return datetime.fromisoformat(text) + try: - a = datetime.fromisoformat(str(first).replace("Z", "+00:00")) - b = datetime.fromisoformat(str(second).replace("Z", "+00:00")) + a, b = parse(first), parse(second) except (TypeError, ValueError): return False try: diff --git a/fluid_build/cli/_test_results_databricks.py b/fluid_build/cli/_test_results_databricks.py index db5e42f0..096a1708 100644 --- a/fluid_build/cli/_test_results_databricks.py +++ b/fluid_build/cli/_test_results_databricks.py @@ -92,7 +92,9 @@ def parse_results_table(value: Any) -> Tuple[str, str, str]: def _expose_tables(contract: Mapping[str, Any]) -> Dict[str, Tuple[str, str, str]]: """``expose id -> (catalog, schema, table)`` for each expose bound to a table.""" out: Dict[str, Tuple[str, str, str]] = {} - for idx, expose in enumerate(contract.get("exposes") or []): + exposes = contract.get("exposes") + # A malformed ``exposes`` (not a list) is the validator's to report. + for idx, expose in enumerate(exposes if isinstance(exposes, list) else []): if not isinstance(expose, dict): continue binding = expose.get("binding") if isinstance(expose.get("binding"), dict) else {} diff --git a/fluid_build/cli/test.py b/fluid_build/cli/test.py index 8b849446..479ac14c 100644 --- a/fluid_build/cli/test.py +++ b/fluid_build/cli/test.py @@ -304,13 +304,13 @@ def _results_options(args: argparse.Namespace, contract_path: Path): # The product's own schema is refused here, before any statement runs, from # the contract as written; write_results checks again with the overlay applied. try: - import yaml + from fluid_build.loader import load_contract - raw = yaml.safe_load(Path(contract_path).read_text(encoding="utf-8")) + written = load_contract(contract_path) except Exception: # noqa: BLE001 - an unreadable contract is the run's to report - raw = None - if isinstance(raw, dict): - refuse_product_schema(parsed, raw) + written = None + if isinstance(written, dict): + refuse_product_schema(parsed, written) if dashboard: from ._test_dashboard_databricks import parse_dashboard_path diff --git a/fluid_build/exporters/sodacl.py b/fluid_build/exporters/sodacl.py index 27dfe19a..c12c7bbd 100644 --- a/fluid_build/exporters/sodacl.py +++ b/fluid_build/exporters/sodacl.py @@ -468,7 +468,7 @@ def _bound_checks(selector: str, threshold, operator: str) -> tuple[list[Any], s if threshold is None: return [], ( "bound rule declares no 'threshold', so there is nothing to " - "assert (the native engine treats this as a no-op pass)" + "assert (the native engine reports it as not run, too)" ) if operator in (">=", ">"): return [f"min({selector}) {operator} {_num(threshold)}"], "" diff --git a/fluid_build/providers/quality_engine.py b/fluid_build/providers/quality_engine.py index df9811ce..175ae20e 100644 --- a/fluid_build/providers/quality_engine.py +++ b/fluid_build/providers/quality_engine.py @@ -17,6 +17,7 @@ from __future__ import annotations import logging +import math import re from dataclasses import dataclass from typing import Any, Callable, Dict, List, Optional, Tuple @@ -140,14 +141,16 @@ def parse_range(constraint: Any) -> List[Tuple[str, float]]: return [] m = _RANGE_BETWEEN.match(constraint) if m: - return [(">=", float(m.group(1))), ("<=", float(m.group(2)))] - bounds = [] - for part in _RANGE_AND.split(constraint.strip()): - m = _RANGE_BOUND.match(part) - if not m: - return [] - bounds.append((m.group(1), float(m.group(2)))) - return bounds + bounds = [(">=", float(m.group(1))), ("<=", float(m.group(2)))] + else: + bounds = [] + for part in _RANGE_AND.split(constraint.strip()): + m = _RANGE_BOUND.match(part) + if not m: + return [] + bounds.append((m.group(1), float(m.group(2)))) + # A bound too large for a float would reach SQL as ``inf``: not a number. + return bounds if all(math.isfinite(n) for _, n in bounds) else [] @dataclass(frozen=True) diff --git a/tests/cli/test_fluid_test_dashboard.py b/tests/cli/test_fluid_test_dashboard.py index 563c55cf..5dc7a6a5 100644 --- a/tests/cli/test_fluid_test_dashboard.py +++ b/tests/cli/test_fluid_test_dashboard.py @@ -423,3 +423,16 @@ def test_unparseable_timestamps_do_not_republish(): api = stale_published("garbage", "also garbage") assert ensure(api)["action"] == "unchanged" assert PUBLISH_CALL not in api.verbs() + + +@pytest.mark.parametrize( + "first, second, later", + [ + ("2026-11-12T21:16:40.1Z", "2026-11-12T21:16:40.05Z", True), + ("2026-11-12T21:16:40.123456789Z", "2026-11-12T21:16:40.1234Z", True), + ("2026-11-12T21:16:39Z", "2026-11-12T21:16:40.000001Z", False), + ], +) +def test_later_reads_any_fraction_length(first, second, later): + # Python 3.10's fromisoformat takes only 3 or 6 fraction digits. + assert dash._later(first, second) is later diff --git a/tests/cli/test_fluid_test_results_table.py b/tests/cli/test_fluid_test_results_table.py index de8963f7..38aa9d80 100644 --- a/tests/cli/test_fluid_test_results_table.py +++ b/tests/cli/test_fluid_test_results_table.py @@ -534,3 +534,8 @@ def test_unparseable_contract_skips_the_early_product_schema_check(tmp_path): args_for(path, results_table="main.energy.results"), path ) assert table == "main.energy.results" and dashboard is None + + +def test_a_malformed_exposes_is_left_to_the_validator(): + rt.refuse_product_schema(("main", "dq", "results"), {"exposes": 5}) + rt.refuse_product_schema(("main", "dq", "results"), {"exposes": "x"}) diff --git a/tests/providers/test_quality_engine_column_rules.py b/tests/providers/test_quality_engine_column_rules.py index 19f6dfb0..cf94a412 100644 --- a/tests/providers/test_quality_engine_column_rules.py +++ b/tests/providers/test_quality_engine_column_rules.py @@ -526,3 +526,8 @@ def test_accuracy_without_a_threshold_is_not_run(): (res,) = execute_quality_checks([rule], "t", execute, "databricks") assert not res.passed and res.actual == "not executed" and "no threshold" in res.message assert seen == [] + + +def test_parse_range_refuses_a_bound_too_large_for_a_float(): + assert parse_range(">= " + "9" * 400) == [] + assert parse_range("between 0 and " + "9" * 400) == []