diff --git a/.github/workflows/ai-provider-matrix.yml b/.github/workflows/ai-provider-matrix.yml index 3584ece58..b0ada9ba3 100644 --- a/.github/workflows/ai-provider-matrix.yml +++ b/.github/workflows/ai-provider-matrix.yml @@ -76,7 +76,7 @@ jobs: # whose emitter differences are not what it is testing. Needed # explicitly because dbt-duckdb declares `dbt-core>=1.8.0` with no # upper bound (verified on PyPI 2026-09-20). - python -m pip install -e ".[dev,local]" "dbt-core<2" dbt-duckdb + python -m pip install -e ".[dev,local,litellm]" "dbt-core<2" dbt-duckdb - name: Run strict hosted-provider E2E if: steps.gate.outputs.enabled == 'true' diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 8ea11e649..8d5a70e7a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -411,7 +411,7 @@ jobs: if: steps.gate.outputs.enabled == 'true' run: | pip install --upgrade pip - pip install -e ".[dev,local]" psycopg + pip install -e ".[dev,local,litellm,mcp]" psycopg - name: Apply seed.sql to dockerized Postgres if: steps.gate.outputs.enabled == 'true' @@ -512,7 +512,7 @@ jobs: if: steps.gate.outputs.enabled == 'true' run: | pip install --upgrade pip - pip install -e ".[dev,snowflake]" + pip install -e ".[dev,snowflake,litellm,mcp]" - name: Run Snowflake e2e if: steps.gate.outputs.enabled == 'true' @@ -585,7 +585,7 @@ jobs: if: steps.gate.outputs.enabled == 'true' run: | pip install --upgrade pip - pip install -e ".[dev,local]" + pip install -e ".[dev,local,litellm,mcp]" - name: Run live LLM gateway tests if: steps.gate.outputs.enabled == 'true' @@ -688,7 +688,7 @@ jobs: # binds `platform: postgres`, and without psycopg the gateway # answers every tools/call with UnsupportedBindingError. `[dev,local]` # alone made this job unpassable from the day it was added. - pip install -e ".[dev,local,postgres]" + pip install -e ".[dev,local,postgres,litellm,mcp]" # Fail HERE if the seed fails. This used to end in # `2>/dev/null || true`, which discarded both the error text and the # exit code, so a seed that never ran surfaced three steps later as diff --git a/.github/workflows/llm-smoke.yml b/.github/workflows/llm-smoke.yml index cdd6c3950..b78ca2851 100644 --- a/.github/workflows/llm-smoke.yml +++ b/.github/workflows/llm-smoke.yml @@ -42,7 +42,7 @@ jobs: - name: Install forge-cli run: | python -m pip install --upgrade pip - pip install -e ".[dev]" + pip install -e ".[dev,litellm]" - name: Smoke-test the LLM path against GitHub Models # Non-blocking: a GitHub Models outage / rate-limit must not fail diff --git a/.secrets.baseline b/.secrets.baseline index f0631d42f..821243e99 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -270,7 +270,7 @@ "filename": "fluid_build/cli/ai_setup.py", "hashed_secret": "fedd98aeb509344663485744497d778515a12158", "is_verified": false, - "line_number": 102 + "line_number": 169 } ], "fluid_build/cli/ai_source_setup.py": [ @@ -556,7 +556,7 @@ "filename": "pyproject.toml", "hashed_secret": "fdb92d6db3585d9dec894d5eb5bba7996cecc1dc", "is_verified": false, - "line_number": 85 + "line_number": 84 } ], "scripts/verify_compose.yml": [ @@ -1586,7 +1586,7 @@ "filename": "tests/test_command_center_ssrf.py", "hashed_secret": "c212d24997054583ddd732a7fae6a73c98b07c3a", "is_verified": false, - "line_number": 95 + "line_number": 97 } ], "tests/test_config_manager.py": [ @@ -1707,7 +1707,7 @@ "filename": "tests/test_datamesh_manager_publish_spec.py", "hashed_secret": "829c3804401b0727f70f73d4415e162400cbe57b", "is_verified": false, - "line_number": 719 + "line_number": 720 } ], "tests/test_doctor_store_backend.py": [ @@ -2154,5 +2154,5 @@ } ] }, - "generated_at": "2026-11-03T19:58:17Z" + "generated_at": "2026-11-04T00:03:27Z" } diff --git a/AGENTS.md b/AGENTS.md index ac84efe8e..409e1f070 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -199,7 +199,7 @@ new extra installed). `uv` is dev-time only — not a runtime dep. - **Changing what `fluid plan` emits**: `plan.json` must carry both `planDigest` and `bundleDigest` (re-run `inject_digests` after any mutation to the dict). Actions must carry BOTH `op` and `action_type` fields — apply.py's provider dispatcher reads `op`, display/viz reads `action_type`. Dropping either silently breaks the pipeline. - **Generating a new CI system template**: extend `fluid_build/forge/core/pipeline_templates.py`. The 11-stage sequence + the `--install-mode {pypi,dev-source}` semantics must be preserved across systems. Jenkins is the reference implementation; GitHub Actions / GitLab / Azure / Bitbucket / CircleCI / Tekton port from it. - **Adding a new Path-B scheduler**: extend `fluid_build/providers//plan/schedule.py` with a `SchedulePlanner`-compatible interface. Stage 6 `fluid plan` invokes it; stage 7 apply executes the resulting actions alongside DDL. -- **Adding a new publish target / catalog backend**: drop a new module under `fluid_build/build_runners/catalog_registrars/.py` that defines a `CatalogRegistrar` (the sync HTTP-level shape — see `datahub.py` as the reference) and ends with a `register_catalog_backend(CatalogBackendSpec(...))` call. The spec declares the canonical name, optional aliases, the registrar factory, and a flat `env_vars` mapping (config key → tuple of accepted env vars). That single file makes the backend visible on both publish surfaces — `fluid publish --target ` and contract `properties.catalog.register: []` — and applies the env-var overrides via `config_manager._apply_catalog_env_overrides`. No edits to `providers/catalogs/__init__.py`, `_catalog.py`, or `publish.py` are required. The two legacy native async providers (`fluid-command-center`, `datamesh-manager`) keep custom config shapes and live as hand-wired `BaseCatalogProvider` subclasses under `providers/catalogs/`; new backends should prefer the registrar shape unless they truly need the async-native surface. +- **Adding a new publish target / catalog backend**: drop a new module under `fluid_build/build_runners/catalog_registrars/.py` that defines a `CatalogRegistrar` (the sync HTTP-level shape — see `datahub.py` as the reference) and ends with a `register_catalog_backend(CatalogBackendSpec(...))` call. The spec declares the canonical name, optional aliases, the registrar factory, and a flat `env_vars` mapping (config key → tuple of accepted env vars). That single file makes the backend visible on both publish surfaces — `fluid publish --target ` and contract `properties.catalog.register: []` — and applies the env-var overrides via `config_manager._apply_catalog_env_overrides`. No edits to `providers/catalogs/__init__.py`, `_catalog.py`, or `publish.py` are required. Command Center (`fluid-command-center`) keeps a custom config shape as a hand-wired `BaseCatalogProvider` subclass under `providers/catalogs/`; Data Mesh Manager is a catalog backend (`build_runners/catalog_registrars/datamesh_manager.py`) reached with `fluid publish --target datamesh-manager`, plus the `fluid datamesh-manager` command, and is not a provider; new backends should prefer the registrar shape unless they truly need the async-native surface. - **Adding a copilot tool that takes a path**: accept `workspace_root: Optional[Path]` as a kwarg from `dispatch_tool_call`; confine the LLM-supplied path with `resolved.relative_to(workspace_root)` and apply a suffix allow-list + size cap. Reference implementation: `cli/forge_copilot_tools.py::_dispatch_read_sample_schema`. - **Emitting new SQL DDL**: identifiers must go through `fluid_build/providers/_sql_safety.py::validate_ident`; string literals through `quote_string_literal`. Inline `.replace("'", "''")` is considered a regression — it diverges from the central helper. - **Adding a secret pattern to logs**: extend `fluid_build/providers/snowflake/util/logging.py::SENSITIVE_PATTERNS`/`SENSITIVE_KEYS` AND mirror the addition into `fluid_build/observability/secret_redactor.py` to keep the two redactor layers symmetric. @@ -677,6 +677,72 @@ The reference plugin for `extension_validators` + `commands` is the declarations in its `pyproject.toml` is the canonical "how to write a plugin" example. +### Shipping an IaC plugin (an OpenTofu emitter for another platform) + +An outside package adds a platform to `fluid generate iac` and `fluid apply` +with an entry point; no forge-cli edit. + +```toml +[project.entry-points."fluid_build.iac_providers"] +onprem = "my_pkg.iac:OnPremIacPlugin" +``` + +- **Interface.** The target is a class (instantiated with no args) or an + instance implementing `fluid_build.iac.base.IacProviderPlugin`: `name`, + `required_providers`, `credential_env_vars`, and `emit`, `emit_data`, + `credential_env`, `discover_imports`, `provider_block`. Optional + capabilities are listed on that Protocol's docstring. The entry-point name is + the platform: `binding.platform: onprem` selects it. +- **Schema.** `binding.platform` is a closed enum in every shipped schema, so a + plugin serves a platform the schema admits and no built-in emitter serves + (`kubernetes`, `postgres`, `kafka`, ...). Opening the enum is a schema change. +- **Loading is lazy and by name** (stevedore `DriverManager` style, via + `iac.registry.get_iac_plugin`). Nothing is imported at startup, so `fluid + --help` and `fluid providers` never run plugin code; the plugin is imported + the first time a contract's platform needs it. `cutover.default_engine` sends + a platform a plugin serves to the OpenTofu engine. +- **Selecting a plugin platform.** A contract's `binding.platform` is + auto-detected. The global `fluid --provider` flag is validated against the + apply-provider registry and accepts built-ins only for now. +- **Policy.** `FLUID_PLUGINS_ALLOWLIST` / `FLUID_PLUGINS_BLOCKLIST` (names match + case-insensitively, `-` == `_`; the blocklist wins; qualified entries restrict + nothing) and the reserved names are checked before `ep.load()`. + `get_plugin_info().requires_cli` (PEP 440) is read from the loaded plugin, so + it is checked right after the import, with the provider registry's gate: a + WARNING `provider_version_warning`, and a refusal when + `FLUID_PLUGIN_STRICT_COMPAT=1`. +- **Reserved names.** An entry point may not claim a built-in platform: `aws`, + `gcp`, `snowflake`, `confluent`, `databricks`, any alias in + `iac/provider_match.py::PROVIDER_ALIASES` (`s3`, `glue`, `athena`, `google`, + `gcs`, `bigquery`, `local`, `duckdb`, ...) or a retired provider name + (`redshift`, `datamesh_manager`). It is skipped before it is imported, with a + WARNING `iac_provider_name_reserved` naming the distribution. A plugin's own + `name` (the Terraform provider's local name) is not reserved. +- **Operator opt-in to replace a built-in.** The operator grants it with a + distribution-qualified entry in `FLUID_PLUGINS_ALLOWLIST`: + `:`, e.g. `acme-fluid-aws:aws` (distribution + names compare PEP 503-normalised, the entry-point name case-insensitively). + The plugin then loads and replaces the built-in for that platform, with an + INFO `iac_provider_override_granted`. A bare `aws` cannot grant it (the + built-in has that name), and qualified entries never make the allowlist + restrictive: plain names keep their meaning. The distribution is read from the + installed distribution's real name (`ep.dist.name`, a valid PEP 508 name), not + its display string. A bare allowlist that omits the name cancels the grant + (WARNING `iac_provider_grant_cancelled`). A granted override replaces the + emitter plugin only, not the in-tree validation helpers + (`iac.providers.gcp._bq_type`, `validate_gcp_binding`, ...) that modules import + directly. +- **Duplicate names.** Several distributions shipping one reserved name: each + ungranted claim is warned about and the one granted claim loads. Several + shipping one non-reserved name: all are refused with one WARNING + (`iac_provider_name_ambiguous`); block the name with `FLUID_PLUGINS_BLOCKLIST` + or uninstall the extras. +- **State guard.** Before a destructive apply attributes a state to a cloud, + every admitted installed plugin is loaded (`state_migration`), so another + plugin cloud's resources are never mistaken for ambiguous ones. +- **Tests.** `tests/iac/test_iac_entrypoint_lazy_plugin.py` builds a real + throwaway `.dist-info` and is the pattern to copy. + ## Links - **Documentation**: [https://agenticstiger.github.io/forge_docs/](https://agenticstiger.github.io/forge_docs/) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a96a1fcb..6deb6e95f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **IaC plugins load when a contract needs their platform, and cannot take a built-in's name + unless the operator says so.** A package that ships an OpenTofu emitter through the + `fluid_build.iac_providers` entry-point group used to be imported whenever `fluid_build.iac` + was; now only its name is read, and it is imported the first time a contract's + `binding.platform` (or `fluid generate iac --provider`) asks for it, so `fluid --help` and + `fluid providers` run no plugin code. The allow/block lists and the reserved names are + checked before the import, and the plugin's `requires_cli` right after it (it is read from the + loaded plugin); allow/block names now match case-insensitively with `-` equal to `_`. A + contract's `binding.platform` selects a plugin platform by auto-detection; the global + `--provider` flag still accepts built-in providers only. Two distributions shipping the same + non-reserved name are both refused with a WARNING. Before a destructive apply attributes a + state to a cloud, every admitted installed plugin is loaded. An entry point claiming a built-in platform (`aws`, `gcp`, + `snowflake`, `confluent`, `databricks`, an alias such as `glue` or `bigquery`, or `local`) is + skipped without being imported, with a WARNING naming its distribution. **Breaking (plugins):** + a plugin that replaced a built-in emitter now needs the operator's grant, a + distribution-qualified `FLUID_PLUGINS_ALLOWLIST` entry such as `acme-fluid-aws:aws`; a + qualified entry does not restrict other plugins. The retired names `redshift` and + `datamesh-manager` cannot be granted. + +- **Extras for AI, MCP and lineage, ahead of their removal from the base install.** + `pip install 'data-product-forge[standard]'` installs litellm and the MCP SDK; `[litellm]` + (alias `[ai]`), `[mcp]` and `[lineage]` install one each. This release still puts all three in + the base install, so existing installs change nothing; 0.28.0 removes them from the base install, + and `fluid doctor` says so. Install `[standard]` now if you use AI authoring, `fluid ai setup` or + `fluid mcp serve`, and `[lineage]` if you set `OPENLINEAGE_URL`. The Docker images install + `[standard,lineage]`. `[mcp]` also declares `pyjwt[crypto]>=2.15.0`, which the MCP output-port + auth imports directly, and `[litellm]` keeps the Python 3.10 pin split from #555. + - **`fluid doctor --strict` turns the "Action needed" summary into exit 1.** `fluid doctor` printed "Action needed" for a copilot that is not ready and still exited 0, so a CI gate could not tell. With `--strict` it exits 1 whenever the summary says "Action needed", in the text report and with @@ -119,8 +147,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 against the live `row_filter` and each column's `mask` from the GetTable it already reads, and each bound function read back for its owner (this verify's identity or the table's owner), body, parameters and return type; a missing or different binding is CRITICAL with `fluid - apply` as the fix, and a contract that restricts nothing skips it. `policy.privacy.masking` - (landing-time masking), `rowLevelPolicy` and `binding.governance` are still refused. Proven + apply` as the fix, and a contract that restricts nothing skips it. `rowLevelPolicy` and + `binding.governance` are still refused, and `policy.privacy.masking` (landing-time masking) is + refused unless a DuckDB acquisition writes the expose (see the bronze landing entry). Proven against the in-process stand-in with a real `tofu` apply, and the function bodies evaluated in DuckDB; not run against a live workspace: that `CREATE FUNCTION IF NOT EXISTS`, `SET ROW FILTER` (with `ON ()` as well) and `SET MASK` behave as the stand-in models them, that `SET` @@ -130,8 +159,131 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 written, that `current_user()` is a service principal's application ID, and what a warehouse of each type enforces are unproven. +- **`lifecycle.expire` on a Databricks table is accepted when the table is rebuilt in full and + its Delta history is limited.** It used to be refused (`databricks-retention-unmapped`): + nothing on Databricks deletes rows by age. A table rewritten whole on every run holds no row + copy older than a run except in its Delta history, which `VACUUM` (run by predictive + optimization on Unity Catalog managed tables) empties after `delta.logRetentionDuration` + (30 days by default) and `delta.deletedFileRetentionDuration` (7 days). So the emitter and + `fluid validate` now accept `expire: true` when every build writing the expose is a dbt build + whose model for the table resolves to `materialized: table`, or forge-cli's DuckDB acquisition + with `properties.source.mode: full_refresh` (its Databricks landing overwrites the whole table, + and runs no other mode; another mode is `databricks-retention-incremental`, none `-unmapped`), + and `lifecycle.retention` is at least the larger of the two history properties (the model's + `tblproperties` when it sets them). The model is the one whose `alias`, else file name, is the + expose's table, with `materialized` resolved as dbt does: `config()` in the SQL, then the + model's YAML `config:`, then `dbt_project.yml` by the most specific path; the default is `view`. + The build must also run that model: its `--select` (`properties.model` / `select`, as + `build_selectors` makes it) must include the model by name, fully qualified name (dbt-core + 1.11's fqn method: an exact prefix, with or without the project's name, a wildcard globbing the + whole rest of the name) or path, else + `databricks-retention-not-selected` (graph operators on another node, tags, configs and state + cannot be evaluated here and are refused too). A table name, retention or source mode that is + still an `{{ env.X }}` placeholder is left by `fluid validate` to the emitter, which checks it + substituted, and `fluid verify` reads the contract with its placeholders substituted, as `fluid + apply` does. + Refused, with the fix: `databricks-retention-incremental` (`incremental`, `snapshot`), + `-not-table` (`view` and the rest), `-dbt-unresolved` (a Git repository, or a value that is not + a plain literal: `var()`, a conditional `config()`, a custom `schema`), `-too-short` + (a retention under the history, so under 30 days by default), `-period` (missing or not ISO 8601) + and `-unmapped` (any other writer, or none). ISO 8601 years and months are counted at their + shortest (P1Y = 365 days, P1M = 28 days), so the check never passes a retention that is too + short. `fluid verify` gains a `retention` dimension: the live table's Delta history + (`properties` and `delta_runtime_properties_kvpairs`) must not exceed the contract's retention + (CRITICAL, with the `ALTER TABLE ... SET TBLPROPERTIES`), and predictive optimization off for + the table is a warning with the fix (enable it, or schedule `VACUUM`), as is a table GetTable + serves without any properties (Delta's defaults were assumed; check `SHOW TBLPROPERTIES`); + skipped without `expire`. Not run against a live workspace: which property names Unity Catalog's GetTable + serves for a table's Delta retention, and what predictive optimization reports, are modelled + on databricks-sdk-go v0.190.0, not observed. + +- **A Databricks `accessPolicy` grant scoped by `resources` lands on the tables it names.** It used + to be refused (`databricks-grant-scope-unmapped`) whatever it named, which blocked every product + whose grants carry `resources: ["$.exposes[?(@.exposeId=='')]"]`. The emitter now evaluates + that shape (single or double quotes, spaces allowed) and `$.exposes[*]` / `$.exposes` (every + expose), and grants only on the matching tables; several resources add up. A grant without + `resources` still covers every table. Any other JSONPath (`$.exposes[0]`, a filter on another + field, `||`, a path below the expose), alone or beside a good one, and an exposeId this contract + does not have, stay `databricks-grant-scope-unmapped`, with a message that names the resource and + says the grant is refused rather than placed on more tables than it names; a blank resource + (`""`) is one of those, not "every expose". `fluid validate` and + the emitter read the one `_compile` verdict, and `fluid verify`'s catalog-access check reads the + per-table grants, so a scoped reader is checked only where it is granted. The scope is parsed + once, in `iac/access.py` (`AccessGrant.scope`); what the AWS and GCP emitters produce is + unchanged. Known gap, not fixed here: the GCP emitter applies every `accessPolicy` grant to every + expose and never reads `resources`, and the AWS emitter does not read `accessPolicy.grants`, so a + scoped grant is not scoped there. + +- **The DuckDB acquisition runner lands a Databricks expose in its table, and `policy.privacy.masking` + is accepted on one it writes.** For `platform: databricks` the runner used to write a LOCAL file + (`location.path`, else `out/`) and report success with nothing in Databricks. It now hashes (or + tokenizes, encrypts, masks) at landing as before, writes Parquet under `.fluid/staging`, uploads it + through the Files API to `DATABRICKS_STAGING_VOLUME` (`/Volumes///`, from + the environment only, like `DATABRICKS_WAREHOUSE_ID`) as `///.parquet`, + and runs one statement on the warehouse, `INSERT OVERWRITE .. + () SELECT FROM read_files('', format => + 'parquet')`: a full refresh, each value into the column of its name, that rewrites the rows and keeps the table's definition, grants, tags, row filter and column masks. + It reads only the file the run uploaded, never the directory, so a file someone else puts there is + not loaded. The staged files are deleted afterwards whether or not the load worked, and on Ctrl-C + too (an INSERT the warehouse has answered for is cancelled first); a file that cannot be deleted + fails the run and is named first in the error. A run killed outright cannot clean up, so each + load first deletes what earlier runs of the same build left under `///`: + only a directory named like a run id that holds nothing but Parquet files, all over an hour old + (a run still in progress keeps its upload, and a directory forge-cli did not stage is left). + Give the staging a volume of its own. The run record gains + `facets.databricks_load` (table, rows loaded, where it staged, that it deleted, and + `earlier_runs_staging_deleted`). The table must exist: a missing one fails naming + `fluid apply`, and the runner never creates or alters a table. An unset volume or warehouse, a + mode other than `full_refresh`, a sink other than Parquet and more than one stream are refused + before anything runs, and the emitter refuses the mode and the sink format already + (`databricks-landing-unsupported`), with any masking rule the runner cannot apply + (`databricks-masking-invalid`), so `fluid validate` says so before `fluid apply` creates the + table. Every other platform's landing is unchanged. The Databricks emitter now + accepts `policy.privacy.masking` on an expose when every build that writes it is a DuckDB + acquisition build (the runner applies the mask before the data leaves the machine, as on AWS), and + keeps refusing it, with the reason, for a dbt-written expose, another engine, or no build at all; + `fluid validate` and `fluid apply` read the one `_compile` verdict. `rowLevelPolicy` and + `binding.governance` are still refused. Proven against the in-process stand-in (Files API, + `INSERT OVERWRITE ... read_files` over the uploaded Parquet in DuckDB) with a real `tofu` apply + followed by the build; not run against a live workspace: what `read_files` and `INSERT OVERWRITE` + do on a managed table that carries a mask and a row filter, that `INSERT OVERWRITE` keeps them, + the Files API's limits, and whether a type the Parquet carries is accepted by the table's are + unproven. The run-time schema gate still compares a declared `string` with DuckDB's `VARCHAR` as a + type change; declare `varchar` for a column the source reads as text. + ### Fixed +- **`fluid apply --mode amend-and-build` provisioned a product, then refused its id.** The build + phase keeps run records, cursors and watermarks under `.fluid/runs///`, + so its ids must start with a letter or underscore, use only letters, digits, `_`, `.` and `-`, + and be at most 128 characters. The contract schema allows more (`1orders`, or a longer id), so + `fluid validate` and `fluid plan` passed such an id. Only the build runner checked it, and on the + OpenTofu engine (AWS, GCP, Snowflake, Confluent, Databricks and IaC plugin clouds) the runner + starts after `tofu apply`: the infrastructure was created, and the run then stopped with + "Unexpected error". A build mode (`amend-and-build`, `replace-and-build`) now checks + `contract.id` and every `builds[].id` before the apply hooks, `tofu` and the runner, on both + engines, and refuses with `ERR_APPLY_BUILD_IDENTIFIER_INVALID`, which names the id, says nothing + was applied and suggests a rename. A mode without a build phase, and a contract without builds, + are not checked, as before. +- **A mistyped `--build-id` provisioned the product, then found no build.** The build runner keeps + the builds whose id equals `--build-id` and, with none left, logged `Build not found` and exited + 1, but on the OpenTofu engine it starts after `tofu apply`, so the infrastructure was created + first. A build mode now checks `--build-id` against the contract's builds before the apply + hooks, `tofu` and the runner, on both engines, and refuses with `ERR_APPLY_BUILD_ID_NOT_FOUND`, + which names the id, lists the ids the contract has and says nothing was applied. Ids match + exactly, as in the runner. An empty `--build-id` and a contract without builds are not checked, + as before. The Command Center run now reports `apply_build_id_not_found`, where it reported + `build_not_found:`. + +- **A missing optional dependency names the extra to install, and silent fallbacks warn.** + litellm, the MCP SDK, openlineage, boto3 and duckdb are imported through one gate: when one is + missing, the command prints ` needs , which is not installed.` and + `pip install 'data-product-forge[]'`, with no traceback, and exits 1, where some paths + failed deep inside with a traceback. A package that is installed but fails to import (litellm >= 1.98 on Python 3.10) is reported as + that. Fallbacks that used to be silent now warn once: the ODCS typed model, the Glue catalog id, + and credentials without pydantic-settings. `fluid doctor` reads these packages' versions from + their metadata instead of importing them (about 1,000 modules loaded instead of 3,300). + - **`fluid diff --baseline` reported nothing for column changes in a 0.7.x contract.** It read columns from `exposes[].schema`, but a 0.7.x contract keeps them in `exposes[].contract.schema` (as does every `fluid init` template), so a removed column, a type change or a new NOT NULL @@ -143,9 +295,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 column, a type change that is not a widening, an added `required: true` column), so a pipeline that runs `fluid diff --baseline --fail-on-breaking` can start exiting 1. A finding for a 0.7.x column is reported at `exposes[N].contract.schema.`, and a nullability one at its - `.required`. Still not compared in a 0.7.x contract: quality-rule severity and a column's - `sensitivity`. A 0.7.x primary key is not recognised either, so a key column that becomes - nullable is reported as non-breaking. + `.required`. + +- **`fluid diff --baseline` ignored a 0.7.x contract's quality rules, sensitivity and primary + key.** A 0.7.x contract keeps them under `exposes[].contract`, where the diff did not look. A + rule in `contract.dq.rules`, `contract.quality` or a column's `quality` whose severity becomes + failing is now `quality_severity_escalated` (breaking), as a pre-0.7 `quality.tests` entry was. + `critical` counts as failing, as it does for `fluid test`: warn -> critical is now an + escalation, and critical -> error no longer is. Rules that share an id are judged by the most + severe of them, so a failing rule cannot hide behind a `warn` one of the same id. A column + whose `sensitivity` becomes or stops being `pii` or `phi` is `column_pii_added` / + `column_pii_removed` (info), and any other change to it is the new + `column_sensitivity_changed` (info), each with the sensitivity values as before and after. A + pre-0.7 `pii: true` rewritten as `sensitivity: pii` is not a change. A column the dbt engine + builds as a key (a `primary-key` tag, a `semanticType` of `identifier` or `primary_key`, or a + `labels.constraint` of `primary_key` or `unique`) that stops being `required` is + `primary_key_nullable` (breaking) rather than a non-breaking loosening. + +- **`fluid diff --baseline --format json` could print JSON that does not parse.** Rich wrapped + each long line at the terminal width (80 columns in CI), putting a line break inside a JSON + string, and read `[...]` in a column name as markup. JSON and Markdown are now printed + verbatim, and text without markup or `:emoji:` codes, so a column named `[/]` no longer + crashes the text output. Credential-shaped values are redacted in each value before the + document is written, by both the logging redactor and the console's, so a name ending in + `token=x` no longer cuts its JSON string short. A value JSON cannot hold, such as a date that + YAML put in a column's quality rule, is written as text: `json.dump` raised on it after the + text report had printed, and left `--out` cut short. `--out` now holds the same redacted + envelope as stdout; it held the raw values. + +- **The `fluid forge --refine` preview read no columns from a `schema: {fields: [...]}` + expose.** The diff now unwraps it at `schema` and `contract.schema` alike, as `fluid verify` + does, and reports its columns at `...schema.fields.`. No bundled schema accepts the + wrapper, so `fluid diff --baseline` refuses such a contract as invalid (see Security). - **A failed `fluid apply` on OpenTofu printed raw JSON and no way forward.** The runner uses `-json`, so the error was the last 800 characters of diagnostic lines. It now reads the @@ -167,6 +348,58 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `pip install fluid-build[...]`; they now name `data-product-forge` with the right extra, and a ratchet test keeps the wrong name out. `data-product-forge[mcp]` resolves as an empty alias. `fluid-build` on PyPI is now only an alias that installs `data-product-forge`. +- **Two applies on the same S3 state could race.** The S3 backend was emitted without locking, so a + scheduled apply and a manual one (or an apply and a teardown) wrote the same state unguarded. + `tofu init` now gets `-backend-config=use_lockfile=true` for an S3 backend on OpenTofu 1.10 or + newer; the emitted module is unchanged. On an older OpenTofu a WARNING says the state is + unlocked. Upgrade note: `tofu plan` takes the lock too, so every role that runs plan, + `fluid apply --dry-run`, `fluid diff` or `fluid verify --state-drift` against S3 state needs + `s3:GetObject`, `s3:PutObject` and `s3:DeleteObject` on `.tflock`, or sets + `FLUID_TOFU_S3_LOCKFILE=0`. A workdir kept from before the upgrade is re-initialised with + `-reconfigure` once, only when it recorded this same bucket, key and region. The opt-out also takes effect on a + workdir that already had locking on. A run killed hard can leave `.tflock` behind; + `tofu force-unlock ` in the workdir (the ID is in tofu's error) clears it. A run that + hits `FLUID_TOFU_TIMEOUT_SECONDS` is now interrupted (SIGINT) so tofu releases the lock, and + killed only if it ignores that for 30 seconds. +- **A column type AWS Glue or BigQuery could not hold was written anyway, and `fluid validate + --strict` passed.** The Glue emitter mapped 22 type names and turned every other one into + `string`: 47 of the 79 types the schema admits, among them `smallint`, `tinyint`, `int2`, + `int4`, `int8`, `real`, `money`, `bit`, `year`, `datetime2`, `timestamptz` and the other + timestamp spellings, `varbinary` and `blob`, and every array, struct, map, object, record and + row. The BigQuery emitter wrote `ARRAY` for an array, which is not a BigQuery column type (an + array is a REPEATED column of an element type), and a `STRUCT` with no fields, which BigQuery + rejects at apply, after the dataset was created; the element type of `array(string)` was + dropped on both clouds. Every type is now classified for each cloud. Each scalar with a Hive or + Athena type gets it (`smallint`, `int`, `bigint`, `float`, `double`, `timestamp`, `binary`, + `money` as `decimal(19,4)`, `bit` as `boolean`, `year` as `smallint`; the zone-aware + timestamps are `timestamp`, which keeps the UTC instant and not the offset; a bare `number` is + `decimal(38,9)`, as a bare `decimal` always was). A type Glue has no form for but that keeps + its value as text (`json`, `jsonb`, `variant`, `super`, `time`, `interval`, `bignumeric`) is + still written as `string`, and `fluid validate` now warns for it, so `--strict` fails; BigQuery + warns for `geometry` and `geom`, which are planar and become the spherical `GEOGRAPHY`, and + for a `decimal(p,s)` wider than a BigQuery NUMERIC holds (29 integer and 9 fractional digits; + declare `bignumeric`). A + type with no faithful form at all is refused with kind `column-type`: an array, struct, map, + object, record or row (also `array(string)`), a `bit(n)` wider than one bit and a type no + table maps, on both clouds, and on Glue also `geography`, `geometry`, `geom`, `point` and + `hll`, and a decimal Hive would reject (precision over 38 or 0, a scale over the precision + or below 0). The + refusal names the expose, each column, its declared type and why, and the fix (declare `json` + or `string` for now; typed nested columns are planned). `fluid validate`, `fluid generate iac` + and `fluid apply` raise the same refusal from the same function, so a contract that passes + validate cannot fail generate on a type. A BigQuery view carries no schema and is not checked. + `fluid verify` and `fluid diff` compare the live table with the new mapping, so a Glue table + applied by an earlier version reports type mismatches (verify: CRITICAL; diff: drift) on those + columns until `fluid apply` runs; verify's "require table recreation" wording is wrong for + these columns, a re-apply updates them in place. Changing a Glue column's type in place is + held by the data-loss gate: `fluid apply` (and its dry run) refuses, naming each table and + `column: old -> new`, until `--allow-data-loss`, because Parquet written under the old type + (Athena INSERT or CTAS, dbt) may stop reading until it is rewritten, and reverting the type + restores reads; a difference in spacing alone (`decimal(10, 2)`) is no change. The 0.16.2 + note that only a bare `array` has no BigQuery form is superseded: an array of any element type and every + struct-like type are refused until the contract schema can declare element and field types. + BigQuery decimal precision and scale are still not emitted. + - **A contract's governance labels were dropped silently on Databricks.** The Databricks emitter never read `labels`, so `data-classification: confidential`, the `gdpr-article`, and the sovereignty jurisdiction, regulation and residency reached AWS (tags, Glue parameters) and GCP @@ -216,6 +449,93 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 dimension that reads the workspace's metastore summary for every table the module manages: it passes when the region matches the declaration, fails with the fix when it does not, and is skipped when the contract declares none. +- **An AWS contract's schedule was dropped without a word, and the one it got was not labelled.** + The workflow Lambda was named `fluid-workflow-` raw, so a dotted id (the schema's own + example is `gold.hr.employee_360_v1`, and 27 of 28 examples use dots) or one over about 49 + characters failed the plan validator, and `native_actions` swallowed the refusal at DEBUG: + `fluid generate iac` and `fluid apply` logged "Planned EventBridge schedule", wrote the Glue and + S3 resources and no `aws_lambda_function`, `aws_lambda_permission` or `aws_scheduler_schedule`, + exit 0. Every name the schedule planner derives from an id (Lambda functions, permission + statement ids, the schedule, event rules, the state machine, the MWAA environment) now goes + through one helper that replaces what the service refuses with `-` and, when it had to change + the name at all (a replacement, or a cut to the service limit), ends it in a hash of the whole + id, so `a.b` and `a-b` cannot share one Lambda and two long ids stay apart; a name the service + already accepts is left as written, so nothing deployed is renamed. A plan its own validator + rejects, an orchestration task the planner rejects and a schedule-planner failure are a typed + `PlanValidationError`, matched by type through the cause chain like a sovereignty refusal: + `generate iac` exits non-zero and `apply` raises, where a planner that could not run (no SDK, + no credentials) stays best-effort. This holds for every AWS plan-validator refusal, so a + contract with no schedule that the validator refuses (a bad Glue name, say) now fails + `generate iac` and `apply` too, where it used to emit the module without a word (`fluid plan` + already refused it). The scheduled Lambda only logs each + task and runs no build, and both roles it names (`fluid-workflow-execution`, + `EventBridgeSchedulerRole`) are never created, so every plan that emits it now warns + `schedule_placeholder_lambda`. When no AWS account can be resolved (no `AWS_ACCOUNT_ID`, no + credentials for `sts:GetCallerIdentity`), a contract that declares orchestration (a schedule, + triggers or tasks) is refused by name, before any name is derived from it, rather than as a Glue + or S3 name `None-fluid-staging` the validator would reject, and the message says so when the + cause is a missing boto3 (`pip install 'data-product-forge[aws]'`) rather than credentials. A + contract without orchestration needs no native plan and applies its exposes as before; the + sovereignty check still runs first either way. +- **The AWS plan validator refused dotted S3 bucket names.** S3 allows lowercase letters, digits, + dots and hyphens (3-63 characters, a letter or digit at each end, no `..`, `.-` or `-.`, not + shaped like an IP address); the validator allowed no dot, which was harmless while its refusals + were swallowed and is not now that they stop `generate iac` and `apply`. +- **A contract value could run as code in the AWS workflow Lambda.** The schedule planner spliced + task params and the contract id into the Lambda's Python source inside quote marks, so a task + param such as `'''+__import__('os').system('id')+'''` (the schema accepts any task params) + closed the literal and ran on every schedule tick under `fluid-workflow-execution`, and an id + with a trailing newline left a Lambda that was a SyntaxError. Every value now reaches the + source as a `repr()` literal. +- **The same AWS contract could emit a different module from one run to the next.** The AWS + planner ordered its actions through a `set`, so the order changed with Python's hash seed, and + with it the emitted module: a Lambda permission, schedule target, event-source mapping or S3 + notification named the function by literal ARN in one run and by + `${aws_lambda_function..arn}` in the next (the reference only when the function happened + to be planned first), so tofu sometimes had no ordering edge to the function. Actions now keep + the order they were planned in. Measured on 21 AWS contracts: modules without a Lambda are + byte-identical to before; scheduled ones now always emit the reference. + +### Removed + +- **`redshift` and `datamesh-manager` are no longer providers.** Neither deployed anything as a + provider: `redshift` only forwarded to `aws`, and Data Mesh Manager is a catalog. `fluid providers` now lists `aws`, `databricks`, `gcp`, `local` and `snowflake`, plus any + third-party plugin. Redshift objects apply through the `aws` provider, and Data Mesh Manager + publishing is unchanged (`fluid publish --target datamesh-manager`, `fluid datamesh-manager`). + **Breaking (CLI surface):** `--provider redshift` and `--provider datamesh-manager` (and + `FLUID_PROVIDER`) are no longer accepted; they exit 2 with the replacement + (`--provider aws`, or `fluid publish --target datamesh-manager`) instead of a bare "unknown + provider". A contract's `redshift` build runtime or expose platform is served by `aws`, and plugins + cannot claim either retired name. + +- **A failed fail-safe no longer locks the run out, and says what it could not close.** When a + mask or row filter cannot be bound, the lock that revokes SELECT from every grantee but the + owner now also skips the identity the run is (it has to repair the table), and when the + table's owner could not be read it says the owner could not be skipped. An owner the contract + itself restricts reads its table whatever is revoked: the message now names that path as still + open, with the `ALTER TABLE ... OWNER TO` to run. When the permission listing shows no + `inherited_from_*` field at all, it says it could not determine whether a schema or catalog + grant is open, where it claimed none was. + +- **A job paused for an unprotected table is announced and found.** The apply decides once, in + `emit`, which dbt jobs to create paused, and its notice reads that decision, so a transient + read error can no longer pause a job without saying so. `fluid verify` now fails the jobs + dimension for a job that is paused while its table's masks or row filter are not in place, + with the apply that binds them and starts the job as the fix. + +- **A changed contract no longer leaves its old masks and row filter functions behind.** After + a successful apply (never in a dry run, never after a failed apply or a failed bind), the + functions in the product schema that carry the table's `fluid_rf_
_`, + `fluid_mask_
__` or deny-all name and are neither expected now nor bound to any + table in the schema (every table is read first; if one cannot be, nothing is dropped) are + dropped, each printed, and only when the run's own identity owns the function: one with + another owner, or an owner that cannot be read, is left and named. A function without the + `fluid_` prefix is never touched. + +- **A plan that fails the residency check prints one line.** The apply showed the last 800 + characters of the tofu JSON, which cut off the sentence naming the live region. It now says + the declared region, the live region (read from the diagnostic's message, or its `values`) + and the fix. ### Security @@ -226,6 +546,88 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 into `main.tf.json`. The module's allowlists now end in `\Z`, so such a name is refused with `Invalid SQL identifier: 'spectrum_events\n'`. That includes a name that reaches it with a trailing newline from an environment variable or a file; strip the value where it is set. +### Upgrade notes + +- A contract with a column typed `array`, `struct`, `map`, `object`, `record` or `row` (or + `array(...)`) on an `aws` Glue table or a `gcp` BigQuery table now fails `fluid validate`, + `fluid generate iac` and `fluid apply` with `column-type`, where it validated and wrote a + `string` column (Glue) or an invalid one (BigQuery). Declare the column `json` (BigQuery's + JSON type; Glue stores it as `string` and validate warns) or `string` until typed nested + columns are supported. +- Other new refusals (kind `column-type`): on Glue, `geography`, `geometry`, `geom`, `point` and + `hll` (previously a silent `string`; declare `string` or `binary`) and a decimal Hive rejects + (precision over 38 or 0, scale over the precision or below 0); on both clouds, `bit(n)` with + n over 1 and any type no table maps. +- New validate warnings, so `--strict` fails: on Glue, `json`, `jsonb`, `variant`, `super`, + `time`, `interval` and `bignumeric` (written as `string`, as before; declare `string` to say + so); on BigQuery, `geometry` and `geom` (planar, written as the spherical `GEOGRAPHY`) and a + `decimal(p,s)` wider than a NUMERIC holds (29 integer or 9 fractional digits; declare + `bignumeric`). +- A Glue table's `smallint`, `tinyint`, `real`, `money`, `timestamptz` and the other types that + used to become `string` now get their Hive type, which is a column type change on a table an + earlier version applied. `fluid apply` refuses it, and so does its dry run, until + `--allow-data-loss`: the refusal names each table and `column: old -> new`. Parquet files + written under the old `string` schema (by Athena INSERT or CTAS, or dbt) hold strings, so + after `string` -> `smallint` Athena fails every read of them until they are rewritten (files + forge's own DuckDB build wrote are fixed by the change). Reverting the column type in the + contract restores reads, and `--allow-data-loss` proceeds (measured against moto with real + `tofu`: refused without the flag, `~1` applied with it). A difference in spacing alone, such as + `decimal(10, 2)` and `decimal(10,2)`, never trips the gate, and neither does an added or + removed column, which the gate did not cover before. Consumers that compare these columns as + strings (`small = '5'`) may need changes. Until the apply, `fluid verify` and `fluid diff` + report the mismatch. With the plan's change list unreadable, an AWS apply that updates + anything in place now also needs the flag, as a Databricks apply already did. +- `decimal(10, 2)` is now written `decimal(10,2)`. Against moto, that alone applies without the + flag; real Glue was not measured. + +- **Other validators let a value with a trailing newline through, too.** The same `$` ended other + whole-string patterns, so a value written as a YAML `|` block, read from a `.env` file or decoded + from `%0A` in a URL kept its newline: `fluid generate iac` put `region = "eu-west-1\n"` on the + AWS provider, and a Databricks job got `branch = "main\n"` from `?branch=main%0A`. They now end + in `\Z` or are matched with `fullmatch`, so such a value is refused with the validator's usual + message, or takes the path the code already has for a value that is not a plain name: a product + id forged from a JDBC source or imported from dbt or ODCS is slugified. An AWS binding region + with whitespace around it is refused (`aws-region-whitespace`), because the sovereignty policy + checks the region as written. Covered: quality-check selectors, catalog and JDBC identifiers, + BigQuery and GCP project ids, contract, build, federation-workspace and overlay-env ids, + row-filter names, Singer tap and target names, prompt profile and overlay names, mission + predicate paths, AWS and GCP resource names, Snowflake account segments, Databricks git refs, + and the identifier and version checks of ODCS and ODPS imports. + +- **Schema validation let a value with a trailing newline through a `pattern`.** JSON Schema + writes `pattern` in ECMA-262, where `$` matches only at the end of the value, but the + jsonschema library runs it with Python's `re.search`, whose `$` also matches before a final + newline. So `fluid validate` accepted `id: "orders\n"`, which an `id: |` block scalar produces, + under every bundled schema version (0.7.1 to 0.7.7). Contract, fragment and bundle validation + now read `$` as the end of the value. Such a value is refused at its path, with a hint when + the line break is the only problem: `(it ends in a line break, which a YAML | or > block keeps + and |- or >- drops)`. A `sink.catalogAuth` field gets no hint, since its value is never shown. + The schemas themselves are unchanged. + +- **`fluid diff --baseline` compared contracts that no schema accepts.** Its rules read only the + shapes a schema allows, so a value of the wrong type hid a change: a column retyped from + `decimal(18,2)` to the number `18` passed `--fail-on-breaking` with exit 0. Both contracts are + now validated first, against the schema their `fluidVersion` names, offline, the gate + `fluid plan` runs. An invalid one exits 2 with `diff_contract_invalid`, which names each + invalid file and logs each error, and nothing is printed or written to `--out`. Exit 2 is + diff(1)'s "trouble", so it never reads as 0 (no changes) or 1 (breaking). A pre-0.7 contract is + refused with the upgrade that `fluid validate` asks for, so a pre-0.7 baseline can no longer be + diffed against its 0.7.x rewrite. In `fluid_build.api.changelog`, `compare_contracts` raises the + new `ContractValidationError`, and `validate=False` diffs without validating; the + `fluid forge --refine` preview uses it, because the contract being refined may be the invalid one. + +- **`fluid diff --baseline` printed contract names raw.** A column or expose name could drive the + terminal (an escape sequence, a hyperlink to another URL, a carriage return or line feed that + forges a line, a bidi control that reorders it), and in `--format markdown` it could add a link, + an image or an HTML comment that hides the rest of a PR comment, or mention someone. Text output + now spells each control, bidi and invisible character as `\x1b` or `\u202e` and doubles a + backslash. Markdown puts the path and the description each in one code span: a backslash escape + is not enough there, because GitHub still links `\@user`, `\#1` and `\:smile\:` after + rendering. The fence is one backtick longer than any run of backticks in the name, and the + document is redacted per value before it is written, so a redaction cannot take the closing + fence with it. A Markdown entry now reads + ``- **column_added** `exposes[0].contract.schema.email` — `column 'email' added to expose 'orders'` ``. + The `fluid forge --refine` preview prints the same text without markup. ## [0.26.0] — 2026-11-01 diff --git a/CLAUDE.md b/CLAUDE.md index 3ec3c630e..551f490d7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -118,7 +118,7 @@ The `feat/cli-pipeline-and-publish-hardening` branch (15 commits) landed the **1 A multi-phase push landed comprehensive UX + architecture upgrades on `feat/source-aligned-acquisition`. Key surfaces and where they live: - **Mode picker (`fluid forge`).** Bare invocation surfaces a 5-mode menu (AI / Compose / Refine / Template / Blank) instead of dropping into AI. Lives at `cli/_forge_mode_picker.py`. Skipped via `FLUID_FORGE_NO_PICKER=1`. Picker default is highlighted by the welcome scan; user always sees the menu. -- **Welcome scan.** Detect-first parallel scan runs in <50ms before any prompt — workspace state, AI configured, CLIs installed, sample data, cloud creds, return-user state. Lives at `cli/_welcome_scan.py`. `~/.fluid/usage.json` carries `forge_count`. +- **Welcome scan.** Detect-first parallel scan runs before any prompt (budget: see the 2026-11-03 section) — workspace state, AI configured, CLIs installed, sample data, cloud creds, return-user state. Lives at `cli/_welcome_scan.py`. `~/.fluid/usage.json` carries `forge_count`. - **Mode-aware interviews.** `forge_copilot_interview.py::run_adaptive_copilot_interview` short-circuits to `_run_compose_interview` (3 questions max for `--from-product`) or `_run_refine_interview` (1 question for `--refine`) when the runtime resolved upstreams or loaded an existing contract. Standard mode runs the world-class bootstrap (below). - **World-class bootstrap.** New default fresh-product interview at `cli/_world_class_interview.py`. Inferences first (`InterviewSignals`), examples in every prompt, productType-first, `:auto` escape, schema-coverage gate. Toggle via `FLUID_INTERVIEW_LEGACY=1` to revert to the legacy bootstrap. - **Slash commands inside the interview.** `cli/_interview_slash_commands.py` — `:ai-setup`, `:override`, `:show-work`, `:doctor`, `:help`, `:quit`. Wrapped into `forge_dialogs.ask_friendly_text` so every prompt accepts them. @@ -416,6 +416,15 @@ would replace; when building it, **depend on the official `agent-client-protocol PyPI SDK** (Pydantic models + async base classes + stdio JSON-RPC) rather than hand-rolling the protocol. +## Recent architectural changes worth knowing (2026-11-03, welcome-scan budget) + +`cli/_welcome_scan.py::run_welcome_scan` now has a budget that means what it says: + +- **One wall-clock cap.** `_TOTAL_TIMEOUT_S` (1.0 s) is the only scan budget; the old per-probe `_PROBE_TIMEOUT_S` is gone. Probes run on daemon threads (`_spawn_probe`: one `Future` each, deliberately not a `ThreadPoolExecutor`, whose non-daemon workers are joined at interpreter exit and would hold the CLI open for an abandoned probe). The scan returns at the deadline and drops stragglers. Python cannot kill a thread, so an abandoned probe keeps running in the background until it finishes; the one that can run long is `_probe_specialization_suggestion` (unbounded `rglob`), and the picker, the interview signals and the welcome panel each start a scan, so up to three walkers can overlap on a huge cwd. Probes must not write to stdout/stderr (a daemon thread caught mid-write at interpreter exit can abort the process). +- **Cold imports are off the clock.** `_warm_probe_imports` imports `_WARM_MODULES` (the AI-config pydantic chain, the dbt runner) from the main thread while the probes run, then the budget starts. Probe imports stay function-local: if a probe starts importing something new, list it in `_WARM_MODULES` or `tests/test_welcome_scan_budget.py::test_probes_import_nothing_new_after_the_warm_up` fails. NB `build_parser()` already imports `ai_setup`, the dbt runner and pydantic, so in a real CLI process only `copilot.unified_config` is still cold; the warm-up matters for tests and library callers. +- **dbt has its own cap.** `_DBT_VERSION_TIMEOUT_S` (0.32 s) sizes the `dbt --version` subprocess and is not derived from the scan budget and must stay below it, so the child is killed by its own timeout instead of being orphaned. With Python dbt-core on PATH it usually waits out that cap, which is the scan's floor (~0.35 s). +- `scan_duration_ms` is the real wall-clock of the call, imports included. Pins: `tests/test_welcome_scan_budget.py`. + ## Working style - **Goal-driven execution.** For non-trivial tasks, state success criteria before implementing and loop until verified. Transform imperatives into tests: "add redactor coverage" → "new pattern is masked by a test assertion"; "fix the bug" → "reproducing test passes first, then fix makes it green". diff --git a/Dockerfile b/Dockerfile index c72ab07e9..e1992b077 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,16 +55,19 @@ COPY fluid_build/ fluid_build/ # Install the package (non-editable) RUN pip install --prefix=/install . -# Install provider dependencies based on profile +# Install provider dependencies based on profile. Every profile names +# [standard] (litellm + mcp) and [lineage], so the image keeps AI, +# `fluid mcp serve` and OpenLineage emission once they leave the base install +# in 0.28.0. RUN if [ "${PROFILE}" = "experimental" ] || [ "${PROFILE}" = "alpha" ]; then \ pip install --prefix=/install \ - ".[local,gcp,snowflake,viz]"; \ + ".[local,gcp,snowflake,viz,standard,lineage]"; \ elif [ "${PROFILE}" = "beta" ]; then \ pip install --prefix=/install \ - ".[local,gcp,viz]"; \ + ".[local,gcp,viz,standard,lineage]"; \ else \ pip install --prefix=/install \ - ".[local]"; \ + ".[local,standard,lineage]"; \ fi # ============================================ diff --git a/HONESTLY_TESTED.md b/HONESTLY_TESTED.md index def4a5a6b..a3ba82d76 100644 --- a/HONESTLY_TESTED.md +++ b/HONESTLY_TESTED.md @@ -85,12 +85,15 @@ Markers declared in `pyproject.toml`. | Stage 1 unit | `tests/iac/test_iac_databricks.py`, `test_iac_databricks_jobs.py`, `test_iac_databricks_auth.py`, `test_iac_databricks_engine_phase2.py`, `tests/test_util_cron.py`, `tests/build_runners/test_databricks_job_builds.py`, `tests/engines/test_dbt_databricks.py`, `tests/cli/test_verify_databricks.py`, `tests/test_doctor_databricks.py`, `tests/test_welcome_scan_databricks.py` | ✅ 521 tests in the Databricks files above (139 tables, 149 jobs, 56 credentials, 47 engine hooks, 45 cron, 31 runner, 40 verify, 14 doctor and welcome scan), plus the dbt profile and type tests. The emitted module: a `databricks_catalog` data source, the owned schema, a managed Delta table, per-principal grants with best-effort imports, the provider's `retry_timeout_seconds`. A declared `binding.location.region` (validated, lower-cased; `databricks-region-invalid` and `databricks-region-conflict` refused) adds `data.databricks_current_metastore` and a precondition on each owned schema; with none the module is unchanged. A contract's governance labels (`iac/governance_labels.py`, the AWS and GCP source) are one `databricks_entity_tag_assignment` per tag on each owned schema (the contract's own) and each table (the expose's too), `-` read as `_` in a key; two labels meeting at one key are `databricks-tag-collision`, more than 50 tags on a securable `databricks-tags-too-many`; a contract without labels gets the module it always did; each assignment has a best-effort import (`entity_type,entity_name,tag_key`); removing one is not data loss. Databricks is a region-placed platform for the `sovereignty` checks (`tests/policy/test_sovereignty_enforcement_mode.py`). Each dbt build on Databricks becomes a serverless job with its workspace files, mapped from the build: Quartz cron, triggers, retries, notifications, timeout and run-as. The project upload's allowlist and refusals; the escaped file names. Every refusal `fluid validate` and the emitter share. The credential policy (host, profile, auth type, blanked variables) and the SDK kwargs. The in-place data-loss and protocol-upgrade readings of a saved plan. Job and file removals left ungated; the `USE CATALOG` notice; the drift report for dbt-written tables. `fluid verify`'s Databricks inspector against a stubbed client (residency: pass, mismatch, skipped; governance tags: pass, a missing tag, a wrong value, the paginated list read to its last page with an empty page in the middle, skipped); the doctor rows and welcome-scan detection; the dbt runner leaving a Databricks job build to its job. | What Databricks accepts (the manual live row). The real databricks-sdk is neither in the dev install nor in CI, so a stubbed client stands in for it; one test answers as the live workspace did, with the privilege under the group that conveys it. | | Stage 1 `tofu validate` | `tests/iac/test_iac_tofu_validate.py` `[databricks]`, `test_a_databricks_dbt_job_passes_tofu_validate[git\|workspace]` | ✅ Both job modules (GIT with schedule, retries, notifications and run-as; WORKSPACE with a continuous trigger and files) and the dbt-written table's `ignore_changes` list validate against provider 1.140.0's schema. | Anything the provider checks only at plan or apply. | | Stage 1 real tofu, stand-in workspace: tables | `tests/iac/test_iac_databricks_fake_e2e.py` (`tests/iac/_fake_databricks.py`) | ✅ 28. The real `fluid` CLI, `tofu` and databricks/databricks 1.140.0 against an in-process stand-in for the Unity Catalog, SQL statement and permissions APIs. validate / plan / `generate iac --validate` / dry-run route to OpenTofu, and a second plan is empty. A partition change (a replace), a dropped column and a positional rename are refused without `--allow-data-loss`; a grant revoke is not. A column reorder is refused even with the flag. An added column applies in place. A table that exists with column mapping is adopted, not replaced; without it the apply is refused (with or without the flag) until its owner runs the `ALTER`, and the refused run, like a dry-run, leaves nothing it imported in state. Existing grants are adopted with their table: the adoption plan is `+0 ~1 -0` (the provider imports neither the table's properties nor its warehouse id, so the first apply writes back the values it already has), and the plan after it is empty. A privilege the contract does not name shows as an update that removes it. A declared `location.region` plans and applies when the stand-in's metastore is in it (case-insensitively) and fails the plan, naming both regions and `DATABRICKS_HOST`, before anything is written when it is not. A type change and a missing catalog fail at plan or import, and a missing host or credential fails before `tofu` starts; each names the fix before any change. `fluid diff` initialises with the committed lock. A contract's governance labels are created as tag assignments on the schema and the table in the stand-in and a second plan is empty; a changed value updates the tag in place; a removed label destroys only its assignment, without `--allow-data-loss`; an assignment already in the workspace (state lost, or set by hand with another value) is adopted and its value corrected, never a failed create, and a tag someone else set is left alone. | The stand-in models the requests provider 1.140.0 sends: SQL dialect, warehouse state and privilege names are not checked. Its metastore region is whatever the test sets; whether a real workspace's `metastore_summary` reports the region in the spelling a contract declares (`eu-central-1`, `westeurope`) is not proven: the residency check and the verify dimension were not run against a live workspace; the governance tags and their verify dimension were not run live either. The tag endpoints are modelled on the SDK's source (databricks-sdk-go v0.190.0, the one provider 1.140.0 builds on), not on a live workspace: the live tier was not run for tags, so what real Databricks accepts as a tag key or value, whether it keeps a table's tags across a dbt CREATE OR REPLACE (undocumented), and whether the import id `entity_type,entity_name,tag_key` resolves a live assignment are unproven; `fluid verify`'s governance-tags dimension is the gate for the last. It stores the table properties it was sent, so the adoption check reads `properties` / `effective_properties` as a real workspace may not fill them. | -| Stage 1 unit: Databricks column masks and row filters | `tests/iac/test_iac_databricks_access.py` (the SQL), `tests/iac/test_iac_databricks_enforce.py` (the hooks and the engine's gate), `tests/cli/test_verify_databricks.py` (21 of its 61), `tests/iac/_databricks_governed.py`, `tests/iac/_fake_databricks.py` | ✅ 97 + 84 + 21. A Customer-Profile-360-shaped contract (six denied columns, a `msisdn` mask, a consent row filter, principals mapped to `fluid-demo-lab-*` groups) compiles to the exact `CREATE FUNCTION IF NOT EXISTS` / `ALTER TABLE ... SET ROW FILTER ... ON (...)` / `ALTER COLUMN ... SET MASK` statements. The function bodies are evaluated in DuckDB (transpiled from the Databricks dialect, `is_account_group_member` stubbed): a restricted group reads only the rows its predicate keeps, a member of several filtered principals reads the union, everyone else every row, a NULL predicate hides the row from a member; `last_four` / `first_four` / `nullify` and a deny give the BigQuery-matching values, a value of four characters or fewer is its SHA-256, a NULL stays NULL, a deny beats a mask, and a mask rule belongs to the principal that names it (nullify for one group and last_four for another on one column: strictest branch first, a member of both reads NULL). Every identity is tested as a group and as a user, so a `group:` principal mapped to a service principal's application ID or a user's email is still restricted (DuckDB's own session user stands in for a service principal that is in no group). Names carry a hash of the definition: the same contract names the same functions, a changed predicate or rule names a new one. Every refusal kind, 30 predicate injection attempts (`;`, comments, subqueries, session functions, unknown or qualified columns, a backslash-escaped quote) and a principal name with a quote or a backslash. The hooks against the stand-in (statements, Tables API reads): pre_apply binds existing tables and leaves a missing table to post_apply, a dry run prints and executes nothing, a second run executes nothing, a changed predicate binds a new function and keeps the old, a removal drops only what this module bound, a statement needing a column the apply adds waits for post_apply, a failed pre_apply stops the run and revokes nothing, a failed post_apply binds a deny-all row filter for the restricted identities (SELECT untouched), and when that cannot be bound, or would replace a filter not made here, revokes SELECT and ALL PRIVILEGES from every grantee but the owner, lists the schema and catalog grants still open with the statement to run, and reports a revoke that failed or a grant listing that failed; a function pre-created under any hashed name (row filter, mask, deny-all) by another owner, with another body, parameters or return type, is never bound; a row filter or mask set by hand is never replaced, with nothing changed on any table; a Tables API read that fails for one table (503, 429) is a failed bind for it and the next table is still tried; a filter needing a column the apply adds is held by the deny-all filter in pre_apply; a dbt job (cron or continuous) writing a table that does not exist, is unbound, has a wrong function or cannot be read is created PAUSED, and a manual job or a job writing no restricted table is left alone, after a `tofu apply` that failed part way, post_apply binds what exists, drops nothing, skips a table never created without a NOT REVOKED message, and the engine prints an enforcement failure there without raising it; the SQL statement poller (running, failed, cancelled, timed out, credentials scrubbed). The engine calls the hooks in order around `tofu apply`. `fluid generate iac` refuses a module with masks or filters (emit stays pure). `fluid verify`'s row-filter-and-masks dimension, including the owner, body, parameters and return type of every bound function (pass, no filter, another function, other input columns, missing and wrong mask, case and backticks ignored, skipped, the CLI output and `--strict`). | What Databricks accepts. The stand-in executes the statements as this module sends them and models the Unity Catalog refusal of `CREATE OR REPLACE FUNCTION` on a bound function, the preserved bindings across `CREATE OR REPLACE TABLE`, and the Tables API fields `row_filter.function_name` / `input_column_names` and `columns[].mask.function_name` (databricks-sdk-go v0.190.0), all from the docs and the SDK source, not from a live workspace. DuckDB evaluates the bodies, not Databricks: `sha2`, `right`, `left` and `is_account_group_member` are Databricks' own. | -| Stage 1 real tofu, stand-in workspace: masks and row filters | `tests/iac/test_iac_databricks_fake_e2e.py` (17 of its 45; needs databricks-sdk, the `[databricks]` extra, and skips without it) | ✅ 17. The real `fluid` CLI and `tofu` apply bind the filter and masks (after the apply, on a table it created); a second apply executes no statement; a table that already exists is bound before the apply's first grant on it; `--mode dry-run` prints the statements and executes none; a changed predicate binds a new function and keeps the old; a removed restriction is dropped and said to widen access without `--allow-data-loss`; a binding that fails revokes SELECT from the restricted principal only, fails the run (`opentofu_access_enforcement_failed`), and the next apply binds before it grants again; a failure before the apply stops it with nothing applied; an invalid predicate is refused before anything runs; an apply that fails after creating the table (the grant on it refused) still binds the filter and masks and the run exits with `opentofu_apply_failed`, an apply that fails before the table exists binds nothing and revokes nothing, and a failed apply drops no binding the contract no longer asks for (it is dropped by the next apply that succeeds); a function taken under the hashed name stops the apply before it runs; a row filter set by hand stops it and is left in place; a binding that fails with no deny-all possible revokes every grantee but the owner and the next apply binds first and grants again; a failed mask leaves the deny-all filter and every grant until the next apply; a filter needing a column the apply adds is held by the deny-all filter, then replaced, in that order around the added column; a dbt job created with its restricted table is paused and the next apply unpauses it in place; `generate iac` refuses the module. | Live behaviour: that `CREATE FUNCTION IF NOT EXISTS`, `SET ROW FILTER` and `SET MASK` run on a real warehouse as modelled, that a second `SET` replaces a binding already on the table (if it does not, a changed predicate fails closed with the old binding in place), that `current_user()` of a service principal is its application ID, that a Free Edition or classic warehouse can run them, and whether the SDK's statement API answers as the stand-in does; whether `SET ROW FILTER ... ON ()` (a zero-column function) is accepted; whether FunctionInfo's `routine_definition`, `input_params` and `full_data_type` come back in the form compared (whitespace-normalised, lower-case types), whose mismatch would refuse a good function; whether `scim/v2/Me` `userName` is the owner `CREATE FUNCTION` records for a service principal; that `SET` replaces a bound filter or mask (the stand-in does; none of the fixes uses a DROP first, which would leave a window with no filter); the permissions and effective-permissions fields used to lock a table; and the pause behaviour of a job on a live workspace. Unproven for ALL_PRIVILEGES grantees on a live workspace: a revoke of ALL PRIVILEGES removes more than read access, by design of the lock. A principal that is a metastore admin or the table's owner is never restricted by these bindings. | +| Stage 1 unit: Databricks column masks and row filters | `tests/iac/test_iac_databricks_access.py` (the SQL), `tests/iac/test_iac_databricks_enforce.py` (the hooks and the engine's gate), `tests/cli/test_verify_databricks.py` (21 of its 61), `tests/iac/_databricks_governed.py`, `tests/iac/_fake_databricks.py` | ✅ 97 + 101 + the verify dimension's. The job pause is decided once, in `emit`, and the notice reads that decision; `fluid verify` fails the jobs dimension for a job paused while its table's masks or row filter are not in place; a plan that fails the residency check prints one line with the declared region, the live region and the fix (`databricks-residency-region`). A Customer-Profile-360-shaped contract (six denied columns, a `msisdn` mask, a consent row filter, principals mapped to `fluid-demo-lab-*` groups) compiles to the exact `CREATE FUNCTION IF NOT EXISTS` / `ALTER TABLE ... SET ROW FILTER ... ON (...)` / `ALTER COLUMN ... SET MASK` statements. The function bodies are evaluated in DuckDB (transpiled from the Databricks dialect, `is_account_group_member` stubbed): a restricted group reads only the rows its predicate keeps, a member of several filtered principals reads the union, everyone else every row, a NULL predicate hides the row from a member; `last_four` / `first_four` / `nullify` and a deny give the BigQuery-matching values, a value of four characters or fewer is its SHA-256, a NULL stays NULL, a deny beats a mask, and a mask rule belongs to the principal that names it (nullify for one group and last_four for another on one column: strictest branch first, a member of both reads NULL). Every identity is tested as a group and as a user, so a `group:` principal mapped to a service principal's application ID or a user's email is still restricted (DuckDB's own session user stands in for a service principal that is in no group). Names carry a hash of the definition: the same contract names the same functions, a changed predicate or rule names a new one. Every refusal kind, 30 predicate injection attempts (`;`, comments, subqueries, session functions, unknown or qualified columns, a backslash-escaped quote) and a principal name with a quote or a backslash. The hooks against the stand-in (statements, Tables API reads): pre_apply binds existing tables and leaves a missing table to post_apply, a dry run prints and executes nothing, a second run executes nothing, a changed predicate binds a new function and, after an apply that succeeds, drops the old one (only when no table in the schema uses it and this run's identity owns it; never after a failed apply or bind, nor in a schema whose tables cannot all be read; a drop that fails is named and the apply still succeeds), a removal drops only what this module bound, a statement needing a column the apply adds waits for post_apply, a failed pre_apply stops the run and revokes nothing, a failed post_apply binds a deny-all row filter for the restricted identities (SELECT untouched), and when that cannot be bound, or would replace a filter not made here, revokes SELECT and ALL PRIVILEGES from every grantee but the owner and this run's identity (naming an owner it could not read, an owner the contract restricts as a path still open, and a permission listing that shows no `inherited_from_*` field as undetermined), lists the schema and catalog grants still open with the statement to run, and reports a revoke that failed or a grant listing that failed; a function pre-created under any hashed name (row filter, mask, deny-all) by another owner, with another body, parameters or return type, is never bound; a row filter or mask set by hand is never replaced, with nothing changed on any table; a Tables API read that fails for one table (503, 429) is a failed bind for it and the next table is still tried; a filter needing a column the apply adds is held by the deny-all filter in pre_apply; a dbt job (cron or continuous) writing a table that does not exist, is unbound, has a wrong function or cannot be read is created PAUSED, and a manual job or a job writing no restricted table is left alone, after a `tofu apply` that failed part way, post_apply binds what exists, drops nothing, skips a table never created without a NOT REVOKED message, and the engine prints an enforcement failure there without raising it; the SQL statement poller (running, failed, cancelled, timed out, credentials scrubbed). The engine calls the hooks in order around `tofu apply`. `fluid generate iac` refuses a module with masks or filters (emit stays pure). `fluid verify`'s row-filter-and-masks dimension, including the owner, body, parameters and return type of every bound function (pass, no filter, another function, other input columns, missing and wrong mask, case and backticks ignored, skipped, the CLI output and `--strict`). | What Databricks accepts. The stand-in executes the statements as this module sends them and models the Unity Catalog refusal of `CREATE OR REPLACE FUNCTION` on a bound function, the preserved bindings across `CREATE OR REPLACE TABLE`, and the Tables API fields `row_filter.function_name` / `input_column_names` and `columns[].mask.function_name` (databricks-sdk-go v0.190.0), all from the docs and the SDK source, not from a live workspace. DuckDB evaluates the bodies, not Databricks: `sha2`, `right`, `left` and `is_account_group_member` are Databricks' own. | +| Stage 1 real tofu, stand-in workspace: masks and row filters | `tests/iac/test_iac_databricks_fake_e2e.py` (17 of its 45; needs databricks-sdk, the `[databricks]` extra, and skips without it) | ✅ 17. The real `fluid` CLI and `tofu` apply bind the filter and masks (after the apply, on a table it created); a second apply executes no statement; a table that already exists is bound before the apply's first grant on it; `--mode dry-run` prints the statements and executes none; a changed predicate binds a new function and the apply drops the old one; a removed restriction is dropped and said to widen access without `--allow-data-loss`; a binding that fails revokes SELECT from the restricted principal only, fails the run (`opentofu_access_enforcement_failed`), and the next apply binds before it grants again; a failure before the apply stops it with nothing applied; an invalid predicate is refused before anything runs; an apply that fails after creating the table (the grant on it refused) still binds the filter and masks and the run exits with `opentofu_apply_failed`, an apply that fails before the table exists binds nothing and revokes nothing, and a failed apply drops no binding the contract no longer asks for (it is dropped by the next apply that succeeds); a function taken under the hashed name stops the apply before it runs; a row filter set by hand stops it and is left in place; a binding that fails with no deny-all possible revokes every grantee but the owner and this run's identity and the next apply binds first and grants again; a failed mask leaves the deny-all filter and every grant until the next apply; a filter needing a column the apply adds is held by the deny-all filter, then replaced, in that order around the added column; a dbt job created with its restricted table is paused and the next apply unpauses it in place; `generate iac` refuses the module. | Live behaviour: that `CREATE FUNCTION IF NOT EXISTS`, `SET ROW FILTER` and `SET MASK` run on a real warehouse as modelled, that a second `SET` replaces a binding already on the table (if it does not, a changed predicate fails closed with the old binding in place), that `current_user()` of a service principal is its application ID, that a Free Edition or classic warehouse can run them, and whether the SDK's statement API answers as the stand-in does; whether `SET ROW FILTER ... ON ()` (a zero-column function) is accepted; whether FunctionInfo's `routine_definition`, `input_params` and `full_data_type` come back in the form compared (whitespace-normalised, lower-case types), whose mismatch would refuse a good function; whether `scim/v2/Me` `userName` is the owner `CREATE FUNCTION` records for a service principal; that `SET` replaces a bound filter or mask (the stand-in does; none of the fixes uses a DROP first, which would leave a window with no filter); the permissions and effective-permissions fields used to lock a table; and the pause behaviour of a job on a live workspace. Unproven for ALL_PRIVILEGES grantees on a live workspace: a revoke of ALL PRIVILEGES removes more than read access, by design of the lock. A principal that is a metastore admin or the table's owner is never restricted by these bindings. | +| Stage 1: Databricks bronze landing and masking at landing (live not run) | `tests/build_runners/test_duckdb_databricks_load.py` (the runner and the loader), `tests/iac/test_iac_databricks_masking_at_landing.py` (the emitter), `tests/iac/test_iac_databricks_landing_fake_e2e.py` (real `fluid` + `tofu` + databricks-sdk; skips without the `[databricks]` extra), `tests/iac/_fake_databricks.py` | ✅ A `platform: databricks` binding no longer writes a local file: the run stages Parquet under `.fluid/staging`, uploads it to `DATABRICKS_STAGING_VOLUME` through the Files API, runs one `INSERT OVERWRITE ... read_files` over the one file it uploaded (a file someone else puts in the run's directory is not loaded) and deletes the staged files (also after a failed upload, a failed or short load and a Ctrl-C, which cancels the statement first; one that cannot be deleted fails the run naming it first); each load first deletes an earlier run's staging of the same build once all of it is over an hour old, and leaves a recent one. The statement's text, quoting and column list are pinned, as are the volume's validation, the staging path (no contract id climbs out), the refusals before any work (unset volume or warehouse, mode, sink, stream count), a missing table (names `fluid apply`, uploads nothing), credential scrubbing and the matrix of when the emitter accepts `policy.privacy.masking` (every writer a DuckDB acquisition build) against when it refuses it, through both `validate` and `emit`. End to end against the stand-in: `fluid apply` creates the table with its mask and row filter, `--mode amend-and-build` loads the rows with the masked column hashed and the cleartext absent, the volume is empty, nothing but the load runs on the warehouse, the table is not replaced, a second plan is empty, and a second run overwrites the rows. | **Not run live.** The stand-in is not Databricks: it reads the uploaded Parquet with DuckDB, so what `read_files` accepts, that `INSERT OVERWRITE` keeps a managed table's row filter and column masks, implicit casts from the Parquet types to the table's, the Files API's limits and a real warehouse's permissions are unproven. The run-time schema gate reads a declared `string` as a type change against DuckDB's `VARCHAR`, so a contract declares `varchar` for a text column. | | Stage 1 real tofu, stand-in workspace: dbt jobs | `tests/iac/test_iac_databricks_jobs_fake_e2e.py` | ✅ 6. A local dbt build applies as one serverless job plus the workspace files dbt reads, each in a folder created before it (no upload meets a missing parent) (never `profiles.yml`, `target/` or a hidden file), and a second plan is empty. A changed model updates its file in place. A GIT build runs its tag on a continuous trigger and uploads nothing. Turning a job build local removes the job, its files and their folders without `--allow-data-loss`, and says so; a folder holding a file the apply did not put there is not deleted. The table a dbt build writes keeps what dbt's CREATE OR REPLACE made of it, and the apply reports the column drift instead of failing on a type change. | Whether Databricks accepts and runs the job: serverless environment version 2, the `dbt-databricks>=1.8.0,<2.0.0` range, the commands, a WORKSPACE `project_directory` under `/Workspace`, `run_as` permissions, a Git credential for a private repository, `dbt deps` with a local package (not uploaded). The manual live row proves the first four. Whether the jobs API echoes a server default the stand-in does not (the fake's builder found `performance_target` and an all-false `notification_settings` would each plan an update), which would make a second plan non-empty (live, it was empty). What dbt's CREATE OR REPLACE does to a table with column mapping (the module sets none on a dbt-written table). Not bound by a plan: an apply uploads the dbt project as it is when it runs (`planDigest` and `bundleDigest` cover the contract, not the project's files, as for a local dbt run), and a bundle does not carry the project. Known gap, by design: a job is not adopted. Its import id is a job id, which neither the contract nor `tofu` can look up by name, so an apply that lost its state creates a second job. `fluid verify` flags the duplicate (a stubbed client proves it). Workspace files need no import: the provider's create overwrites. | +| Stage 1 unit: Databricks retention (`lifecycle.expire`) | `tests/iac/test_iac_databricks_retention.py`, `tests/cli/test_verify_databricks_retention.py` | ✅ 195 + 18. ISO 8601 durations (years and months at their shortest, P1Y = 365 and P1M = 28 days; a fraction, a sign, a lower-case letter, `PT`, `P1DT`, out-of-order and over-long numbers refused), Delta `interval` properties (rounded up, months refused) and the history as the larger of the two properties with Delta's defaults. A dbt model's `materialized` resolved from `config()` in the SQL, the model's YAML `config:` and `dbt_project.yml` (the `+` prefix or none, root, project, folder and model-name levels, the most specific wins, SQL over YAML over project), the default `view`, the model found by alias or file name, commented-out and `raw` configs ignored, a `config()` inside a Jinja block, `var()`, `**`, a custom schema or database, a Python model, a versioned or twin model, a linked file and a Git repository refused as undeterminable. The build's `--select` must include the resolved model (name, `fqn`, `path:`, `*`, unions and intersections; a graph operator on another node, a tag, a config or run state is refused as not evaluable). Accepted: a dbt `table` model and a DuckDB acquisition with P30D, P365D, P6Y; refused: P7D, P29D, P4W, P1M, a history set longer than the retention, `incremental`, `snapshot`, `view`, another acquisition engine, no writer, a mixed set of writers. `fluid validate` and the emitter agree on kind and message. `fluid verify`'s retention dimension against a stubbed client and, over HTTP, the stand-in workspace (pass, fail with the `ALTER TABLE`, the runtime property over the declared one, skip, predictive optimization off as a warning). | Live behaviour: no real Databricks endpoint was called. Whether Unity Catalog's GetTable serves a table's `delta.logRetentionDuration` and `delta.deletedFileRetentionDuration` in `properties` or `delta_runtime_properties_kvpairs` (read from databricks-sdk-go v0.190.0 `model.go`, not observed), whether `effective_predictive_optimization_flag` is filled for a managed table, that predictive optimization runs `VACUUM` within the retention (a Databricks guarantee, not tested), and that dbt-databricks `table` replaces the table each run. Delta history is only half the promise: the acquisition acceptance rests on the landing being an `INSERT OVERWRITE` of the whole table in the one mode it runs (`full_refresh`, read from the landing's own `LOAD_MODES`; another source mode is refused as `incremental`, none as `unmapped`), which the bronze landing row covers against the stand-in, not live. A GetTable answer with no properties is a verify warning (Delta's defaults assumed). Embedded-logic DuckDB builds, `dlt`, `airbyte` and the other acquisition engines are refused. Selection is matched on names and paths only, not on dbt's graph: `+other+` is refused even if `other` is upstream of the model. | | Manual live, Free Edition (founder-approved) | the real `fluid` CLI, no test file: run 2026-11-01 from main dfd6f145, databricks-sdk 0.158.0, every `fluid_iactest_*` object swept after (1 schema, 1 table, 1 folder; 0 left) | ✅ validate, `generate iac --validate`, plan, dry-run and apply `+9`; a second plan is EMPTY against the real Jobs API; Databricks accepts the Quartz schedule, serverless environment 2 with `dbt-databricks>=1.8.0,<2.0.0` (dbt 1.12.6, adapter 1.13.0), the WORKSPACE `project_directory` and the commands as emitted; one `run-now` ends SUCCESS and dbt builds the table; `fluid verify` with the real SDK (columns, types, one job); `fluid diff`'s state check (its live check has no Databricks inspector); a model change `~1`; the build made local removes job, files and folders `-5` without `--allow-data-loss` (the empty `.fluid//` folders above `files/` stay: only `files/` and below are managed). It found the USE CATALOG check reading a group-conveyed privilege as none. After the fix, `fluid verify` on a fresh one-table product (`+4`) passes every dimension, catalog access included (1 schema and 1 table swept, 0 left). | A GIT source, profile or OAuth auth, a run-as service principal, table adoption and a larger project, live; the region residency check (the metastore precondition and `fluid verify`'s residency dimension), which was added after this run and not run live; dbt's CREATE OR REPLACE drops a contract's NOT NULL, so verify flags a `required` column a dbt model does not constrain itself. | +| Manual live, Free Edition (founder-approved), masks and row filters | the real `fluid` CLI, no test file: run 2026-11-03 at the tip of #785, a real workspace with an AWS us-west-2 metastore; every object swept after (0 left) | ✅ 10 steps passed. Functions are created and read back (owner is the user's email; `routine_definition` is the body without `RETURN`, backticks kept; `full_data_type` is STRING or BOOLEAN; a zero-parameter function's FunctionInfo has no `input_params`). The row filter and masks are bound and enforced on the restricted user's own SELECT: only the consented rows, `msisdn` as `XXXXX4567`, a 3-character value hashed, the denied column NULL. `SET ROW FILTER ... ON ()` is accepted and a `SET` replaces a live filter. Tags are present, `fluid verify` passes every dimension, a second apply is `+0 ~0 -0`, the residency precondition fails the plan and names both regions, `generate iac` refuses, and the sweep left 0. | Still open, as this run showed: superseded functions were left behind (orphans; the follow-up that drops them after a successful apply is not run live); `tofu destroy` of the module fails at a schema that still holds functions, so a teardown must drop them first; and the residency failure text was forge-cli's 800-character tail of the tofu JSON, which cut off the sentence naming the live region (the one-line explanation is not run live). A metastore in another cloud or region, a service principal as the run identity, and a group-conveyed grant on the live workspace are still not run. | | Stage 2 emulator (nightly, non-gating) | `tests/iac/test_iac_databricks_minilake.py` (job `nightly-databricks-minilake`) | ⚠️ first run 2026-10-30 (manual dispatch, run 41111425504, minilake v1.9.0): 1/4. Schema, table and grant creates work, and the replace gate holds. | A second plan is not empty and every in-place table update fails: minilake strips `TBLPROPERTIES` at create and its SQL parser rejects `ALTER TABLE ... SET TBLPROPERTIES`, which the provider sends before each column change. An upstream issue is drafted, not filed. Jobs and workspace files are not in this flow yet. | -| Stage 3 live | — | ❌ not automated: manual maintainer runs only, never CI. One has run: the manual row above (2026-11-01). | Everything the rows above leave to the live tier. | +| Stage 3 live | — | ❌ not automated: manual maintainer runs only, never CI. Two have run: the manual rows above (2026-11-01 and 2026-11-03). | Everything the rows above leave to the live tier. | ## What's NOT tested (deliberately deferred) diff --git a/README.md b/README.md index 36afada5e..5bc2ec502 100644 --- a/README.md +++ b/README.md @@ -292,19 +292,15 @@ something is a deployment target: `--provider` also accepts `confluent`, for Kafka-side resources, and `auto`, which is the default and reads the target out of the contract's binding. -**`fluid providers` prints a longer list, and it is a different thing.** It is -the registry of everything that registers as a provider plugin, which includes -publishers and a partly-landed emitter: +**`fluid providers` prints these providers and any third-party plugins.** Two +names that older releases listed are no longer providers (0.27.0): - `datamesh_manager` is a **publish** target, reached by - [`fluid publish`](https://agenticstiger.github.io/forge_docs/cli/datamesh-manager.html). - It receives a contract's metadata; it never provisions infrastructure. -- `redshift` has an emitter in the tree but is **not selectable with - `--provider`** — `fluid generate iac --provider redshift` exits with - `invalid choice`. Treat it as unfinished rather than as a fourth cloud. - -Neither belongs in a list of places a data product can be deployed, and this -README previously listed both as though they were. + [`fluid publish --target datamesh-manager`](https://agenticstiger.github.io/forge_docs/cli/datamesh-manager.html) + or `fluid datamesh-manager`. It receives a contract's metadata; it never + provisions infrastructure. `--provider datamesh-manager` exits 2 and says so. +- `redshift` is part of the `aws` provider. `--provider redshift` exits 2 and + says to use `--provider aws`. That column is the set each emitter is capable of. Which resources a particular contract actually produces is visible before anything is applied: `fluid generate iac ` writes the diff --git a/fluid_build/_error_catalog.py b/fluid_build/_error_catalog.py index 30c31226f..f03503c93 100644 --- a/fluid_build/_error_catalog.py +++ b/fluid_build/_error_catalog.py @@ -74,7 +74,7 @@ def slug_for(event: str) -> str: # ── providers ────────────────────────────────────────────────────────── "provider_not_specified": ( [ - "Pass a provider: --provider local|gcp|snowflake|aws|azure", + "Pass a provider: --provider aws|databricks|gcp|local|snowflake", "Or set the FLUID_PROVIDER environment variable", "Run 'fluid providers' to list the available providers", ], @@ -91,7 +91,14 @@ def slug_for(event: str) -> str: "provider_unknown": ( [ "Run 'fluid providers' to see the available providers", - "Valid built-ins: local, gcp, snowflake, aws, azure", + "Valid built-ins: aws, databricks, gcp, local, snowflake", + ], + "providers", + ), + "provider_retired": ( + [ + "Run 'fluid providers' to see the available providers", + "CHANGELOG.md lists it under Removed, with its replacement", ], "providers", ), @@ -258,6 +265,25 @@ def slug_for(event: str) -> str: ["Run 'fluid validate ' first to rule out a contract problem"], None, ), + "apply_build_identifier_invalid": ( + [ + "Rename the id to fit, for example orders_v1 instead of 1orders, and apply again", + ( + "A renamed id starts over: its builds get new cursors and watermarks under " + ".fluid/runs/, and a renamed contract.id gets a new OpenTofu workdir under " + ".fluid/iac//" + ), + ], + None, + ), + "apply_build_id_not_found": ( + [ + "Pass one of the available build ids as --build-id: ids match exactly, case included", + "Run 'fluid apply --mode amend-and-build' without --build-id to run " + + "every build", + ], + None, + ), "output_write_failed": ( ["Check that the --out path is writable and the parent directory exists"], None, @@ -315,6 +341,21 @@ def slug_for(event: str) -> str: ], "secrets", ), + # Optional-dependency gate (fluid_build/_optional.py). The message already + # carries the exact `pip install` line; these only give the stable slug a + # curated follow-up and a docs anchor. + "copilot_litellm_unavailable": ( + ["Run 'fluid doctor' to see which optional packages are installed"], + None, + ), + "mcp_sdk_unavailable": ( + ["Run 'fluid doctor' to see which optional packages are installed"], + None, + ), + "optional_dependency_missing": ( + ["Run 'fluid doctor' to see which optional packages are installed"], + None, + ), "copilot_missing_llm_model": ( [ "Set FLUID_LLM_MODEL or pass --llm-model", diff --git a/fluid_build/_mcp_compat.py b/fluid_build/_mcp_compat.py index 83423fedb..7cb2261ba 100644 --- a/fluid_build/_mcp_compat.py +++ b/fluid_build/_mcp_compat.py @@ -41,8 +41,10 @@ Tier-0 leaf rules: no module-scope ``mcp`` import (the startup budget in ``tests/perf/test_startup_budget.py`` forbids ``mcp`` on ``fluid --help``); -every SDK import below is function-local, and the version probe uses -``find_spec`` (metadata only, imports nothing). +every SDK import below is function-local. The generation probe uses +``find_spec``, which is not free: for a dotted name it imports the +parent packages (``mcp`` and ``mcp.server``, about 430 modules). It therefore +runs only from :func:`is_v2`, never on a cold path. """ from __future__ import annotations @@ -58,16 +60,35 @@ _MISSING = object() +def require_sdk(feature: str = "The MCP server") -> None: + """Raise ``MissingOptionalDependency`` (``[mcp]``) when the SDK is absent. + + Called at the entry of every function that is the first to touch the SDK on + a real path (``fluid mcp serve``, the output-port server, tool errors), so a + missing install reads as one install line and not ``No module named 'mcp'``. + Probes with ``find_spec``; nothing is imported. + """ + from fluid_build._optional import MissingOptionalDependency, is_available + + if not is_available("mcp"): + raise MissingOptionalDependency("mcp", "mcp", feature) + + @functools.lru_cache(maxsize=1) def is_v2() -> bool: """True when the installed SDK is the 2.x generation. Feature-probe (``mcp.server.mcpserver`` exists only in 2.x) — never - parse ``mcp.__version__``. ``find_spec`` reads packaging metadata - without importing ``mcp``, so the cold-path budget is unaffected. - Memoised with ``lru_cache`` per house style (lock-guarded, and + parse ``mcp.__version__``. ``find_spec`` on a dotted name imports its + parent packages (``mcp``, ``mcp.server``: about 430 modules), so this is + not free and must stay off the cold ``--help`` path. It raises + ``ModuleNotFoundError`` when ``mcp`` itself is absent (CPython bpo-30436), + so the top-level package is probed first; with no SDK installed this is + ``False``. Memoised with ``lru_cache`` per house style (lock-guarded, and ``is_v2.cache_clear()`` is the test reset hook). """ + if importlib.util.find_spec("mcp") is None: + return False return importlib.util.find_spec("mcp.server.mcpserver") is not None @@ -103,6 +124,7 @@ def get_tool_error() -> type: The class also moved: ``mcp.server.fastmcp.exceptions`` on v1, ``mcp.server.mcpserver.exceptions`` on v2. """ + require_sdk() if is_v2(): from mcp.server.mcpserver.exceptions import ToolError @@ -120,6 +142,7 @@ def get_server_api() -> tuple[type, type]: ``.tool(**kwargs)(fn)``, and both inject ``ctx`` as a handler parameter, so callers need no further branching. """ + require_sdk() if is_v2(): from mcp.server.mcpserver import Context, MCPServer @@ -157,6 +180,7 @@ def build_lowlevel_server( ``on_*`` constructor kwargs and wraps returns in the typed ``*Result`` models the 2.x lowlevel server requires. """ + require_sdk() from mcp.server.lowlevel import Server # local: keep ``fluid --help`` light # Forward ``version`` / ``lifespan`` only when set: an explicit None diff --git a/fluid_build/_optional.py b/fluid_build/_optional.py new file mode 100644 index 000000000..e01351dab --- /dev/null +++ b/fluid_build/_optional.py @@ -0,0 +1,126 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Optional-dependency gate: one call, one install line, no traceback. + +``require("litellm", "litellm", "fluid ai setup")`` returns the imported module +or raises :class:`MissingOptionalDependency`, whose message names the exact +``pip install 'data-product-forge[]'`` line. The CLI entry point renders +that message and exits 1, with no traceback. + +Borrowed from pandas' ``import_optional_dependency`` +(``pandas/compat/_optional.py``, BSD-3-Clause): an import-name to distribution +mapping for the message, and the ``ImportError`` chained ``from err``. Two +deliberate differences: + +* The package is probed with :func:`importlib.util.find_spec` BEFORE it is + imported, so a missing package never half-imports. For a dotted name the + top-level package is probed first, because ``find_spec("a.b")`` itself raises + ``ModuleNotFoundError`` when ``a`` is absent (CPython bpo-30436). +* A package that IS present but fails to import re-raises the original error + chained, instead of being misreported as "not installed". + +:class:`MissingOptionalDependency` subclasses :class:`ImportError`, so existing +``except ImportError`` callers keep working. + +Stdlib only and import-light: import it where a gate is needed, never at the +top of a module on the cold ``fluid --help`` path. +""" + +from __future__ import annotations + +import importlib +import importlib.util +import sys +from types import ModuleType + +__all__ = ["MissingOptionalDependency", "is_available", "require"] + +# import name -> distribution name, where they differ (pandas' ``INSTALL_MAPPING``). +_DISTRIBUTION = { + "openlineage": "openlineage-python", + "yaml": "pyyaml", +} + +# import name -> stable error code (see ``fluid_build/_error_catalog.py``). +_EVENTS = { + "litellm": "copilot_litellm_unavailable", + "mcp": "mcp_sdk_unavailable", +} +_GENERIC_EVENT = "optional_dependency_missing" + + +class MissingOptionalDependency(ImportError): + """An optional package is not installed; ``str(exc)`` is the install hint.""" + + exit_code = 1 + + def __init__(self, module: str, extra: str, feature: str) -> None: + self.module = module + self.extra = extra + self.feature = feature + top = module.split(".", 1)[0] + self.package = _DISTRIBUTION.get(top, top) + self.event = _EVENTS.get(top, _GENERIC_EVENT) + super().__init__( + f"{feature} needs {self.package}, which is not installed.\n\n" + f" pip install 'data-product-forge[{extra}]'", + name=module, + ) + + def __reduce__(self) -> tuple: # type: ignore[type-arg] + # ``__init__`` takes (module, extra, feature) but ``args`` holds the + # message, so the default reduce could not rebuild it (pickle, copy). + return (type(self), (self.module, self.extra, self.feature)) + + +def _present(module: str) -> bool: + """True when *module* can be located, without importing it.""" + if module in sys.modules: # already imported (or deliberately injected) + return sys.modules[module] is not None + top = module.split(".", 1)[0] + try: + if importlib.util.find_spec(top) is None: + return False + return module == top or importlib.util.find_spec(module) is not None + except (ImportError, ValueError): + return False + + +def is_available(module: str) -> bool: + """Whether *module* is installed, without importing it. + + For a dotted name, ``find_spec`` imports the parent packages (CPython + semantics); probe the top-level name when that cost matters. + """ + return _present(module) + + +def require(module: str, extra: str, feature: str) -> ModuleType: + """Import and return *module*, or raise :class:`MissingOptionalDependency`. + + ``extra`` is the ``data-product-forge`` extra that provides it and + ``feature`` is the user-facing name of what needs it ("fluid ai setup"). + """ + if not _present(module): + raise MissingOptionalDependency(module, extra, feature) + # Present but broken (a half-installed wheel, an ABI mismatch): let the real + # error through, chained, rather than telling the user to install it. + try: + return importlib.import_module(module) + except ImportError as err: + raise ImportError( + f"{feature} needs {module}, which is installed but failed to import: {err}", + name=module, + ) from err diff --git a/fluid_build/api/changelog.py b/fluid_build/api/changelog.py index e5547907b..767653ed5 100644 --- a/fluid_build/api/changelog.py +++ b/fluid_build/api/changelog.py @@ -15,17 +15,25 @@ """Contract-vs-contract semantic diff (the version-history side of ``fluid diff``). Public entry point: :func:`compare_contracts(baseline, new) -> ChangelogReport`. -The output is the structured envelope from ``changelog_types``; CLI/render -formatting lives in ``cli/diff.py`` (text / json / markdown emitters). +The output is the structured envelope from ``changelog_types``; the text and +Markdown renderers are below, and ``cli/diff.py`` picks one and prints it. The classifier is fluid-schema-native — it understands acquisition engines, ``agentPolicy``, ``sovereignty``, and the SDP/ADP/CDP × Bronze/Silver/Gold cross-check directly, rather than treating contracts as generic schema docs. + +Both contracts are schema-validated before they are compared (see +:class:`ContractValidationError`), and the renderers print every name a +contract supplies as data: spelled out for a terminal, inert in Markdown. """ from __future__ import annotations -from typing import Any, Mapping +import contextlib +import re +import tempfile +from pathlib import Path +from typing import Any, Dict, Iterator, List, Mapping from .changelog_rules import ( diff_agent_policy, @@ -39,11 +47,35 @@ from .changelog_types import Change, ChangelogReport -def compare_contracts(baseline: Mapping[str, Any], new: Mapping[str, Any]) -> ChangelogReport: +class ContractValidationError(ValueError): + """A contract handed to :func:`compare_contracts` is not a valid FLUID contract. + + ``errors`` maps each invalid side, ``"baseline"`` or ``"new"``, to its + errors, worded as ``fluid validate`` words them. A side that validated is + absent. + """ + + def __init__(self, errors: Mapping[str, List[str]]) -> None: + self.errors: Dict[str, List[str]] = {side: list(found) for side, found in errors.items()} + super().__init__( + "; ".join( + f"the {side} contract is not a valid FLUID contract ({len(found)} error(s))" + for side, found in self.errors.items() + ) + ) + + +def compare_contracts( + baseline: Mapping[str, Any], new: Mapping[str, Any], *, validate: bool = True +) -> ChangelogReport: """Return a structured changelog comparing two parsed contracts. ``baseline`` is treated as "old"; ``new`` as the candidate. Idempotent — passing the same dict twice produces an empty report. + + Both are first validated (:func:`_require_valid`), and an invalid one + raises :class:`ContractValidationError`. ``validate=False`` diffs whatever + it is given, for an advisory diff of a contract still being repaired. """ report = ChangelogReport() @@ -53,6 +85,9 @@ def compare_contracts(baseline: Mapping[str, Any], new: Mapping[str, Any]) -> Ch f"got {type(baseline).__name__} / {type(new).__name__}" ) + if validate: + _require_valid({"baseline": baseline, "new": new}) + # Top-level metadata churn — info-level signals. report.extend(diff_metadata(baseline, new)) @@ -99,11 +134,140 @@ def compare_contracts(baseline: Mapping[str, Any], new: Mapping[str, Any]) -> Ch return report +def _require_valid(contracts: Mapping[str, Mapping[str, Any]]) -> None: + """Raise :class:`ContractValidationError` naming every side that is not valid. + + The rules read only the shapes a schema allows, so an invalid contract can + hide a change from them: a column whose ``type`` is a number is never + compared. This is the gate ``fluid plan`` runs before it writes a plan: the + JSON Schema of the contract's own ``fluidVersion``, offline, from the + bundled schemas. A pre-0.7 version has no schema left, and is refused with + the upgrade that ``fluid validate`` asks for instead of "schema not + available". A side the validator cannot even read (a ``fluidVersion`` that + is not a version, such as ``7``) is refused too, never let through. + """ + errors: Dict[str, List[str]] = {} + with _schema_manager() as manager: + for side, contract in contracts.items(): + try: + found = _schema_errors(manager, contract) + except Exception as exc: # noqa: BLE001 — a contract nothing can validate is refused + found = [f"the contract cannot be validated: {exc}"] + if found: + errors[side] = found + if errors: + raise ContractValidationError(errors) + + +@contextlib.contextmanager +def _schema_manager() -> Iterator[Any]: + """The schema manager, with its cache in a temporary directory when HOME is not writable. + + The manager creates ``~/.fluid/schema_cache`` when it is built, which + raised on a read-only or missing HOME (a locked-down CI container) and + made the gate exit 1, as for a breaking change. The diff validates + offline against the bundled schemas, so a temporary cache loses nothing + a bundled version needs. + """ + # jsonschema stays off the import path of every caller that does not validate. + from fluid_build.schema_manager import FluidSchemaManager + + try: + manager = FluidSchemaManager() + except OSError: + manager = None + if manager is not None: + yield manager + return + with tempfile.TemporaryDirectory(prefix="fluid-schema-cache-") as cache: + yield FluidSchemaManager(cache_dir=Path(cache)) + + +def _schema_errors(manager: Any, contract: Mapping[str, Any]) -> List[str]: + version = manager.detect_version(dict(contract)) + if version is not None and (version.major, version.minor) < (0, 7): + return [ + f"fluidVersion {str(contract.get('fluidVersion'))!r} is no longer supported: " + "pre-0.7 schemas have been removed; upgrade the contract to fluidVersion 0.7.x " + "or later" + ] + result = manager.validate_contract(dict(contract), offline_only=True) + return [] if result.is_valid else list(result.errors) + + +# Characters ``str.isprintable`` passes that still draw nothing: the Unicode +# Default_Ignorable_Code_Point letters and marks (the combining grapheme joiner, +# the Hangul fillers, the Khmer inherent vowels, the Mongolian free variation +# selectors and the variation selectors), and the blank Braille pattern, +# which is a symbol but draws as a space. The other default-ignorables are +# format characters, which ``isprintable`` already refuses. +_DRAWS_NOTHING = re.compile( + "[\u034f\u115f\u1160\u17b4\u17b5\u180b-\u180d\u180f\u2800\u3164\ufe00-\ufe0f\uffa0" + "\U000e0100-\U000e01ef]" +) + + +def _spell(ch: str) -> str: + if ch == "\\": + return "\\\\" + if ch.isprintable() and not _DRAWS_NOTHING.match(ch): + return ch + code = ord(ch) + if code < 0x100: + return f"\\x{code:02x}" + if code < 0x10000: + return f"\\u{code:04x}" + return f"\\U{code:08x}" + + +def _text(value: Any) -> str: + """``value`` as one line of text that shows every character it holds. + + A path or a description quotes names from the contract, and printed raw a + name could drive the terminal: ESC opens an escape sequence (a colour, a + retitled window, a link to another URL), a line feed or a carriage return + forges or overwrites a line, and a bidi control reorders what the reader + sees (Trojan Source, CVE-2021-42574). Every character that is not + printable, or prints as nothing, is spelled as ``repr`` spells it + (``\\x1b``, ``\\u202e``), and a backslash is doubled, so a name holding the + four characters ``\\x1b`` still reads differently from one holding ESC. + """ + return "".join(_spell(ch) for ch in str(value)) + + +def _code(value: Any) -> str: + """``value`` as one Markdown code span, the one inline construct nothing is read out of. + + Outside a code span a backslash escape stops CommonMark markup but not + what a host adds after rendering: on GitHub ``\\@name`` still notifies, + ``\\#1`` links an issue, ``\\:smile\\:`` is an emoji and + ``foo\\@example\\.com`` a mail link, and GitLab also reads ``!1``, + ``~label`` and ``%milestone``. Inside a code span every character is + literal on every host. The fence is one backtick longer than the longest + run of backticks in the text, and the text is padded with a space when it + holds a backtick, or when it starts and ends with a space (CommonMark then + strips one space from each end), as mdformat's ``code_inline`` writes it + (MIT, github.com/hukkin/mdformat); empty text becomes one space. The text is spelled by + :func:`_text` first, so it holds no line break, which would let the next + line start a heading before the span is ever parsed. + """ + text = _text(value) + longest = max((len(run) for run in re.findall("`+", text)), default=0) + if longest: + fence = "`" * (longest + 1) + return f"{fence} {text} {fence}" + if text.startswith(" ") and text.endswith(" ") and text.strip(): + return f"` {text} `" + return f"`{text or ' '}`" + + def render_text(report: ChangelogReport) -> str: """Plain-text render: grouped by severity with one-line entries. Used by ``fluid diff --baseline ...`` default output. Kept simple so the - output is grep-friendly in CI logs. + output is grep-friendly in CI logs. Each field is spelled by + :func:`_text`, so a name in it cannot drive the terminal or forge a line; + print the result without Rich markup (``cprint_literal``). """ lines: list[str] = [] @@ -113,7 +277,7 @@ def _section(title: str, changes: list[Change]) -> None: lines.append(title) lines.append("-" * len(title)) for c in changes: - lines.append(f" {c.kind:32s} {c.path}: {c.description}") + lines.append(f" {_text(c.kind):32s} {_text(c.path)}: {_text(c.description)}") lines.append("") _section(f"BREAKING ({len(report.breaking)})", report.breaking) @@ -131,7 +295,16 @@ def _section(title: str, changes: list[Change]) -> None: def render_markdown(report: ChangelogReport) -> str: - """Markdown render — for PR-comment bots that paste this into a discussion.""" + """Markdown render — for PR-comment bots that paste this into a discussion. + + The path and the description quote names from the contract, so each is + one code span (:func:`_code`): a name can add no link, image, comment, + heading or list item, and mentions no one. ``kind`` is this module's own + vocabulary (``[a-z_]``). Redact each value before rendering and print the + result verbatim: Rich would read ``[...]`` in it as markup and wrap its + long lines, and a redaction regex run over the finished text can take a + closing fence with a ``token=x`` value (``fluid diff`` does both right). + """ parts: list[str] = ["# Contract changelog", ""] def _section(title: str, changes: list[Change], emoji: str) -> None: @@ -140,7 +313,7 @@ def _section(title: str, changes: list[Change], emoji: str) -> None: parts.append(f"## {emoji} {title} ({len(changes)})") parts.append("") for c in changes: - parts.append(f"- **{c.kind}** `{c.path}` — {c.description}") + parts.append(f"- **{c.kind}** {_code(c.path)} — {_code(c.description)}") parts.append("") _section("Breaking", report.breaking, "") diff --git a/fluid_build/api/changelog_rules.py b/fluid_build/api/changelog_rules.py index 0439e1103..f1aff1504 100644 --- a/fluid_build/api/changelog_rules.py +++ b/fluid_build/api/changelog_rules.py @@ -29,6 +29,8 @@ import re from typing import Any, Iterable, List, Mapping, Optional, Sequence, Tuple +from ..severity import is_error +from ..util.contract import column_is_key from .changelog_types import Change # Type widening table — keys are the "from" type, values are the set of @@ -181,17 +183,22 @@ def _expose_schema(expose: Mapping[str, Any]) -> Tuple[Sequence[Any], Optional[s Pre-0.7 contracts keep their columns at ``schema``; 0.7.x contracts keep them at ``contract.schema``. The flat ``schema`` is read when it is present and ``contract.schema`` only when it is absent, so a contract - that carries both is read as it was before 0.7.x. The location is - ``None`` when the expose carries no schema at all. + that carries both is read as it was before 0.7.x. Either one may wrap + its list as ``{"fields": [...]}``, as ``_verify_reconcile`` and + ``forge_copilot_drift_guard`` accept, and the columns are then found at + ``schema.fields``. The location is ``None`` when the expose carries no + schema at all. """ - columns = expose.get("schema") - if columns is not None: - return columns or [], "schema" - contract = expose.get("contract") - columns = contract.get("schema") if isinstance(contract, Mapping) else None - if columns is not None: - return columns or [], "contract.schema" - return [], None + columns, location = expose.get("schema"), "schema" + if columns is None: + contract = expose.get("contract") + columns = contract.get("schema") if isinstance(contract, Mapping) else None + location = "contract.schema" + if columns is None: + return [], None + if isinstance(columns, Mapping): + columns, location = columns.get("fields"), f"{location}.fields" + return (columns if isinstance(columns, (list, tuple)) else []), location def _is_nullable(column: Mapping[str, Any]) -> bool: @@ -225,7 +232,8 @@ def _primary_key_set(expose: Mapping[str, Any]) -> frozenset[str]: """Extract the set of primary-key column names from an expose dict. Supports both ``primaryKey: [col1, col2]`` at the expose root and the - per-column ``primaryKey: true`` flag — both occur in real contracts. + per-column ``primaryKey: true`` flag — both occur in real contracts — + and a 0.7.x column's primary-key tag (see :func:`_in_primary_key`). """ pk: set[str] = set() expose_pk = expose.get("primaryKey") @@ -234,13 +242,106 @@ def _primary_key_set(expose: Mapping[str, Any]) -> frozenset[str]: if isinstance(n, str): pk.add(n) for col in _expose_schema(expose)[0]: - if isinstance(col, Mapping) and col.get("primaryKey") is True: + if isinstance(col, Mapping) and _in_primary_key(col): name = col.get("name") if isinstance(name, str): pk.add(name) return frozenset(pk) +def _in_primary_key(column: Mapping[str, Any]) -> bool: + """Whether *column* is a key of its expose, the way forge builds it. + + The columns the dbt engine gives a ``primary_key`` constraint + (``util.contract.column_is_key``: a truthy key flag, a ``semanticType`` of + ``identifier`` or ``primary_key``, a ``labels`` ``constraint`` of + ``primary_key`` or ``unique``, a truthy ``labels.unique``), plus the + ``primary-key`` (or ``primaryKey``) tag the ODCS exporter turns into + ``primaryKey: true`` (``providers/odcs/mappers``). + """ + tags = column.get("tags") + if isinstance(tags, (list, tuple)) and ("primary-key" in tags or "primaryKey" in tags): + return True + return column_is_key(column) + + +#: Column sensitivities that mark personal data, as ``cli/docs_build.py`` and +#: the MCP output port's PII redaction read them. +_PERSONAL_DATA = frozenset({"pii", "phi"}) + + +def _is_pii(column: Mapping[str, Any]) -> bool: + """Whether *column* holds personal data. + + Pre-0.7 columns say ``pii: true``. A 0.7.x column has no ``pii`` key and + says ``sensitivity: pii`` (or ``phi``). + """ + sensitivity = column.get("sensitivity") + return bool(column.get("pii")) or ( + isinstance(sensitivity, str) and sensitivity in _PERSONAL_DATA + ) + + +def _sensitivity(column: Mapping[str, Any]) -> Any: + """A column's ``sensitivity``; a pre-0.7 ``pii: true`` with none reads as ``pii``. + + So rewriting ``pii: true`` as ``sensitivity: pii`` is not a change. + """ + sensitivity = column.get("sensitivity") + if sensitivity is None and column.get("pii"): + return "pii" + return sensitivity + + +def _sensitivity_changes( + old: Mapping[str, Any], new: Mapping[str, Any], col_name: str, path_prefix: str +) -> List[Change]: + """PII annotation drift, and any other change to a 0.7.x column's ``sensitivity``. + + A column gaining ``pii: true`` is a governance signal — downstream + consumers may have access controls that key off this flag. Surface as + INFO-level so it shows up in PR-review feedback without breaking the + build. A 0.7.x column says it with ``sensitivity`` (see :func:`_is_pii`), + and the change is reported at the key that carried it. + """ + old_pii, new_pii = _is_pii(old), _is_pii(new) + old_sensitivity, new_sensitivity = _sensitivity(old), _sensitivity(new) + if old_pii != new_pii: + if old.get("sensitivity") != new.get("sensitivity"): + key, before, after = "sensitivity", old_sensitivity, new_sensitivity + else: + key, before, after = "pii", old_pii, new_pii + return [ + Change( + path=f"{path_prefix}.{col_name}.{key}", + kind="column_pii_added" if new_pii else "column_pii_removed", + severity="info", + description=( + f"column '{col_name}' is now PII" + if new_pii + else f"column '{col_name}' PII annotation removed" + ), + before=before, + after=after, + ) + ] + if old_sensitivity != new_sensitivity: + return [ + Change( + path=f"{path_prefix}.{col_name}.sensitivity", + kind="column_sensitivity_changed", + severity="info", + description=( + f"column '{col_name}' sensitivity changed " + f"{old_sensitivity or 'unset'} -> {new_sensitivity or 'unset'}" + ), + before=old_sensitivity, + after=new_sensitivity, + ) + ] + return [] + + def diff_columns( baseline_expose: Mapping[str, Any], new_expose: Mapping[str, Any], @@ -360,27 +461,7 @@ def diff_columns( ) ) - # PII annotation drift. A column gaining ``pii: true`` is a - # governance signal — downstream consumers may have access - # controls that key off this flag. Surface as INFO-level so it - # shows up in PR-review feedback without breaking the build. - old_pii = bool(old.get("pii")) - new_pii = bool(new.get("pii")) - if old_pii != new_pii: - changes.append( - Change( - path=f"{path_prefix}.{col_name}.pii", - kind="column_pii_added" if new_pii else "column_pii_removed", - severity="info", - description=( - f"column '{col_name}' is now PII" - if new_pii - else f"column '{col_name}' PII annotation removed" - ), - before=old_pii, - after=new_pii, - ) - ) + changes.extend(_sensitivity_changes(old, new, col_name, path_prefix)) # Nullability changes. old_null = _is_nullable(old) @@ -587,33 +668,90 @@ def diff_sovereignty(baseline: Mapping[str, Any], new: Mapping[str, Any]) -> Lis return changes +#: Where an expose keeps its quality rules, and the keys that name one there: +#: pre-0.7 ``quality.tests``; in 0.7.x ``contract.dq.rules`` (``$defs.dqRule``) +#: and the inline ``contract.quality``. +_QUALITY_RULES: Tuple[Tuple[Tuple[str, ...], Tuple[str, ...], str], ...] = ( + (("quality", "tests"), ("name", "id"), "quality test"), + (("contract", "dq", "rules"), ("id",), "quality rule"), + (("contract", "quality"), ("rule",), "quality rule"), +) + + def diff_quality_severity( baseline_expose: Mapping[str, Any], new_expose: Mapping[str, Any], expose_id: str, expose_idx: int, ) -> List[Change]: - """A quality rule whose severity escalates warn -> error breaks downstream gating.""" - old_tests = _quality_tests_by_name(baseline_expose) - new_tests = _quality_tests_by_name(new_expose) + """A quality rule whose severity escalates to a failing one breaks downstream gating. + Reads the rules of every location in ``_QUALITY_RULES`` and, from 0.7.6, + each column's own ``quality`` rules (named by ``id``, ODCS's shape). + ``critical`` fails a run as ``error`` does (``fluid_build.severity``), so + either one is an escalation from ``warn``, and one to the other is not. A + rule is matched across versions by name; one without a name is skipped. + """ changes: List[Change] = [] - path_prefix = f"exposes[{expose_idx}].quality.tests" - for name, new_test in new_tests.items(): - old_test = old_tests.get(name) - if old_test is None: + for location, keys, noun in _QUALITY_RULES: + changes.extend( + _severity_escalations( + _rules_by_name(_dig(baseline_expose, location), keys), + _rules_by_name(_dig(new_expose, location), keys), + path_prefix=f"exposes[{expose_idx}].{'.'.join(location)}", + owner=f"expose '{expose_id}'", + noun=noun, + ) + ) + + base_columns = _columns_by_name(_expose_schema(baseline_expose)[0]) + new_cols, new_at = _expose_schema(new_expose) + for col_name, new_col in _columns_by_name(new_cols).items(): + old_col = base_columns.get(col_name) + if old_col is None: continue - old_sev = str(old_test.get("severity", "warn")).lower() - new_sev = str(new_test.get("severity", "warn")).lower() - if old_sev != "error" and new_sev == "error": + changes.extend( + _severity_escalations( + _rules_by_name(old_col.get("quality"), ("id", "name")), + _rules_by_name(new_col.get("quality"), ("id", "name")), + path_prefix=f"exposes[{expose_idx}].{new_at}.{col_name}.quality", + owner=f"column '{col_name}' of expose '{expose_id}'", + noun="quality rule", + ) + ) + return changes + + +def _severity_escalations( + old_rules: Mapping[str, Sequence[Mapping[str, Any]]], + new_rules: Mapping[str, Sequence[Mapping[str, Any]]], + *, + path_prefix: str, + owner: str, + noun: str, +) -> List[Change]: + """A named rule escalated: failing in the new contract, and not in the old one. + + A name can be on several rules (nothing makes an id unique), and each + version is judged by its most severe rule of that name, so a failing rule + cannot hide behind a ``warn`` one of the same id. + """ + changes: List[Change] = [] + for name, new_group in new_rules.items(): + old_group = old_rules.get(name) + if not old_group: + continue + old_sev = _most_severe(old_group) + new_sev = _most_severe(new_group) + if not is_error(old_sev) and is_error(new_sev): changes.append( Change( path=f"{path_prefix}.{name}.severity", kind="quality_severity_escalated", severity="breaking", description=( - f"quality test '{name}' on expose '{expose_id}' " - f"escalated to severity=error (previously {old_sev}); " + f"{noun} '{name}' on {owner} " + f"escalated to severity={new_sev} (previously {old_sev}); " f"downstream gating will now hard-fail" ), before=old_sev, @@ -623,17 +761,28 @@ def diff_quality_severity( return changes -def _quality_tests_by_name(expose: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: - out: dict[str, Mapping[str, Any]] = {} - quality = expose.get("quality") or {} - if not isinstance(quality, Mapping): +def _dig(node: Any, keys: Sequence[str]) -> Any: + for key in keys: + if not isinstance(node, Mapping): + return None + node = node.get(key) + return node + + +def _most_severe(rules: Sequence[Mapping[str, Any]]) -> str: + severities = [str(rule.get("severity", "warn")).lower() for rule in rules] + return next((sev for sev in severities if is_error(sev)), severities[0]) + + +def _rules_by_name(rules: Any, keys: Sequence[str]) -> dict[str, List[Mapping[str, Any]]]: + out: dict[str, List[Mapping[str, Any]]] = {} + if not isinstance(rules, (list, tuple)): return out - tests = quality.get("tests") or [] - for t in tests: - if isinstance(t, Mapping): - name = t.get("name") or t.get("id") + for rule in rules: + if isinstance(rule, Mapping): + name = next((rule.get(k) for k in keys if rule.get(k)), None) if isinstance(name, str): - out[name] = t + out.setdefault(name, []).append(rule) return out diff --git a/fluid_build/build_runners/_credentials.py b/fluid_build/build_runners/_credentials.py index 7349b69af..d6360867e 100644 --- a/fluid_build/build_runners/_credentials.py +++ b/fluid_build/build_runners/_credentials.py @@ -53,6 +53,7 @@ from __future__ import annotations +import logging from typing import Any, Callable, Dict, FrozenSet, Optional, Tuple, Type try: @@ -274,6 +275,15 @@ class AzureCredentials(BaseSettings): } else: _CREDENTIAL_CLASSES = {} + # An empty registry makes every platform fall through to "whatever the + # binding supplied", i.e. no env-var credential resolution at all. That is a + # silent behaviour change, so say so. (pydantic-settings is a base + # dependency, so this means a broken install, not a missing extra.) + logging.getLogger(__name__).warning( + "pydantic-settings could not be imported, so platform credentials are NOT " + "resolved from the environment (SNOWFLAKE_*, AWS_*, ...). Reinstall it with: " + "pip install --force-reinstall 'pydantic-settings>=2'" + ) def get_credentials_for(platform: str, binding: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: diff --git a/fluid_build/build_runners/_databricks_load.py b/fluid_build/build_runners/_databricks_load.py new file mode 100644 index 000000000..969c0b7ab --- /dev/null +++ b/fluid_build/build_runners/_databricks_load.py @@ -0,0 +1,547 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Land a build's Parquet file in the Unity Catalog table ``fluid apply`` created. + +A Databricks binding is not a file destination. The acquisition runner used to treat its +``location.path`` (or ``out/``) as one: it wrote a LOCAL file and reported success with nothing in +Databricks. Now the build lands its file under ``.fluid/staging`` (after the runner's own masking, +which hashes, tokenizes, encrypts or masks columns before the data leaves the machine), and this +module moves it into the table in three steps: + +1. upload the file to a Unity Catalog volume through the Files API, + ``////.parquet``; +2. run ONE statement on ``DATABRICKS_WAREHOUSE_ID`` through the SQL Statement Execution API:: + + INSERT OVERWRITE `c`.`s`.`t` (`col`, ...) SELECT `col`, ... FROM read_files('', format => 'parquet') + + reading the one file this run uploaded, never the directory (a file someone else put there is not + loaded), a full refresh that rewrites the table's rows and nothing else: the table's definition, grants, + tags, row filter and column masks stay (``INSERT OVERWRITE`` is a data operation; ``CREATE OR + REPLACE`` is the statement that rebuilds a table); +3. delete the staged files and their directory, whether the load worked or not. The volume must never + keep landed data (masked, but still the product's data). A file that cannot be deleted fails the + run and is named, so that someone deletes it. Ctrl-C (or any other interrupt) during the upload or + the INSERT still deletes the files before it propagates. An INSERT the warehouse has already + answered for is cancelled first; one still inside its first 30-second wait runs on, and Delta + commits it whole or not at all. A run killed outright (SIGKILL, a CI runner torn down) cannot clean up, so each + load first deletes what an earlier run of the same build left under + ``///``, once all of it is over an hour old. + +The table is the IaC's: a missing table is an error that names ``fluid apply``, and this module never +creates or alters one. The volume is named by ``DATABRICKS_STAGING_VOLUME`` +(``/Volumes///``), from the environment only, like +``DATABRICKS_WAREHOUSE_ID``: a contract that named a path could aim the upload anywhere the +credential reaches. Give the staging a volume of its own: the sweep of earlier runs deletes only +directories named like a run id that hold nothing but old Parquet files, but it runs under the +deploy credential. The client and the statement runner are the ones that enforce masks and row +filters (``iac/providers/databricks_enforce.py``), on the same pinned credential; no message carries +a credential. + +Not proven without a workspace: what ``read_files`` and ``INSERT OVERWRITE`` accept on a managed table +carrying a mask and a row filter, and the Files API's limits. ``tests/iac/_fake_databricks.py`` stands +in for them. +""" + +from __future__ import annotations + +import logging +import os +import re +import time +from dataclasses import dataclass +from typing import Any, Dict, List, Mapping, Optional, Tuple +from urllib.parse import quote + +from ..iac.provider_match import is_cloud +from ..iac.providers.databricks_access import quote as quote_ident +from ..iac.providers.databricks_access import quote_table, string_literal +from ..iac.providers.databricks_enforce import ( + DATABRICKS_EXTRA_INSTALL, + WAREHOUSE_ENV, + SqlRunner, + StatementFailed, + default_client, + is_not_found, + read_table, + scrub, +) +from ..providers._sql_safety import validate_ident +from ..util.binding_paths import ENV_PLACEHOLDER_RE + +#: The Unity Catalog volume the run's file is staged in: ``/Volumes///``. +STAGING_VOLUME_ENV = "DATABRICKS_STAGING_VOLUME" + +_VOLUME_FORM = "/Volumes///" +_VOLUME_RE = re.compile(r"^/Volumes/([^/]+)/([^/]+)/([^/]+)/?$") +#: How old every file of an earlier run's staging directory must be before a load deletes it: +#: well past a load's own span (the INSERT is cancelled after 600 s), so a run of the same build +#: still in progress keeps its upload. +STALE_STAGING_SECONDS = 3600 +#: The only mode a load supports: ``INSERT OVERWRITE`` replaces the table's rows. +LOAD_MODES = ("full_refresh",) +_SINK_FORMATS = ("parquet",) + + +class DatabricksLoadError(RuntimeError): + """The landed file could not be loaded into the table, or loaded the wrong number of rows.""" + + +@dataclass(frozen=True) +class StagingVolume: + catalog: str + schema: str + volume: str + + @property + def path(self) -> str: + return f"/Volumes/{self.catalog}/{self.schema}/{self.volume}" + + +@dataclass(frozen=True) +class LoadTarget: + """The table a Databricks expose lands in, named as the emitter creates it.""" + + catalog: str + schema: str + table: str + #: The contract's columns, in table order. + columns: Tuple[str, ...] + + @property + def full_name(self) -> str: + return f"{self.catalog}.{self.schema}.{self.table}" + + +def staging_volume(env: Mapping[str, str]) -> StagingVolume: + """``DATABRICKS_STAGING_VOLUME`` as three identifiers, or a refusal that names the variable.""" + raw = str(env.get(STAGING_VOLUME_ENV) or "").strip() + if not raw: + raise DatabricksLoadError( + f"{STAGING_VOLUME_ENV} is not set. The build stages its Parquet file in a Unity Catalog " + f"volume before loading the table; set it to {_VOLUME_FORM}." + ) + match = _VOLUME_RE.match(raw) + parts = match.groups() if match else () + try: + for part in parts: + validate_ident(part) + except ValueError: + parts = () + if len(parts) != 3: + raise DatabricksLoadError( + f"{STAGING_VOLUME_ENV} is {raw[:120]!r}, which is not {_VOLUME_FORM}, with each part a " + "plain identifier (letters, digits and underscores)." + ) + return StagingVolume(*parts) + + +def run_prefix(volume: StagingVolume, product_id: Any, build_id: Any, run_id: Any) -> str: + """The directory one run stages in: ``///``.""" + from .duckdb.runner import _path_part # one safe path component, as the local staging uses + + parts = (_path_part(product_id), _path_part(build_id), _path_part(run_id)) + return "/".join((volume.path, *parts)) + + +def _resolved(value: Any) -> Any: + """``{{ env.NAME }}`` in a name, resolved as the runner resolves a bucket; unset ones stay.""" + if not isinstance(value, str) or "{{" not in value: + return value + return ENV_PLACEHOLDER_RE.sub(lambda m: os.environ.get(m.group(1), m.group(0)), value) + + +def databricks_load_target( + binding: Mapping[str, Any], expose: Mapping[str, Any] +) -> Optional[LoadTarget]: + """The table a binding loads into, or ``None`` when it is not a Databricks binding. + + Resolved with the emitter's own helpers, so the load names the catalog, schema, table and + columns ``fluid apply`` created, and every name has been through the same identifier check. + A Databricks binding that cannot be resolved raises :class:`DatabricksLoadError` with the + emitter's reasons. + """ + if not is_cloud(binding, "databricks"): + return None + from ..iac.providers import databricks as dbx + + where = f"exposes[{expose.get('exposeId')}]" + raw_loc = binding.get("location") + loc: Dict[str, Any] = dict(raw_loc) if isinstance(raw_loc, Mapping) else {} + loc.update({k: _resolved(loc.get(k)) for k in ("database", "schema", "table")}) + refusals: List[Any] = [] + names = dbx._location_names(where, loc, refusals, validating=False) + columns = dbx._columns(where, expose, refusals, validating=False) + if refusals: + raise DatabricksLoadError("; ".join(str(r) for r in refusals)) + return LoadTarget( + catalog=str(names["database"]), + schema=str(names["schema"]), + table=str(names["table"]), + columns=tuple(column["name"] for column in columns), + ) + + +def unsupported_reason(mode: str, sink_format: str, stream_count: int) -> Optional[str]: + """Why this build cannot be loaded into a Databricks table, or ``None`` if it can. + + Checked before the build runs, so an unsupported shape fails by name instead of landing + somewhere else and reporting success. + """ + if stream_count != 1: + return f"a Databricks binding loads exactly one stream; this build has {stream_count}" + if sink_format not in _SINK_FORMATS: + return f"a Databricks load takes {list(_SINK_FORMATS)}; the sink format is {sink_format!r}" + if mode not in LOAD_MODES: + return ( + f"a Databricks load is an INSERT OVERWRITE, so it supports modes {list(LOAD_MODES)}; " + f"this build is {mode!r}" + ) + return None + + +def insert_overwrite_statement(target: LoadTarget, source: str) -> str: + """The one statement that loads the run's file: every name an identifier the emitter + accepted, quoted; the path a Spark string literal (backslash escapes). + + The target columns are listed, so each value goes to the column of its name whatever order + the live table keeps them in (a column added later by ``fluid apply`` sits at the end, not + where the contract declares it); without the list Databricks inserts by position. + """ + columns = ", ".join(quote_ident(column) for column in target.columns) + return ( + f"INSERT OVERWRITE {quote_table(target.full_name)} ({columns}) SELECT {columns} " + f"FROM read_files({string_literal(source)}, format => 'parquet')" + ) + + +#: A run directory's name: the acquisition runner's run id (``01`` + 16 Crockford base32 +#: characters, ``_acquisition_common.generate_run_id``). The sweep touches nothing else. +_RUN_DIR_RE = re.compile(r"01[0-9A-HJKMNP-TV-Z]{16}") + + +def _stale_files(listed: Any, run_dir: str, cutoff_ms: float) -> Optional[List[str]]: + """The files of an earlier run's directory listing, when it looks like what a load stages + (Parquet files only, no subdirectory) and every file was last modified before + ``cutoff_ms`` (epoch milliseconds, as the Files API reports ``last_modified``); ``None`` + otherwise: a file that is newer, undated or not Parquet, a subdirectory, or an empty + directory (a run that has made it and is still uploading).""" + files: List[str] = [] + if not listed: + return None + for entry in listed: + entry = _mapping(entry) + path = str(entry.get("path") or "") + if entry.get("is_directory") or not path.startswith(run_dir + "/"): + return None + if not path.endswith(".parquet"): + return None + try: + modified = float(entry.get("last_modified")) + except (TypeError, ValueError): + return None + if modified >= cutoff_ms: + return None + files.append(path) + return files + + +def _fs(kind: str, path: str) -> str: + return f"/api/2.0/fs/{kind}{quote(path, safe='/')}" + + +def _failure(exc: BaseException) -> str: + return f"{type(exc).__name__}: {scrub(exc)}" + + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _rows_reported(doc: Mapping[str, Any]) -> Optional[Tuple[int, str]]: + """The rows an INSERT says it wrote: ``num_inserted_rows``, else ``num_affected_rows``.""" + schema = _mapping(_mapping(doc.get("manifest")).get("schema")) + names = [str(_mapping(c).get("name")) for c in schema.get("columns") or []] + data = _mapping(doc.get("result")).get("data_array") or [] + if not data or not isinstance(data[0], list): + return None + for name in ("num_inserted_rows", "num_affected_rows"): + if name in names and names.index(name) < len(data[0]): + try: + return int(data[0][names.index(name)]), name + except (TypeError, ValueError): + return None + return None + + +class Landing: + """A client, a runner and a staging volume, for loading one build's file.""" + + def __init__(self, target: LoadTarget, volume: StagingVolume, client: Any, sql: SqlRunner): + self.target = target + self.volume = volume + self._client = client + self._sql = sql + + # ── the three steps ──────────────────────────────────────────────── + + def _upload(self, path: str, prefix: str, remote: str) -> None: + # One PUT to the Files API's own endpoint (the one databricks-sdk's FilesAPI.upload + # calls), not WorkspaceClient.files.upload: from databricks-sdk 0.4x its FilesExt can + # switch to multipart uploads through presigned cloud-storage URLs, a second host the + # data would go to. The cost is the single PUT's 5 GiB limit per file. + do = self._client.api_client.do + try: + do("PUT", _fs("directories", prefix)) + except Exception as exc: # noqa: BLE001 — every transport failure is the upload's + raise DatabricksLoadError( + f"could not create {prefix} in the staging volume: {_failure(exc)}" + ) from exc + try: + with open(path, "rb") as fh: + do( + "PUT", + _fs("files", remote), + query={"overwrite": "true"}, + headers={"Content-Type": "application/octet-stream"}, + data=fh, + ) + except Exception as exc: # noqa: BLE001 + raise DatabricksLoadError(f"could not upload to {remote}: {_failure(exc)}") from exc + + def _count(self, full: str) -> int: + doc = self._sql.execute(f"SELECT COUNT(*) FROM {quote_table(full)}") + data = _mapping(doc.get("result")).get("data_array") or [] + try: + return int(data[0][0]) + except (IndexError, TypeError, ValueError) as exc: + raise DatabricksLoadError(f"counting {full} returned no readable count") from exc + + def _insert(self, remote: str) -> Tuple[int, str]: + full = self.target.full_name + try: + doc = self._sql.execute(insert_overwrite_statement(self.target, remote)) + except StatementFailed as exc: + raise DatabricksLoadError(f"loading {full} failed: {exc}") from exc + reported = _rows_reported(doc) + if reported is not None: + return reported + # A full refresh leaves exactly the file's rows, so the table's count is the load's. + try: + return self._count(full), "count_after_load" + except StatementFailed as exc: + raise DatabricksLoadError( + f"loaded {full}, but counting its rows afterwards failed: {exc}" + ) from exc + + def _delete(self, prefix: str, remote: str) -> List[str]: + """Delete the staged file and its directory; what could not be deleted, named.""" + do = self._client.api_client.do + try: + do("DELETE", _fs("files", remote)) + except Exception as exc: # noqa: BLE001 + if not is_not_found(exc): + return [f"{remote} ({_failure(exc)})"] + try: + do("DELETE", _fs("directories", prefix)) + except Exception as exc: # noqa: BLE001 + if not is_not_found(exc): + return [f"{prefix} ({_failure(exc)})"] + return [] + + def _list_dir(self, path: str) -> List[Any]: + """Every entry of a volume directory, following ``next_page_token``.""" + entries: List[Any] = [] + token: Optional[str] = None + for _ in range(100): + page = _mapping( + self._client.api_client.do( + "GET", _fs("directories", path), query={"page_token": token} if token else None + ) + ) + entries += list(page.get("contents") or []) + token = page.get("next_page_token") or None + if not token: + return entries + raise DatabricksLoadError(f"the listing of {path} never ended") + + def _sweep_stale(self, build_dir: str, current: str, logger: logging.Logger) -> List[str]: + """Delete what earlier runs of this build left in ``build_dir`` (a run killed before its + own delete); the run directories deleted. Only a run directory whose every file is older + than :data:`STALE_STAGING_SECONDS` is deleted: a run of the same build still in progress + keeps its upload. Best effort: a failure is logged, not raised, because this run's own + files are deleted by this run whatever happens here.""" + do = self._client.api_client.do + swept: List[str] = [] + cutoff_ms = (time.time() - STALE_STAGING_SECONDS) * 1000 + try: + runs = self._list_dir(build_dir) + except Exception as exc: # noqa: BLE001 + if not is_not_found(exc): + logger.warning( + "databricks.load could not list %s for earlier runs' staged files: %s", + build_dir, + _failure(exc), + ) + return swept + for run in runs: + run_dir = str(_mapping(run).get("path") or "").rstrip("/") + if not _mapping(run).get("is_directory") or run_dir == current: + continue + name = run_dir[len(build_dir) + 1 :] + if not run_dir.startswith(build_dir + "/") or not _RUN_DIR_RE.fullmatch(name): + continue # only a run directory, directly in this build's directory + try: + listed = self._list_dir(run_dir) + files = _stale_files(listed, run_dir, cutoff_ms) + if files is None: + continue # a file there is recent, or undated: perhaps a run in progress + for path in files: + do("DELETE", _fs("files", path)) + do("DELETE", _fs("directories", run_dir)) + except Exception as exc: # noqa: BLE001 + logger.warning( + "databricks.load could not delete an earlier run's staged files in %s; " + "delete them: %s", + run_dir, + _failure(exc), + ) + continue + swept.append(run_dir) + logger.warning( + "databricks.load deleted staged files an earlier run left behind: %s", run_dir + ) + return swept + + # ── the load ─────────────────────────────────────────────────────── + + def load( + self, + path: str, + *, + product_id: Any, + build_id: Any, + run_id: Any, + expected_rows: int, + logger: logging.Logger, + ) -> Dict[str, Any]: + """Upload ``path``, replace the table's rows with it, delete the upload. Returns the facts. + + The table must exist. The staged files are deleted whether or not the load worked, and + a load that wrote the wrong number of rows, or a file that cannot be deleted, fails. + """ + full = self.target.full_name + try: + live = read_table(self._client, full) + except Exception as exc: # noqa: BLE001 + raise DatabricksLoadError( + f"could not read {full} from Unity Catalog: {_failure(exc)}" + ) from exc + if live is None: + raise DatabricksLoadError( + f"{full} does not exist in Unity Catalog. The build loads the table fluid apply " + "creates and never creates one itself: run fluid apply for this contract first." + ) + prefix = run_prefix(self.volume, product_id, build_id, run_id) + remote = f"{prefix}/{os.path.basename(path)}" + swept = self._sweep_stale(prefix.rsplit("/", 1)[0], prefix, logger) + loaded = 0 + rows_from = "" + failure: Optional[DatabricksLoadError] = None + try: + self._upload(path, prefix, remote) + loaded, rows_from = self._insert(remote) + if loaded != expected_rows: + raise DatabricksLoadError( + f"loaded {loaded} rows into {full}, but the landed file holds {expected_rows}" + ) + except DatabricksLoadError as exc: + failure = exc + except Exception as exc: # noqa: BLE001 — a failure must still reach the delete below + failure = DatabricksLoadError(_failure(exc)) + except BaseException: + # Ctrl-C or SystemExit mid-load (the statement runner has cancelled the INSERT): the + # staged files still go before the interrupt propagates. + left = self._delete(prefix, remote) + if left: + logger.error( + "databricks.load interrupted, and the staged files, which hold the landed " + "data, could not be deleted; delete them: %s", + "; ".join(left), + ) + raise + left = self._delete(prefix, remote) + if left: + # the paths lead the message: a caller that shortens it must not cut them off + logger.error( + "databricks.load could not delete the staged files, which hold the landed data; " + "delete them: %s", + "; ".join(left), + ) + if failure is not None: + if left: + raise DatabricksLoadError( + f"could not delete the staged files, which hold the landed data; delete them: " + f"{'; '.join(left)}. The load had failed: {failure}" + ) from None + raise DatabricksLoadError( + f"{failure} (the staged files in {prefix} were deleted)" + ) from None + if left: + raise DatabricksLoadError( + f"loaded {loaded} rows into {full}, but could not delete the staged files, which " + f"hold the landed data; delete them: {'; '.join(left)}" + ) + logger.info( + "databricks.load table=%s rows=%d rows_from=%s staged_to=%s deleted=true", + full, + loaded, + rows_from, + prefix, + ) + return { + "table": full, + "rows": loaded, + "rows_from": rows_from, + "staged_to": prefix, + "staged_files_deleted": True, + "earlier_runs_staging_deleted": swept, + } + + +def open_landing(target: LoadTarget, *, env: Optional[Mapping[str, str]] = None) -> Landing: + """A :class:`Landing` for ``target``, or a :class:`DatabricksLoadError` naming what is missing. + + Called before the build runs, so a missing volume, warehouse, credential or SDK refuses the + run before anything is read or written. + """ + env = os.environ if env is None else env + volume = staging_volume(env) + warehouse = str(env.get(WAREHOUSE_ENV) or "").strip() + if not warehouse: + raise DatabricksLoadError( + f"{WAREHOUSE_ENV} is not set. The load runs on that SQL warehouse: set it to the id " + "of a warehouse in the workspace DATABRICKS_HOST names." + ) + try: + client = default_client() + except ImportError: + raise DatabricksLoadError( + f"loading into Databricks needs the databricks extra: {DATABRICKS_EXTRA_INSTALL}" + ) from None + except Exception as exc: # noqa: BLE001 — a bad credential must not become a traceback + raise DatabricksLoadError( + f"could not create the Databricks client: {_failure(exc)}" + ) from exc + return Landing(target, volume, client, SqlRunner(client, warehouse)) diff --git a/fluid_build/build_runners/_ids.py b/fluid_build/build_runners/_ids.py index b2d3abde5..5fdb0f241 100644 --- a/fluid_build/build_runners/_ids.py +++ b/fluid_build/build_runners/_ids.py @@ -37,6 +37,7 @@ from __future__ import annotations import re +from typing import Any, Mapping, Optional # A permissive regex, identical to ``validate_ident`` in # ``providers/_sql_safety``: alphanumerics + dots + dashes + underscores, @@ -44,11 +45,21 @@ # DOS-via-filename attacks. Crucially this excludes ``/``, ``\``, ``..`` # (a leading dot is rejected) and any path separator, so a validated id # can never traverse out of the directory it is joined into. -_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.\-]{0,127}$") +_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.\-]{0,127}\Z") class IdentifierViolation(ValueError): - """Raised when a contract identifier fails validation.""" + """Raised when a contract identifier fails validation. + + ``kind`` and ``value`` name the id that failed when the raiser knows it + (:func:`validate_identifier` always does), so a caller can report it + without parsing the message. + """ + + def __init__(self, message: str, *, kind: Optional[str] = None, value: Any = None) -> None: + super().__init__(message) + self.kind = kind + self.value = value def validate_identifier(value: str, *, kind: str) -> str: @@ -65,6 +76,30 @@ def validate_identifier(value: str, *, kind: str) -> str: f"{kind} {value!r} is not a valid identifier " "(must match ^[A-Za-z_][A-Za-z0-9_.\\-]{0,127}$). This guard " "blocks template-injection / path-traversal attacks via " - "attacker-controlled contract metadata." + "attacker-controlled contract metadata.", + kind=kind, + value=value, ) return value + + +def validate_build_ids(contract: Mapping[str, Any]) -> None: + """Validate the ids a contract's build phase uses: ``contract.id`` and each ``builds[].id``. + + The build runner (``base.run_builds_from_args``) runs this before any + build, and ``fluid apply`` runs it before it applies anything, so the two + refuse the same ids. A contract with no builds has no build phase and is + not checked. An empty or absent id is skipped: it cannot traverse, and the + state store falls back to a safe default. A ``builds`` entry that is not a + mapping is left to the schema gate. + + Raises :class:`IdentifierViolation` for the first id that fails. + """ + builds = contract.get("builds") or [] + if not builds: + return + if contract.get("id"): + validate_identifier(contract["id"], kind="contract.id") + for build in builds: + if isinstance(build, Mapping) and build.get("id"): + validate_identifier(build["id"], kind="build.id") diff --git a/fluid_build/build_runners/_lineage.py b/fluid_build/build_runners/_lineage.py index 87414eef7..087366247 100644 --- a/fluid_build/build_runners/_lineage.py +++ b/fluid_build/build_runners/_lineage.py @@ -165,6 +165,7 @@ def run_id_to_uuid(run_id: str, event_time: str) -> str: timestamp produced a *different* ``runId`` for START and COMPLETE of the same run — which makes the pair uncorrelatable in Marquez/DataHub. """ + _require_openlineage() from openlineage.client.uuid import generate_static_uuid return str(generate_static_uuid(_parse_event_time(event_time), run_id.encode("utf-8"))) @@ -186,6 +187,14 @@ def _as_run_facet(payload: Dict[str, Any], *, schema_path: str) -> Dict[str, Any } +def _require_openlineage() -> None: + """Raise :class:`~fluid_build._optional.MissingOptionalDependency` when the + OpenLineage client is not installed (``pip install 'data-product-forge[lineage]'``).""" + from fluid_build._optional import require + + require("openlineage.client", "lineage", "OpenLineage emission") + + def to_openlineage(event: RunEvent) -> Any: """Translate a FLUID ``RunEvent`` into a spec-conformant OpenLineage event. @@ -193,6 +202,7 @@ def to_openlineage(event: RunEvent) -> Any: supplies ``producer`` and ``schemaURL`` defaults, but we set ``producer`` explicitly so it identifies FLUID rather than the client library. """ + _require_openlineage() from openlineage.client.event_v2 import ( InputDataset, Job, @@ -244,6 +254,7 @@ def to_openlineage(event: RunEvent) -> Any: def encode_event(event: RunEvent) -> dict: """Serialise a FLUID ``RunEvent`` as conformant OpenLineage JSON-ready dict.""" + _require_openlineage() from openlineage.client.serde import Serde return Serde.to_dict(to_openlineage(event)) @@ -284,9 +295,11 @@ def _warn_once(key: str, message: str) -> None: _WARNED.add(key) LOG.warning("%s", message) try: - from fluid_build._console import warning as console_warning + from fluid_build._console import cprint, literal_text - console_warning(message) + # Literal: the message names an install extra such as ``[lineage]``, + # which Rich markup would read as a tag and drop. + cprint(literal_text(f"⚠️ {message}", style="yellow")) except Exception: # pragma: no cover — console is best-effort pass @@ -392,6 +405,13 @@ def emit(self, event: RunEvent) -> None: resp = client.post(self.endpoint, json=self._encode(event), headers=headers) resp.raise_for_status() except Exception as exc: # noqa: BLE001 + from fluid_build._optional import MissingOptionalDependency + + if isinstance(exc, MissingOptionalDependency): + # Not a transport failure: the receiver is configured and the + # client library that builds the event is absent. Name the extra. + _warn_once("missing-openlineage", f"Lineage is not being emitted.\n{exc}") + return # Class-only, because httpx error messages can echo the endpoint URL. # Surfaced on the CLI too: "lineage configured but nothing shipped" # is precisely the state an operator must not have to guess at. diff --git a/fluid_build/build_runners/base.py b/fluid_build/build_runners/base.py index b1d0d914a..d451e2f90 100644 --- a/fluid_build/build_runners/base.py +++ b/fluid_build/build_runners/base.py @@ -1407,17 +1407,17 @@ def run_builds_from_args( # rather than traversing the filesystem. The parallel pipeline # (``cli/_acquisition_stage_ext``) already validates the same fields; # this closes the one-sided-guard gap on the runtime path. - from ._ids import validate_identifier + from ._ids import validate_build_ids # Only non-empty ids are checked: an absent/empty id cannot traverse # (``_build_dir`` falls back to a safe "product" default and # ``FileStateStore._confine()`` is the backstop), while a non-empty id # like ``../../../../tmp/escape`` is rejected here before any path is made. - if contract.get("id"): - validate_identifier(contract["id"], kind="contract.id") - for _b in builds: - if _b.get("id"): - validate_identifier(_b["id"], kind="build.id") + # ``fluid apply`` runs the same check before it applies anything + # (``cli/apply.py::_gate_build_ids_for_apply``), so on the OpenTofu engine + # an id is not refused here after ``tofu apply``; this one stays as the + # backstop for any caller that skips it. + validate_build_ids(contract) # The running ``fluid apply``'s run report, if any: each build is recorded on it. from fluid_build.observability.apply_run import current_apply_run @@ -1425,7 +1425,11 @@ def run_builds_from_args( report = current_apply_run() product_id = str(contract.get("id") or "") - # Filter builds if specific ID requested + # Filter builds if specific ID requested. ``fluid apply`` refuses an id no + # build has before it applies anything + # (``cli/apply.py::_gate_selected_build_for_apply``), so on the OpenTofu + # engine it is not refused here after ``tofu apply``; this one stays as the + # backstop for any caller that skips it. if args.build_id: builds = [b for b in builds if b.get("id") == args.build_id] if not builds: diff --git a/fluid_build/build_runners/catalog_registrars/_http_retry.py b/fluid_build/build_runners/catalog_registrars/_http_retry.py index 110c2549f..5692db7e5 100644 --- a/fluid_build/build_runners/catalog_registrars/_http_retry.py +++ b/fluid_build/build_runners/catalog_registrars/_http_retry.py @@ -45,12 +45,10 @@ from __future__ import annotations -import email.utils import logging import random import time from dataclasses import dataclass, field -from datetime import datetime, timezone from typing import Callable, FrozenSet, Optional, TypeVar # Retryable HTTP status codes — mirrors DataHub's @@ -115,28 +113,9 @@ def retry_after_seconds(exc: BaseException) -> Optional[float]: if not isinstance(exc, httpx.HTTPStatusError): return None - raw = exc.response.headers.get("Retry-After") - if not raw: - return None - raw = raw.strip() - try: - secs = float(int(raw)) - return secs if secs >= 0 else None - except ValueError: - pass - # HTTP-date form → seconds from now. ``parsedate_to_datetime`` - # returns ``None`` on some Pythons and raises ``ValueError`` / - # ``TypeError`` on others for a malformed value — handle both. - try: - parsed = email.utils.parsedate_to_datetime(raw) - except (ValueError, TypeError): - return None - if parsed is None: - return None - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - delta = (parsed - datetime.now(timezone.utc)).total_seconds() - return max(0.0, delta) + from fluid_build.util.http_client import parse_retry_after + + return parse_retry_after(exc.response.headers.get("Retry-After")) def compute_delay(attempt: int, policy: RetryPolicy, retry_after: Optional[float] = None) -> float: diff --git a/fluid_build/build_runners/duckdb/runner.py b/fluid_build/build_runners/duckdb/runner.py index 203add1b4..4976b8cb4 100644 --- a/fluid_build/build_runners/duckdb/runner.py +++ b/fluid_build/build_runners/duckdb/runner.py @@ -1619,6 +1619,34 @@ def _execute(ctx: RunContext, runner: DuckdbRunner) -> RunResult: ctx, streams_to_run, started_at, t_start, f"bigquery load: {reason}" ) + # A Databricks binding lands the file locally, uploads it to a Unity Catalog volume and + # loads the table fluid apply created (see ``_databricks_load``). Everything it needs is + # checked before any work runs: it must never land somewhere else and succeed. + dbx_landing = None + if _is_databricks_binding(ctx): + from .._databricks_load import ( + DatabricksLoadError, + databricks_load_target, + open_landing, + ) + from .._databricks_load import ( + unsupported_reason as databricks_unsupported_reason, + ) + + try: + expose = _build_expose(ctx) or {} + dbx_target = databricks_load_target(expose.get("binding") or {}, expose) + reason = databricks_unsupported_reason( + _mode_name(ctx), sink_format, len(streams_to_run) + ) + if reason: + raise DatabricksLoadError(reason) + if dbx_target is None: # unreachable: _is_databricks_binding said it is + raise DatabricksLoadError("the binding does not name a Databricks table") + dbx_landing = open_landing(dbx_target) + except DatabricksLoadError as exc: + return _refused_run(ctx, streams_to_run, started_at, t_start, f"databricks load: {exc}") + stream_results: List[StreamResult] = [] failures = 0 records_total = 0 @@ -1870,6 +1898,34 @@ def _execute(ctx: RunContext, runner: DuckdbRunner) -> RunResult: error=f"bigquery load: {type(exc).__name__}: {exc}", ) + # Databricks: the file is on local disk now; upload it and load the table. After the + # late-arrival split, so the rows loaded are the rows that stayed. A failed or short load + # fails the stream, and so the run and the apply. The staged files are deleted either way. + databricks_load: Dict[str, Any] = {} + if dbx_landing is not None and failures == 0 and stream_results: + from ...iac.providers.databricks_enforce import scrub as _scrub_dbx + + for i, sr in enumerate(stream_results): + if sr.state is not RunState.SUCCEEDED: + continue + staged = _resolve_destination_path(ctx, sr.name, sink_format, out_dir) + try: + databricks_load = dbx_landing.load( + staged, + product_id=ctx.product_id, + build_id=ctx.build_id, + run_id=ctx.run_id, + expected_rows=_count_file_rows(staged, sink_format), + logger=LOG, + ) + except Exception as exc: # noqa: BLE001 + failures += 1 + stream_results[i] = dataclasses.replace( + sr, + state=RunState.FAILED, + error=f"databricks load: {type(exc).__name__}: {_scrub_dbx(exc)}", + ) + # ``onError: fail`` mode — when ANY bad rows landed in the DLQ, # promote the run to FAILED so the run record reflects the issue. # The DLQ records are still on disk (audit trail intact); the @@ -1907,6 +1963,8 @@ def _execute(ctx: RunContext, runner: DuckdbRunner) -> RunResult: facets["pii_findings"] = pii_findings if bigquery_load: facets["bigquery_load"] = bigquery_load + if databricks_load: + facets["databricks_load"] = databricks_load rows_from = "write" if counted_at_write else "source_count" if any(split.get("late") for split in late_arrival_split_results.values()): # The split moved rows out of the file the COPY wrote. @@ -1951,11 +2009,15 @@ def _resolve_destination_path( ``_object_store_uri``. For local paths, behavior is unchanged: relative paths are rooted under ``ctx.workdir`` and parent directories are created. A BigQuery binding is not a file destination: - the file is staged locally for the load job, see ``_bigquery_target``. + the file is staged locally for the load job, see ``_bigquery_target``. A + Databricks binding is not one either: its file is staged locally, uploaded to + a volume and loaded into the table, see ``_databricks_staging_path``. """ bq = _bigquery_target(ctx) if bq is not None: return str(_bigquery_staging_path(ctx, bq, sink_format)) + if _is_databricks_binding(ctx): + return str(_databricks_staging_path(ctx, stream, sink_format)) expose = _build_expose(ctx) if expose is not None: binding = expose.get("binding", {}) or {} @@ -2166,9 +2228,10 @@ def _binding_destination_uri(ctx: RunContext) -> Optional[str]: no secret, so the COPY reached S3 unauthenticated and came back 403. None for a BigQuery binding too: its file is staged locally and loaded by a - job, so the ``gs://`` path it may carry is never written to. + job, so the ``gs://`` path it may carry is never written to; and for a + Databricks binding, whose file is staged locally and uploaded to a volume. """ - if _bigquery_target(ctx) is not None: + if _bigquery_target(ctx) is not None or _is_databricks_binding(ctx): return None expose = _build_expose(ctx) if expose is None: @@ -2221,6 +2284,33 @@ def _bigquery_staging_path(ctx: RunContext, target: Dict[str, Any], sink_format: return path +def _is_databricks_binding(ctx: RunContext) -> bool: + """True when the expose this build writes is bound to Databricks. + + Its ``location.path`` (and ``out/``) are never written: the rows land in the table. + """ + from ...iac.provider_match import is_cloud + + expose = _build_expose(ctx) + return expose is not None and is_cloud(expose.get("binding") or {}, "databricks") + + +def _databricks_staging_path(ctx: RunContext, stream: str, sink_format: str) -> Path: + """Where a Databricks-bound build lands its file before the upload. + + Beside the BigQuery staging, under ``.fluid/staging``, and not ``out/``, which is the + local target's own output. Named after the table, as the catalogue names it. + """ + ext = _FILE_FORMAT_EXT.get((sink_format or "parquet").lower(), sink_format) + expose = _build_expose(ctx) or {} + loc = (expose.get("binding") or {}).get("location") or {} + build = _path_part(getattr(ctx, "build_id", None) or "build") + name = _path_part(loc.get("table") or stream) + path = Path(ctx.workdir) / ".fluid" / "staging" / build / f"{name}.{ext}" + path.parent.mkdir(parents=True, exist_ok=True) + return path + + def _path_part(value: Any) -> str: """One safe path component: contract-supplied ids must not climb out of staging.""" return re.sub(r"[^A-Za-z0-9_-]", "_", str(value)) or "_" diff --git a/fluid_build/build_runners/meltano/runner.py b/fluid_build/build_runners/meltano/runner.py index 25dc10565..25b2769f6 100644 --- a/fluid_build/build_runners/meltano/runner.py +++ b/fluid_build/build_runners/meltano/runner.py @@ -112,8 +112,8 @@ def collect_singer_output( # ── Tap invocation ────────────────────────────────────────────────────── -_TAP_NAME_RE = re.compile(r"^tap-[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?$") -_TARGET_NAME_RE = re.compile(r"^target-[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?$") +_TAP_NAME_RE = re.compile(r"^tap-[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?\Z") +_TARGET_NAME_RE = re.compile(r"^target-[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?\Z") def _resolve_tap_binary(tap_name: str, *, project_dir: Optional[Path] = None) -> Optional[str]: diff --git a/fluid_build/cli/__init__.py b/fluid_build/cli/__init__.py index 6e4c65356..68990887a 100644 --- a/fluid_build/cli/__init__.py +++ b/fluid_build/cli/__init__.py @@ -20,7 +20,7 @@ import os import sys import time -from typing import List, Optional +from typing import Any, List, Optional from fluid_build.cli.console import cprint, info, warning from fluid_build.cli.console import error as console_error @@ -152,9 +152,10 @@ def build_parser() -> argparse.ArgumentParser: # Deliberately NO argparse ``choices``. The provider set is a registry # (built-ins + ``fluid_build.providers`` entry-point plugins), and the # hardcoded enum disagreed with it in both directions: it rejected - # ``redshift`` and ``datamesh_manager``, which ``fluid providers`` - # lists, and accepted ``azure``, which is not registered at all — and a - # pip-installed provider plugin could never be passed. Enumerating the + # ``redshift`` and ``datamesh_manager`` (then registered; since retired, + # see ``RETIRED_PROVIDERS``), and accepted ``azure``, which is not + # registered at all — and a pip-installed provider plugin could never be + # passed. Enumerating the # registry here is not an option: discovery imports every provider # module, and ``build_parser()`` is the ``fluid --help`` cold path. # The value is validated against the live registry in @@ -257,6 +258,29 @@ def _force_utf8_streams() -> None: pass +def _render_missing_optional(e: Any, args: argparse.Namespace, context: Any, cli: Any) -> int: + """Print an optional-dependency hint (message + stable slug) and return exit 1.""" + from fluid_build._console import _redact_str, literal_text + from fluid_build._error_catalog import slug_for + + slug = slug_for(e.event) + cli.logger.log_safe( + "error", + "CLI optional dependency missing", + extra={ + "cmd": getattr(args, "cmd", None), + "event": e.event, + "error_slug": slug, + "module": e.module, + "extra": e.extra, + "exit_code": e.exit_code, + }, + ) + context.console.print(literal_text(_redact_str(f"❌ {e}"), "red")) + context.console.print(literal_text(f"[{slug}]", "dim")) + return int(e.exit_code) + + def main(argv: Optional[List[str]] = None) -> int: """Enhanced main entry point with production features and comprehensive error handling""" # Must run before ANY output: the banner below is Unicode-heavy and would @@ -344,12 +368,19 @@ def main(argv: Optional[List[str]] = None) -> int: # `--provider` / `FLUID_PROVIDER` keeps its current startup cost # (tests/perf/test_startup_budget.py). requested_provider = getattr(args, "provider", None) + # A retired name needs no discovery: the answer is the same whatever the + # registry holds, and a registry that failed to load must not hide it. + from fluid_build.providers import retired_provider_hint + + retired = bool(requested_provider) and retired_provider_hint(requested_provider) is not None known_providers = ( _known_provider_names() - if requested_provider and requested_provider != _PROVIDER_AUTO + if requested_provider and requested_provider != _PROVIDER_AUTO and not retired else [] ) _validate_global_args(args, cli.logger, known_providers=known_providers) + if retired: + return 2 if requested_provider and known_providers and requested_provider not in known_providers: return 2 @@ -523,6 +554,16 @@ def main(argv: Optional[List[str]] = None) -> int: raise except Exception as e: # noqa: BLE001 + # An optional package is absent (``fluid_build._optional``). Checked + # here rather than as its own ``except`` clause so the module is only + # imported on the failure path: ``fluid --help`` stays at its budget. + # ``str(e)`` is the whole remedy (`` needs ... pip + # install the extra``): print it and exit 1, no traceback. + from fluid_build._optional import MissingOptionalDependency + + if isinstance(e, MissingOptionalDependency): + return _render_missing_optional(e, args, context, cli) + # The exception text is data, not markup, as in the CLIError branch # above. Parsed as markup, an error quoting ``[/]`` raised # ``MarkupError`` from this handler, and the outer fallback then @@ -750,8 +791,17 @@ def _validate_global_args( # would drag provider discovery onto the `--help` path. Checking it # here means the accepted values are exactly what `fluid providers` # lists, plugins included. - known = _known_provider_names() if known_providers is None else known_providers - if known and args.provider not in known and args.provider != _PROVIDER_AUTO: + from fluid_build.providers import retired_provider_hint + + retired_hint = retired_provider_hint(args.provider) + known = ( + [] + if retired_hint + else (_known_provider_names() if known_providers is None else known_providers) + ) + if retired_hint: + issues.append(retired_hint) + elif known and args.provider not in known and args.provider != _PROVIDER_AUTO: issues.append( f"Unknown provider '{args.provider}' — installed providers: " f"{', '.join(known)} (see `fluid providers`)" diff --git a/fluid_build/cli/_ai_setup_storage.py b/fluid_build/cli/_ai_setup_storage.py index ecef764be..7d74ff9dd 100644 --- a/fluid_build/cli/_ai_setup_storage.py +++ b/fluid_build/cli/_ai_setup_storage.py @@ -115,7 +115,7 @@ def _plaintext_env_var() -> str: # Privacy guard: only a bare domain slug is ever persisted. Mirrors # ``forge_domain_enrichment._SAFE_DOMAIN_RE``, lower-cased and length-capped — # the regex is what keeps prompt text / paths / PII out of the history file. -_DOMAIN_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$") +_DOMAIN_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}\Z") def _allow_plaintext_ai_secrets() -> bool: diff --git a/fluid_build/cli/_apply_opentofu_engine.py b/fluid_build/cli/_apply_opentofu_engine.py index d20b70a50..ba47d00f0 100644 --- a/fluid_build/cli/_apply_opentofu_engine.py +++ b/fluid_build/cli/_apply_opentofu_engine.py @@ -34,7 +34,7 @@ from pathlib import Path from typing import Any, Dict, List, Mapping, Optional, Tuple -from fluid_build.cli.console import cprint +from fluid_build.cli.console import cprint, cprint_literal from fluid_build.iac import build_module, get_iac_plugin, runner from fluid_build.iac.backend import ( STATE_BACKEND_ENV, @@ -106,10 +106,16 @@ def _apply_via_opentofu(args, logger: logging.Logger, contract: Optional[Mapping plugin = get_iac_plugin(provider) if plugin is None: + from fluid_build.iac.registry import iac_plugin_unavailable_reason + + why = iac_plugin_unavailable_reason(provider) raise CLIError( 1, "opentofu_engine_unsupported_provider", - {"error": f"no OpenTofu plugin for provider {provider!r}"}, + { + "error": f"no OpenTofu plugin for provider {provider!r}" + + (f": {why}" if why else "") + }, ) if runner.tofu_path() is None: # `tofu` isn't on PATH. When the operator opted in (--ensure-opentofu, @@ -1074,9 +1080,11 @@ def _enforce_data_loss_gate( changes["remove"], ) if lost_in_place: - cprint( + # The lines quote live objects (a table's column types): data, not markup. + cprint_literal( "\n ⚠️ --allow-data-loss: existing data is changed in place with no snapshot: " - + "; ".join(lost_in_place) + + "; ".join(lost_in_place), + soft_wrap=True, ) logger.warning( "--allow-data-loss: bypassing the data-loss gate for in-place data changes " @@ -1470,12 +1478,11 @@ def _report_plan_notices(plugin: Any, contract: Mapping[str, Any], logger: loggi if not callable(notices): return try: - lines = list(notices(contract)) + for line in notices(contract): + # A notice can quote live objects, so it is data, not markup. + cprint_literal(f" note: {line}") except Exception as exc: # noqa: BLE001 — advisory only, never fails an apply logger.debug("plan notices unavailable: %s", type(exc).__name__) - return - for line in lines: - cprint(f" note: {line}") def _reconcile_with_state( diff --git a/fluid_build/cli/_command_center.py b/fluid_build/cli/_command_center.py index e2eb9ef15..038e80281 100644 --- a/fluid_build/cli/_command_center.py +++ b/fluid_build/cli/_command_center.py @@ -46,14 +46,23 @@ # the same host moved up to https): the reporter's redirect rule. A Command # Center that answers with a redirect to another origin, a private or a cloud # metadata address included, is not followed there. -_PROBE_HOOKS = {"response": refuse_foreign_redirect} +_PROBE_HOOKS = [refuse_foreign_redirect] -try: - import requests - REQUESTS_AVAILABLE = True -except ImportError: - REQUESTS_AVAILABLE = False +def _probe(method: str, url: str, *, timeout: float, **kwargs: Any) -> Any: + """One probe request: a fresh client, the redirect rule on every hop. + + Imported lazily: ``httpx`` stays off the cold ``fluid --help`` path. A HEAD + does not follow redirects (as ``requests.head`` did not); a GET does. + """ + from fluid_build.util.http_client import new_client + + with new_client( + timeout=timeout, + response_hooks=_PROBE_HOOKS, + follow_redirects=method.upper() != "HEAD", + ) as client: + return client.request(method, url, **kwargs) @dataclass @@ -182,10 +191,6 @@ def _check_availability(self) -> None: Performs a health check and queries available endpoints. Sets self.available and self.features. """ - if not REQUESTS_AVAILABLE: - self.logger.warning("requests library not available, skipping Command Center detection") - return - # The reporter's host gate, before any request leaves the process: # the URL is user/config-supplied, and the credential goes to it # elsewhere. The warning names the origin only. @@ -199,10 +204,12 @@ def _check_availability(self) -> None: self.available = False return + import httpx # lazy: off the cold --help path + try: # Quick health check with timeout health_url = f"{self.url}/health" - response = requests.get(health_url, timeout=2, hooks=_PROBE_HOOKS) + response = _probe("GET", health_url, timeout=2) if response.status_code == 200: self.available = True @@ -214,11 +221,11 @@ def _check_availability(self) -> None: self.logger.debug(f"Command Center health check failed: {response.status_code}") self.available = False - except requests.exceptions.Timeout: + except httpx.TimeoutException: self.logger.debug(f"Command Center health check timeout: {self.url}") self.available = False - except requests.exceptions.ConnectionError: + except (httpx.NetworkError, httpx.RemoteProtocolError, httpx.ProxyError): self.logger.debug(f"Command Center not reachable: {self.url}") self.available = False @@ -264,7 +271,7 @@ def _check_endpoint(self, endpoint: str) -> bool: url = f"{self.url}{endpoint}" # Try HEAD request first (faster) - response = requests.head(url, timeout=1, hooks=_PROBE_HOOKS) + response = _probe("HEAD", url, timeout=1) # 200 OK or 405 Method Not Allowed (HEAD not supported but endpoint exists) if response.status_code in (200, 405): @@ -275,7 +282,9 @@ def _check_endpoint(self, endpoint: str) -> bool: return False # For other status codes, try GET with limit - response = requests.get(url, params={"limit": 1}, timeout=1, hooks=_PROBE_HOOKS) + from fluid_build.util.http_client import merge_query + + response = _probe("GET", merge_query(url, {"limit": 1}), timeout=1) return response.status_code in (200, 401, 403) # Exists (may need auth) except Exception as e: @@ -290,12 +299,12 @@ def _get_version(self) -> str: Version string or "unknown" """ try: - response = requests.get(f"{self.url}/api/v1/version", timeout=1, hooks=_PROBE_HOOKS) + response = _probe("GET", f"{self.url}/api/v1/version", timeout=1) if response.status_code == 200: data = response.json() return data.get("version", "unknown") except Exception: - pass + pass # the version is informational; an unreachable server reads "unknown" return "unknown" diff --git a/fluid_build/cli/_common.py b/fluid_build/cli/_common.py index acdd84537..c652de397 100644 --- a/fluid_build/cli/_common.py +++ b/fluid_build/cli/_common.py @@ -211,23 +211,28 @@ def resolve_provider_from_contract(contract: Dict[str, Any]) -> tuple[str, Dict[ falls back to top-level binding.platform (Snowflake-style), then builds[].execution.runtime.platform. + A retired provider name used as a platform (``redshift``) resolves to its + successor provider (``aws``). + Returns: (provider_name, location_dict) where location may contain project, region, dataset, bucket, etc. """ + from fluid_build.providers import resolve_retired_platform + for exp in contract.get("exposes", []): binding = exp.get("binding", {}) platform = binding.get("platform", "") if platform: - return platform, binding.get("location", {}) + return resolve_retired_platform(platform), binding.get("location", {}) # Top-level binding (Snowflake-style contracts) top_binding = contract.get("binding", {}) if isinstance(top_binding, dict) and top_binding.get("platform"): - return top_binding["platform"], top_binding.get("location", {}) + return resolve_retired_platform(top_binding["platform"]), top_binding.get("location", {}) for build in contract.get("builds", []): platform = (build.get("execution") or {}).get("runtime", {}).get("platform", "") if platform: - return platform, {} + return resolve_retired_platform(platform), {} return "", {} @@ -251,7 +256,11 @@ def build_provider( # blocked provider as "unknown" sends the operator hunting for a # typo or a missing extra. from fluid_build.plugin_manager import is_allowed + from fluid_build.providers import retired_provider_hint + hint = retired_provider_hint(name) + if hint: + raise CLIError(2, "provider_retired", {"error": hint}) if not is_allowed(name): raise CLIError(2, "provider_blocked_by_policy", {"requested": name}) raise CLIError( diff --git a/fluid_build/cli/_forge_data_model_jdbc.py b/fluid_build/cli/_forge_data_model_jdbc.py index 6daddcfac..0be635a48 100644 --- a/fluid_build/cli/_forge_data_model_jdbc.py +++ b/fluid_build/cli/_forge_data_model_jdbc.py @@ -36,7 +36,7 @@ # Canonical FLUID contract-id pattern (lowercased form — ``product_id`` is # lowercased before the check). Mirrors the JSON-schema id constraint. -_FLUID_ID_RE = re.compile(r"^[a-z0-9_][a-z0-9_.-]*[a-z0-9_]$|^[a-z0-9_]$") +_FLUID_ID_RE = re.compile(r"^[a-z0-9_][a-z0-9_.-]*[a-z0-9_]\Z|^[a-z0-9_]\Z") def _run_from_jdbc_source( diff --git a/fluid_build/cli/_preview_panel.py b/fluid_build/cli/_preview_panel.py index 572ee4780..bd1320b1e 100644 --- a/fluid_build/cli/_preview_panel.py +++ b/fluid_build/cli/_preview_panel.py @@ -714,7 +714,10 @@ def render_refine_diff( try: from fluid_build.api.changelog import compare_contracts, render_text - report = compare_contracts(old_contract, new_contract) + # Not validated: the existing contract is often invalid (that can be why + # it is being refined), and the new one passed the runtime's schema + # gate already. ``fluid diff --baseline`` validates both. + report = compare_contracts(old_contract, new_contract, validate=False) except Exception: # noqa: BLE001 — diff is advisory; never block the write LOG.debug("render_refine_diff: changelog engine unavailable", exc_info=True) return False @@ -743,13 +746,18 @@ def render_refine_diff( try: from rich.panel import Panel as _RichPanel + from fluid_build._console import literal_text + + # The body quotes column names: a ``str`` would be read as markup. border = "red" if report.has_breaking else "yellow" - console.print(_RichPanel(body, title=f"[bold]{title}[/bold]", border_style=border)) + console.print( + _RichPanel(literal_text(body), title=f"[bold]{title}[/bold]", border_style=border) + ) return True except Exception: # noqa: BLE001 — fall through to plain text try: console.print(f"\n=== {title} ===") - console.print(body) + console.print(body, markup=False, emoji=False) return True except Exception: # noqa: BLE001 pass diff --git a/fluid_build/cli/_template_mode.py b/fluid_build/cli/_template_mode.py index 00679778f..bf05b26e3 100644 --- a/fluid_build/cli/_template_mode.py +++ b/fluid_build/cli/_template_mode.py @@ -301,6 +301,26 @@ def _create_project_agent_loop( return False +def _is_missing_optional(exc: BaseException) -> bool: + """True when *exc* is, or was caused by, a missing optional dependency.""" + from fluid_build._optional import MissingOptionalDependency + + cur: Optional[BaseException] = exc + for _ in range(8): + if cur is None: + return False + if isinstance(cur, MissingOptionalDependency): + return True + cur = cur.__cause__ or cur.__context__ + return False + + +def _rich_escape(text: str) -> str: + from rich.markup import escape + + return escape(text) + + def _emit_incomplete_forge_span(logger: logging.Logger) -> None: """Emit a ``run_completed=False`` ``forge.invocation`` span on failure. @@ -1018,13 +1038,18 @@ def _serialize_logical_model(value: Any) -> Optional[str]: return True except CopilotGenerationError as exc: - logger.exception("AI Copilot minimal flow failed") + if _is_missing_optional(exc): + # An absent optional package is an install problem, not a crash: + # the message and hint are the whole story, so no traceback. + logger.error("AI Copilot minimal flow unavailable: %s", exc.message) + else: + logger.exception("AI Copilot minimal flow failed") _emit_incomplete_forge_span(logger) if console: try: console.print(f"[red]❌ {exc.message}[/red]") for suggestion in getattr(exc, "suggestions", []) or []: - console.print(f"[dim]• {suggestion}[/dim]") + console.print(f"[dim]• {_rich_escape(str(suggestion))}[/dim]") except Exception: # noqa: BLE001 pass else: diff --git a/fluid_build/cli/_verify_databricks.py b/fluid_build/cli/_verify_databricks.py index 051c555ff..4808f834c 100644 --- a/fluid_build/cli/_verify_databricks.py +++ b/fluid_build/cli/_verify_databricks.py @@ -48,9 +48,20 @@ or the table's owner, and its body, parameters and return type the expected ones, since a content-hashed name is computable from the contract and could have been taken by someone else's function. ``skip`` when the contract restricts nothing. -5. **The jobs.** Each dbt build that writes the expose has exactly one job of +5. **Retention.** An expose with ``lifecycle.expire`` is accepted because its table is rebuilt in + full and the contract's retention covers its Delta history (``databricks_retention.py``). + The table's live ``delta.logRetentionDuration`` and ``delta.deletedFileRetentionDuration`` + (``properties`` and ``delta_runtime_properties_kvpairs`` of the GetTable already read; + unset means Delta's 30 and 7 days) must not make the history longer than the contract's + retention: CRITICAL if they do, with the ``ALTER TABLE`` that sets them. Predictive + optimization, which runs ``VACUUM`` on managed tables, off for the table + (``effective_predictive_optimization_flag``) is a warning: nothing removes the old files + until a ``VACUUM`` runs. ``skip`` when the contract sets no expire. +6. **The jobs.** Each dbt build that writes the expose has exactly one job of its name. None means ``fluid apply`` did not run (or the job was deleted); - several mean an apply without its state created a duplicate. + several mean an apply without its state created a duplicate. A job that is paused while + its table's masks or row filter are not in place fails too, with the apply that binds them + and starts the job as the fix. The client is ``databricks.sdk.WorkspaceClient`` built from ``sdk_config`` (the credential policy the provider and the doctor share), used only through its @@ -63,6 +74,7 @@ from __future__ import annotations +import copy import os from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence from urllib.parse import quote @@ -384,6 +396,118 @@ def _residency_dimension(client: Any, table: Any) -> Dict[str, Any]: } +def _live_properties(live_table: Mapping[str, Any]) -> Dict[str, Any]: + """The table's properties as GetTable serves them: ``properties``, with the Delta runtime + ones (``delta_runtime_properties_kvpairs``, the table's current state) over them.""" + props: Dict[str, Any] = {} + declared = live_table.get("properties") + if isinstance(declared, Mapping): + props.update(declared) + runtime = live_table.get("delta_runtime_properties_kvpairs") + runtime = runtime.get("delta_runtime_properties") if isinstance(runtime, Mapping) else None + if isinstance(runtime, Mapping): + props.update(runtime) + return props + + +def _predictive_optimization(live_table: Mapping[str, Any]) -> Optional[str]: + """``ENABLE`` or ``DISABLE`` for the table, as Unity Catalog reports it (the effective flag, + else the table's own unless it inherits); ``None`` when it does not say.""" + effective = live_table.get("effective_predictive_optimization_flag") + value = effective.get("value") if isinstance(effective, Mapping) else None + if not value: + value = live_table.get("enable_predictive_optimization") + value = str(value or "").upper() + return value if value in {"ENABLE", "DISABLE"} else None + + +def _retention_dimension(table: Any, live_table: Mapping[str, Any]) -> Dict[str, Any]: + """The table's live Delta history retention against the contract's ``lifecycle.retention``. + + ``skip`` when the contract sets no expire: nothing was promised. ``fail`` when the longer + of ``delta.logRetentionDuration`` and ``delta.deletedFileRetentionDuration`` (each at + Delta's default when unset) exceeds the contract's retention, or one is not an interval. + ``warn`` (not a failure) when predictive optimization is off for the table. + """ + from fluid_build.iac.providers.databricks_retention import ( + DELETED_FILE_RETENTION, + LOG_RETENTION, + days_text, + history_seconds, + ) + + promised = getattr(table, "retention_seconds", None) + if promised is None: + return { + "status": "skip", + "message": "the contract sets no lifecycle.expire to check the Delta history against", + "findings": [], + "warnings": [], + } + period = getattr(table, "retention_period", None) + optimization = _predictive_optimization(live_table) + quoted = ".".join(_quote_name(part) for part in table.full_name.split(".")) + schema = ".".join(_quote_name(part) for part in table.full_name.split(".")[:2]) + warnings: List[str] = [] + if optimization == "DISABLE": + warnings.append( + f"predictive optimization is off for {table.full_name}, so nothing runs VACUUM and " + "the old data files stay past the retention. Enable it " + f"(ALTER SCHEMA {schema} ENABLE PREDICTIVE OPTIMIZATION), or schedule " + f"VACUUM {quoted}" + ) + result: Dict[str, Any] = { + "retention": period, + "predictive_optimization": optimization, + "findings": [], + "warnings": warnings, + } + try: + effective = history_seconds(_live_properties(live_table)) + except ValueError as exc: + return { + **result, + "status": "fail", + "message": f"a Delta retention property of {table.full_name} cannot be read: {exc}", + "findings": [{"message": str(exc)}], + "fix": ( + f"Set {LOG_RETENTION} and {DELETED_FILE_RETENTION} on the table to " + "'interval N days'" + ), + } + result["history_seconds"] = effective + if "properties" not in live_table and "delta_runtime_properties_kvpairs" not in live_table: + warnings.append( + f"Unity Catalog returned no table properties for {table.full_name}, so Delta's " + f"defaults were assumed. Check them with SHOW TBLPROPERTIES {quoted}" + ) + if effective > promised: + days = max(promised // 86400, 1) + fix = ( + f"ALTER TABLE {quoted} SET TBLPROPERTIES ('{LOG_RETENTION}' = 'interval {days} " + f"days', '{DELETED_FILE_RETENTION}' = 'interval {days} days'), or raise " + "lifecycle.retention" + ) + message = ( + f"the Delta history of {table.full_name} is {days_text(effective)}, longer than " + f"the contract's retention {period} ({days_text(promised)})" + ) + return { + **result, + "status": "fail", + "message": message, + "findings": [{"message": message}], + "fix": fix, + } + message = ( + f"the Delta history of {table.full_name} is {days_text(effective)}, within the " + f"contract's retention {period} ({days_text(promised)})" + ) + if warnings: + return {**result, "status": "warn", "message": message, "fix": warnings[0]} + return {**result, "status": "pass", "message": message} + + def _tags_dimension(client: Any, table: Any) -> Dict[str, Any]: """The governance tags the module sets on the table and its schema, against the live ones. @@ -609,8 +733,31 @@ def _function_findings( return out -def _jobs_dimension(client: Any, expose_id: str, contract: Mapping[str, Any], jobs: Any) -> Any: - """The dimension for the dbt jobs that write ``expose_id``; ``None`` when there are none.""" +def _paused(job: Mapping[str, Any]) -> bool: + """Whether the job's schedule or continuous block is ``PAUSED``.""" + settings = job.get("settings") + if not isinstance(settings, Mapping): + return False + return any( + isinstance(settings.get(block), Mapping) and settings[block].get("pause_status") == "PAUSED" + for block in ("schedule", "continuous") + ) + + +def _jobs_dimension( + client: Any, + expose_id: str, + contract: Mapping[str, Any], + jobs: Any, + enforcement: Optional[Mapping[str, Any]] = None, + table_name: str = "", +) -> Any: + """The dimension for the dbt jobs that write ``expose_id``; ``None`` when there are none. + + A job that is paused while the table's masks or row filter are not in place is the one + ``fluid apply`` creates paused for that reason (``DatabricksIacPlugin.emit``): it is a finding + whose fix is the apply that binds them and starts it. + """ from fluid_build.util.contract import builds_writing_expose writers = { @@ -633,6 +780,17 @@ def _jobs_dimension(client: Any, expose_id: str, contract: Mapping[str, Any], jo "run fluid apply", } ) + elif len(hits) == 1 and _paused(hits[0]) and (enforcement or {}).get("status") == "fail": + findings.append( + { + "build": job.build_id, + "message": ( + f"job {job.name!r} is paused because {table_name or 'its table'} is not " + "protected (the row_filter_masks dimension fails): run fluid apply, which " + "binds the masks and row filter, then starts the job" + ), + } + ) elif len(hits) > 1: findings.append( { @@ -664,6 +822,7 @@ def _severity( residency: Optional[Mapping[str, Any]] = None, tags: Optional[Mapping[str, Any]] = None, enforcement: Optional[Mapping[str, Any]] = None, + retention: Optional[Mapping[str, Any]] = None, ) -> Dict[str, Any]: """The shared drift grading, raised to CRITICAL by a missing table, reader, job, a workspace outside the declared region, a governance tag that is not there or a row @@ -699,6 +858,9 @@ def _severity( enforcement["message"] + ": " + "; ".join(f["message"] for f in enforcement["findings"]) ) actions.append(enforcement["fix"]) + if (retention or {}).get("status") == "fail": + reasons.append(retention["message"]) + actions.append(retention["fix"]) if not reasons: return severity already = severity["level"] == "CRITICAL" @@ -742,11 +904,15 @@ def verify_databricks_expose( gives one (an external pipeline owns its creation). ``client_factory()`` defaults to ``WorkspaceClient(**sdk_config(os.environ))``. """ + from fluid_build.cli._common import resolve_env_templates_in_contract from fluid_build.iac.base import UnsupportedBindingError from fluid_build.iac.providers.databricks import managed_objects try: - tables, jobs = managed_objects(contract, os.environ) + # What fluid apply provisions, from the contract as apply reads it: with its + # ``{{ env.X }}`` placeholders substituted. + resolved = resolve_env_templates_in_contract(copy.deepcopy(dict(contract or {}))) + tables, jobs = managed_objects(resolved, os.environ) except UnsupportedBindingError as exc: return _error(f"{exc} ({exc.kind})", remediation=tuple(exc.remediation)) except Exception as exc: # noqa: BLE001 — the emitter's refusal is the result @@ -801,7 +967,8 @@ def verify_databricks_expose( residency = _residency_dimension(client, table) tags = _tags_dimension(client, table) enforcement = _enforcement_dimension(table, live_table, client) - job_dim = _jobs_dimension(client, expose_id, contract, jobs) + retention = _retention_dimension(table, live_table) + job_dim = _jobs_dimension(client, expose_id, contract, jobs, enforcement, target) except Exception as exc: # noqa: BLE001 — a check that could not run is an error return _error( f"Databricks check for {target} could not run: {_failure(exc)}", @@ -813,6 +980,7 @@ def verify_databricks_expose( dimensions["residency"] = residency dimensions["governance_tags"] = tags dimensions["row_filter_masks"] = enforcement + dimensions["retention"] = retention if job_dim is not None: dimensions["jobs"] = job_dim @@ -826,6 +994,7 @@ def verify_databricks_expose( "residency", "governance_tags", "row_filter_masks", + "retention", "jobs", ) if name in dimensions @@ -836,7 +1005,13 @@ def verify_databricks_expose( "table_id": target, "target": target, "severity": _severity( - diff, access, job_dim, residency=residency, tags=tags, enforcement=enforcement + diff, + access, + job_dim, + residency=residency, + tags=tags, + enforcement=enforcement, + retention=retention, ), "dimensions": dimensions, "metadata": { diff --git a/fluid_build/cli/_welcome_scan.py b/fluid_build/cli/_welcome_scan.py index ecacdb7e2..4c2f58e14 100644 --- a/fluid_build/cli/_welcome_scan.py +++ b/fluid_build/cli/_welcome_scan.py @@ -29,28 +29,51 @@ un-inferred bits. Recurring users (≥5 forges) skip the welcome entirely and land straight in the interview. -50ms scan budget — every probe is best-effort, never blocking. +Every probe is best-effort and never raises. Budget: once the one-time +module imports are done, the probes share ``_TOTAL_TIMEOUT_S`` of wall-clock +and the scan returns by then, dropping any probe still running. The dbt probe +sets the floor: with Python dbt-core on PATH it usually waits out +``_DBT_VERSION_TIMEOUT_S`` and reports nothing. ``scan_duration_ms`` is the +measured wall-clock of the whole call, imports included. """ from __future__ import annotations +import importlib import json import logging import os import shutil +import threading import time -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future, wait from dataclasses import asdict, dataclass, field from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Tuple LOG = logging.getLogger(__name__) -# Probe budgets — every individual probe must finish under this. -_PROBE_TIMEOUT_S = 0.4 -# Aggregate scan budget — shut down probes that overrun. +# Wall-clock budget for the whole scan, counted once the cold imports are done +# (see ``_warm_probe_imports``). A probe still running at the deadline is dropped. _TOTAL_TIMEOUT_S = 1.0 +# ``dbt --version`` cap. Fusion answers in milliseconds, Python dbt-core in +# seconds, so a slow binary simply reports nothing. Independent of the scan +# budget (it sizes one subprocess, not the scan) but it must stay below it: the +# subprocess is then killed by its own timeout before the scan can abandon it. +_DBT_VERSION_TIMEOUT_S = 0.32 +# Modules the probes import lazily, paid up front by ``_warm_probe_imports``. +# They stay function-local in the probes, not imported here, so this module is +# cheap to import. ``tests/test_welcome_scan_budget.py`` fails if a probe starts +# importing something that is not listed here. +_WARM_MODULES = ( + "fluid_build.cli.workspace_config", # _probe_workspace + "fluid_build.cli.ai_setup", # _probe_ai_credentials + # ``_load_ai_config`` imports it lazily; it pulls in pydantic and is most of + # the AI probe's cold cost (~0.1s CPU, several times that on a loaded host). + "fluid_build.copilot.unified_config", + "fluid_build.build_runners.dbt.runner", # _probe_dbt_engine +) # Repeat-user threshold — at this many forges we stop showing the panel. _REPEAT_USER_THRESHOLD = 5 # File extensions we recognise as candidate sample data. @@ -244,11 +267,11 @@ def _probe_dbt_engine() -> Dict[str, Any]: """Detect the dbt engine flavor (Python dbt-core v1 vs Fusion / v2). Fusion (Rust) answers ``dbt --version`` in milliseconds; Python - dbt-core takes seconds, so the subprocess gets a sub-budget timeout - and a slow binary simply reports nothing — the CLIs row then shows a - plain ``dbt`` exactly as before. The detection cache is keyed on the - timeout too, so this short-budget miss never poisons the build - runner's full-budget probe. + dbt-core takes seconds, so the subprocess gets its own short timeout + (``_DBT_VERSION_TIMEOUT_S``) and a slow binary simply reports nothing — + the CLIs row then shows a plain ``dbt`` exactly as before. The detection + cache is keyed on the timeout too, so this short-timeout miss never + poisons the build runner's full-timeout probe. """ try: from fluid_build.build_runners.dbt.runner import ( @@ -259,7 +282,7 @@ def _probe_dbt_engine() -> Dict[str, Any]: dbt_bin = _resolve_dbt_executable() if not dbt_bin: return {} - flavor, version = _detect_dbt_engine(dbt_bin, timeout=_PROBE_TIMEOUT_S * 0.8) + flavor, version = _detect_dbt_engine(dbt_bin, timeout=_DBT_VERSION_TIMEOUT_S) if flavor == "unknown": return {} return {"dbt_engine": f"{flavor} {version}".strip()} @@ -381,18 +404,68 @@ def _probe_specialization_suggestion(start: Path) -> Dict[str, Any]: # --------------------------------------------------------------------------- +def _warm_probe_imports() -> None: + """Import every module in ``_WARM_MODULES`` (best-effort, idempotent). + + A cold import is CPU work, not the I/O wait the scan budget exists for, and + a loaded machine stretches it several times over. Paying it here, before + the budget clock starts, keeps the first scan of a process from dropping + results that a warm scan returns. + """ + for name in _WARM_MODULES: + try: + importlib.import_module(name) + except Exception as exc: # noqa: BLE001 — best-effort; the probe reports its own failure + LOG.debug("welcome_scan_warm_failed: %s — %s", name, exc) + + +def _spawn_probe( + name: str, fn: Callable[..., Dict[str, Any]], args: Tuple[Any, ...] +) -> Future[Dict[str, Any]]: + """Run ``fn(*args)`` on a daemon thread and return a Future for its result. + + Not a ``ThreadPoolExecutor``: its workers are non-daemon and joined at + interpreter exit (documented in ``concurrent.futures``), so a probe the scan + gave up on would keep the CLI alive until it finished. Same idiom as the + non-SIGALRM path of ``ProcessManager.run_with_timeout`` (``cli/security.py``), + with a Future per probe so the caller can wait on all of them against one + deadline. + + Probes must not write to stdout/stderr: a daemon thread caught mid-write + when the interpreter exits can abort the process. Only ``Exception`` is + recorded; a probe that ends its thread with ``SystemExit`` or + ``KeyboardInterrupt`` is simply dropped at the deadline. + """ + future: Future[Dict[str, Any]] = Future() + + def _run() -> None: + try: + future.set_result(fn(*args)) + except Exception as exc: # noqa: BLE001 — recorded on the Future, never raised here + future.set_exception(exc) + + threading.Thread(target=_run, name=f"welcome-scan-{name}", daemon=True).start() + return future + + def run_welcome_scan(*, start: Optional[Path] = None) -> WelcomeFindings: """Run every probe in parallel and return a :class:`WelcomeFindings`. - Total wall-clock <= ``_TOTAL_TIMEOUT_S``. Each individual probe runs - under ``_PROBE_TIMEOUT_S``. A probe that times out is silently - dropped — its fields stay at their defaults. + The probes start together on daemon threads. The cold imports they trigger + finish first (``_warm_probe_imports``, overlapping the probes' own work, + the dbt subprocess included); the probes then get ``_TOTAL_TIMEOUT_S`` of + wall-clock. A probe still running at the deadline is dropped — its fields + stay at their defaults — and abandoned, so a hung probe delays neither the + prompt nor the process exit. + + Wall-clock is the cold imports plus at most ``_TOTAL_TIMEOUT_S``; + ``scan_duration_ms`` reports all of it. """ target = (start or Path.cwd()).resolve() findings = WelcomeFindings() - started = time.time() + started = time.monotonic() - probes = { + probes: Dict[str, Tuple[Callable[..., Dict[str, Any]], Tuple[Any, ...]]] = { "workspace": (_probe_workspace, (target,)), "ai": (_probe_ai_credentials, ()), "clis": (_probe_clis, ()), @@ -404,21 +477,22 @@ def run_welcome_scan(*, start: Optional[Path] = None) -> WelcomeFindings: "specialization": (_probe_specialization_suggestion, (target,)), } - with ThreadPoolExecutor(max_workers=len(probes)) as pool: - futures = {name: pool.submit(fn, *args) for name, (fn, args) in probes.items()} - deadline = started + _TOTAL_TIMEOUT_S - for name, fut in futures.items(): - remaining = max(0.0, deadline - time.time()) - try: - result = fut.result(timeout=min(_PROBE_TIMEOUT_S, remaining)) - except Exception as exc: # noqa: BLE001 — concurrent.futures.TimeoutError + others - LOG.debug("welcome_scan_probe_failed: %s — %s", name, exc) - continue - for key, value in (result or {}).items(): - if hasattr(findings, key): - setattr(findings, key, value) + futures = {name: _spawn_probe(name, fn, args) for name, (fn, args) in probes.items()} + _warm_probe_imports() + done, _ = wait(futures.values(), timeout=_TOTAL_TIMEOUT_S) + for name, future in futures.items(): + if future not in done: + LOG.debug("welcome_scan_probe_timeout: %s", name) + continue + exc = future.exception() + if exc is not None: + LOG.debug("welcome_scan_probe_failed: %s — %s", name, exc) + continue + for key, value in (future.result() or {}).items(): + if hasattr(findings, key): + setattr(findings, key, value) - findings.scan_duration_ms = int((time.time() - started) * 1000) + findings.scan_duration_ms = int((time.monotonic() - started) * 1000) return findings diff --git a/fluid_build/cli/ai_setup.py b/fluid_build/cli/ai_setup.py index 4177d2f07..8fc120e87 100644 --- a/fluid_build/cli/ai_setup.py +++ b/fluid_build/cli/ai_setup.py @@ -241,12 +241,16 @@ def _validate_api_key(provider: Any, api_key: str) -> Optional[str]: """Lightweight API call to verify the key works. One litellm completion validates every provider — there's no - per-provider httpx logic anymore. litellm is a core dep so the - import always resolves. + per-provider httpx logic anymore. litellm is an optional dependency + (``[litellm]``): when it is absent this raises + :class:`~fluid_build._optional.MissingOptionalDependency`, which the CLI + renders as a one-line install hint. Returns ``None`` on success or a short human-readable error message. """ - import litellm # core dep + from fluid_build._optional import require + + litellm = require("litellm", "litellm", "Validating an AI provider key") from fluid_build.cli.forge_copilot_llm_litellm import ( _LITELLM_PREFIX_BY_PROVIDER, @@ -338,17 +342,13 @@ def _pick_provider(console: Any) -> Optional[str]: Returns the canonical provider key, or ``None`` if the user skipped. """ - from fluid_build.cli.forge_ui import ask_numbered_choice - # Detect litellm availability once so we can offer the broader # list when present, and fall back to the native four when it # isn't installed. - try: - import litellm # noqa: F401 + from fluid_build._optional import is_available + from fluid_build.cli.forge_ui import ask_numbered_choice - litellm_available = True - except ImportError: - litellm_available = False + litellm_available = is_available("litellm") base_choices: List[Tuple[str, str]] = [ ("gemini", "Google Gemini (free tier available)"), diff --git a/fluid_build/cli/apply.py b/fluid_build/cli/apply.py index f2952a31e..080c08a12 100644 --- a/fluid_build/cli/apply.py +++ b/fluid_build/cli/apply.py @@ -308,6 +308,85 @@ def _gate_contract_for_apply(contract: Dict[str, Any], logger: logging.Logger) - _gate_contract_for_plan_or_apply(contract, logger, command="apply") +def _gate_build_ids_for_apply(contract: Dict[str, Any]) -> None: + """Refuse, before anything is applied, an id the build phase would refuse. + + The build phase turns ``contract.id`` and ``builds[].id`` into directory + names, so it takes a narrower grammar than the schema: ``1orders``, or an + id of 129 characters, passes ``fluid validate`` and ``fluid plan``. The + build runner checks the ids itself, but on the OpenTofu engine it runs + after ``tofu apply``, so such an id was refused once the product was + provisioned, as an "Unexpected error". This is the runner's own check + (``build_runners._ids.validate_build_ids``), run ahead of the hooks, + ``tofu`` and the runner on both engines. + + Raises: + CLIError: ``apply_build_identifier_invalid``. + """ + from fluid_build.build_runners._ids import IdentifierViolation, validate_build_ids + + try: + validate_build_ids(contract) + except IdentifierViolation as exc: + raise CLIError( + 1, + "apply_build_identifier_invalid", + { + "field": exc.kind, + "value": repr(exc.value), + "hint": ( + "The build phase keeps each build's run records, cursors and " + "watermarks under .fluid/runs///, so both ids " + "must start with a letter or underscore, use only letters, digits, " + "'_', '.' and '-', and be at most 128 characters. The contract schema " + "allows more, so fluid validate and fluid plan accept it. Nothing was " + "applied." + ), + }, + ) from exc + + +def _gate_selected_build_for_apply(contract: Dict[str, Any], build_id: Optional[str]) -> None: + """Refuse, before anything is applied, a ``--build-id`` that names no build. + + The build runner keeps the builds whose id equals ``--build-id`` and, with + none left, logs "Build not found" and returns 1. On the OpenTofu engine it + runs after ``tofu apply``, so a mistyped id provisioned the product and + then failed. This is the runner's own match + (``build_runners.base.run_builds_from_args``), run ahead of the hooks, + ``tofu`` and the runner on both engines. Like the runner, it checks nothing + for an empty ``--build-id``, which filters nothing, or for a contract with + no builds, where the runner returns before it filters. + + Raises: + CLIError: ``apply_build_id_not_found``. + """ + builds = contract.get("builds") or [] + if not (isinstance(build_id, str) and build_id) or not builds: + return + # The runner's filter: an exact match, case and spaces included. + if any(isinstance(build, dict) and build.get("id") == build_id for build in builds): + return + available = [build["id"] for build in builds if isinstance(build, dict) and build.get("id")] + if available: + why, fix = "no build of the contract has that id", "Pass one of the available ids" + else: + why, fix = "none of the contract's builds declares an id", "Give the build you mean an id" + raise CLIError( + 1, + "apply_build_id_not_found", + { + "build_id": repr(build_id), + "available": available, + "hint": ( + "--build-id selects the build whose id equals it exactly, case and spaces " + f"included, and {why}. {fix}, or drop --build-id to run every build. " + "Nothing was applied." + ), + }, + ) + + def _log_apply_failure(logger: logging.Logger, start_time: float, error: Exception) -> None: """Record a pre-dispatch ``apply_contract`` failure for the structured log.""" log_operation_failure(logger, "apply_contract", error=error, duration=time.time() - start_time) @@ -1208,7 +1287,10 @@ def _declared_platforms(contract: Dict[str, Any]) -> Tuple[List[_Declared], List ) def _key(platform: Any) -> str: - return str(platform).strip().lower().replace("-", "_") + from fluid_build.providers import resolve_retired_platform + + # ``redshift`` (retired as a provider) is served by its successor, ``aws``. + return resolve_retired_platform(str(platform).strip().lower().replace("-", "_")) builds = [b for b in contract.get("builds") or [] if isinstance(b, dict)] lands: List[_Declared] = [] @@ -1234,7 +1316,7 @@ def _simple_apply_provider(contract: Dict[str, Any], logger: logging.Logger) -> A ``binding.platform`` says where an expose lands and a build's ``execution.runtime.platform`` where it runs; neither is a provider name, - though ``local``, ``gcp``, ``aws``, ``snowflake`` and ``redshift`` are also + though ``local``, ``gcp``, ``aws`` and ``snowflake`` are also providers. Read as one, a Kafka binding or a Flink build's ``runtime.platform: kubernetes`` ended the run as ``provider_unknown``. @@ -1961,8 +2043,15 @@ def run(args, logger: logging.Logger) -> int: # IaC plugin clouds) and both engines' ``needs_build`` returns used to # skip it, so a schema-invalid contract reached ``tofu`` or the build # runner and could exit 0. One call per run, on the one loaded copy. + # A build mode also checks the ids the build phase turns into paths, which + # the schema admits more of, and then that ``--build-id`` names a build, in + # the runner's order: on the OpenTofu engine the runner would only refuse + # either after ``tofu apply``. try: _gate_contract_for_apply(source_contract, logger) + if needs_build(resolved_mode): + _gate_build_ids_for_apply(source_contract) + _gate_selected_build_for_apply(source_contract, resolved_build_id) except CLIError as exc: _log_apply_failure(logger, start_time, exc) raise diff --git a/fluid_build/cli/config.py b/fluid_build/cli/config.py index e9f9150ca..7abc3644d 100644 --- a/fluid_build/cli/config.py +++ b/fluid_build/cli/config.py @@ -457,7 +457,7 @@ def create_production_documentation() -> str: ### Required Environment Variables ```bash # Infrastructure (Required) -export FLUID_PROVIDER="gcp" # or aws, azure, snowflake +export FLUID_PROVIDER="gcp" # or aws, databricks, snowflake export FLUID_PROJECT="your-project-id" # Cloud project identifier export FLUID_REGION="us-central1" # Cloud region diff --git a/fluid_build/cli/contract_validation.py b/fluid_build/cli/contract_validation.py index 6c7c63bc9..85a601350 100644 --- a/fluid_build/cli/contract_validation.py +++ b/fluid_build/cli/contract_validation.py @@ -111,7 +111,7 @@ def _dist_installed(dist_name: str) -> bool: ), "azure": ( "Provider 'azure' validation not yet implemented", - "Supported providers: gcp, snowflake, aws, local. Others coming soon.", + "Supported providers: aws, databricks, gcp, local, snowflake. Others coming soon.", ), } diff --git a/fluid_build/cli/dbt_mcp.py b/fluid_build/cli/dbt_mcp.py index 61e9e57df..721da3932 100644 --- a/fluid_build/cli/dbt_mcp.py +++ b/fluid_build/cli/dbt_mcp.py @@ -55,6 +55,7 @@ from typing import Any, AsyncIterator, Dict, List, Mapping, Optional, Tuple from fluid_build._mcp_compat import attr as _mcp_attr +from fluid_build._optional import MissingOptionalDependency LOG = logging.getLogger("fluid.cli.dbt_mcp") @@ -184,12 +185,10 @@ def _open_session(self): @asynccontextmanager async def _open_real_session(self) -> AsyncIterator[Any]: - try: - from mcp import ClientSessionGroup, StdioServerParameters - except ImportError as exc: # pragma: no cover - mcp is a base dep - raise RuntimeError( - "The dbt MCP delegate requires the 'mcp' SDK (pip install 'data-product-forge[mcp]')." - ) from exc + from fluid_build._optional import require + + require("mcp", "mcp", "The dbt MCP delegate") + from mcp import ClientSessionGroup, StdioServerParameters params = StdioServerParameters( command=self.command, @@ -255,7 +254,12 @@ def dbt_mcp_tool_definitions(env: Optional[Mapping[str, str]] = None) -> List[Di except Exception as exc: # noqa: BLE001 - never break tool listing # Type-only log (no message interpolation) — the dbt-mcp server's # stderr may carry connection-string-shaped text. - LOG.warning("dbt MCP tool discovery unavailable: %s", type(exc).__name__) + if isinstance(exc, MissingOptionalDependency): + # Carries no secrets (a fixed install hint), and it is the one + # thing the user needs to see. + LOG.warning("dbt MCP tool discovery unavailable: %s", exc) + else: + LOG.warning("dbt MCP tool discovery unavailable: %s", type(exc).__name__) return [] @@ -274,6 +278,9 @@ def dispatch_dbt_mcp_tool( try: return DbtMcpClient(env=env).call_tool(bare, arguments or {}) except Exception as exc: # noqa: BLE001 + if isinstance(exc, MissingOptionalDependency): + LOG.warning("dbt MCP tool %s failed: %s", name, exc) + return {"error": type(exc).__name__, "message": str(exc)} LOG.warning("dbt MCP tool %s failed: %s", name, type(exc).__name__) return { "error": type(exc).__name__, diff --git a/fluid_build/cli/diff.py b/fluid_build/cli/diff.py index b1ec2e8e2..405799552 100644 --- a/fluid_build/cli/diff.py +++ b/fluid_build/cli/diff.py @@ -17,12 +17,16 @@ import argparse import copy import logging +import math import os +import sys import time from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set +from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set +from .._console import _redact_str from .._errors import FluidUserError +from ..observability.secret_redactor import redact_secret_text from ..observability.tracing import traced_span as _traced_span from ..observability.tracing import traced_stage as _traced_stage from ._common import ( @@ -34,12 +38,14 @@ resolve_provider_from_contract, write_json, ) -from ._logging import info, warn +from ._logging import error, info, warn from .console import cprint if TYPE_CHECKING: from fluid_build.iac.drift import StateDriftReport + from ..api.changelog import ContractValidationError + from ..api.changelog_types import ChangelogReport from ._diff_live import LiveDriftReport COMMAND = "diff" @@ -164,7 +170,8 @@ def register(subparsers: argparse._SubParsersAction): help=( "Path to a baseline (old) contract.fluid.yaml. When set, switches " "from drift mode to contract-vs-contract version diff and emits a " - "breaking-change classification." + "breaking-change classification. Both contracts must be schema-valid: " + "an invalid one exits 2 (diff_contract_invalid) before any output." ), ) p.add_argument( @@ -779,11 +786,11 @@ def _run_version_diff(args, logger: logging.Logger) -> int: Loads two contracts, runs the changelog engine, prints in the requested format, optionally writes a JSON envelope to ``--out``, and returns a non-zero exit code when ``--fail-on-breaking`` is set and any breaking - change was detected. + change was detected. A contract that is not schema-valid exits 2 (diff(1)'s + "trouble") before anything is printed or written, so it never reads as 0 + (no changes) or 1 (breaking). """ - from fluid_build.cli.console import cprint - - from ..api.changelog import compare_contracts, render_markdown, render_text + from ..api.changelog import ContractValidationError, compare_contracts if getattr(args, "env", None): # Environment overlays are a drift-mode concept (they shape the @@ -838,29 +845,25 @@ def _run_version_diff(args, logger: logging.Logger) -> int: "fluid.diff.format": getattr(args, "format", "text") or "text", }, ) as span: - report = compare_contracts(baseline, new) + try: + report = compare_contracts(baseline, new) + except ContractValidationError as exc: + raise _invalid_contracts_error( + exc, {"baseline": baseline_path, "new": new_path}, logger + ) from exc span.set_attribute("fluid.diff.breaking_count", len(report.breaking)) span.set_attribute("fluid.diff.non_breaking_count", len(report.non_breaking)) span.set_attribute("fluid.diff.info_count", len(report.info)) - # Render to stdout in the requested format. - fmt = getattr(args, "format", "text") or "text" - if fmt == "json": - # The JSON envelope is also written to --out (below) for CI - # artifact collection. Print to stdout here for piping. - import json as _json + # Render to stdout in the requested format, then always write the same + # envelope to --out so CI runners that don't parse stdout still get a + # machine-readable artifact. Both get the redacted report. + safe = _safe_report(report) + _print_version_diff(safe, getattr(args, "format", "text") or "text") - cprint(_json.dumps(report.to_dict(), indent=2)) - elif fmt == "markdown": - cprint(render_markdown(report)) - else: - cprint(render_text(report)) - - # Always write the structured envelope to --out so CI runners that don't - # parse stdout still get a machine-readable artifact. from ._common import write_json - write_json(args.out, report.to_dict()) + write_json(args.out, safe.to_dict()) info( logger, @@ -874,3 +877,112 @@ def _run_version_diff(args, logger: logging.Logger) -> int: if getattr(args, "fail_on_breaking", False) and report.has_breaking: return 1 return 0 + + +def _safe_report(report: ChangelogReport) -> ChangelogReport: + """``report`` with every value redacted and made JSON-safe, for stdout and ``--out``. + + A string goes through both redactors, as the CLI's error renderer does + (``cli/__init__.py``): ``redact_secret_text`` (JWTs, provider keys, + credentialed URLs, ``key=value``) and the console's pattern, which alone + catches ``api key: x``. A value JSON cannot hold, such as a date, bytes or + a set that YAML put in a column's quality rule, becomes text, so the + envelope always serialises: ``json.dump`` raised on it after the text + report had printed, and left ``--out`` cut short. + """ + import dataclasses + + from fluid_build.observability.secret_redactor import redact_value + + from ..api.changelog_types import ChangelogReport as _Report + + safe = _Report() + for change in (*report.breaking, *report.non_breaking, *report.info): + safe.add( + dataclasses.replace( + change, + **{ + field: _jsonable(redact_value(getattr(change, field))) + for field in ("path", "description", "before", "after") + }, + ) + ) + return safe + + +def _jsonable(value: Any) -> Any: + if isinstance(value, str): + return _redact_str(value) + if value is None or isinstance(value, (bool, int)): + return value + if isinstance(value, float): + return value if math.isfinite(value) else str(value) + if isinstance(value, Mapping): + # A key is contract text too: a free-form ``labels`` key can hold a credential. + return { + _jsonable(redact_secret_text(str(key))): _jsonable(item) for key, item in value.items() + } + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + if isinstance(value, (set, frozenset)): + return sorted((_jsonable(item) for item in value), key=repr) + return _redact_str(str(value)) + + +def _print_version_diff(report: ChangelogReport, fmt: str) -> None: + """Print a version-diff ``report`` (already made safe) to stdout as ``fmt``. + + JSON and Markdown are documents a program reads, so they are written + verbatim: a regex pass over the finished document cut through its + structure (a name ending in ``token=x`` lost the closing quote of its JSON + string, or the closing fence of its Markdown code span, which let the rest + of the line render as Markdown), and Rich would wrap a long line at the + terminal width and read ``[...]`` in a name as markup. Each value was + redacted before (:func:`_safe_report`). Text is printed literally, so a + name cannot style it, swap in an emoji or (``[/]``) crash it, and + soft-wrapped, as the drift report is, so each entry stays one greppable + line. + """ + from fluid_build.cli.console import cprint_literal + + from ..api.changelog import render_markdown, render_text + + if fmt == "json": + import json as _json + + sys.stdout.write(_json.dumps(report.to_dict(), indent=2) + "\n") + elif fmt == "markdown": + sys.stdout.write(render_markdown(report) + "\n") + else: + cprint_literal(render_text(report), soft_wrap=True, highlight=False) + + +def _invalid_contracts_error( + exc: ContractValidationError, paths: Dict[str, str], logger: logging.Logger +) -> CLIError: + """``diff_contract_invalid`` for the contracts ``compare_contracts`` refused. + + Each error is logged as ``fluid plan`` logs its schema errors, and the + message names every invalid file with the command that reports on it. + """ + for side, errors in exc.errors.items(): + for err in errors: + error(logger, "diff_contract_schema_error", side=side, path=paths[side], detail=err) + invalid = list(exc.errors) + named = "; ".join( + f"the {side} contract {paths[side]} has {len(exc.errors[side])} error(s)" + for side in invalid + ) + commands = " and ".join(f"``fluid validate {paths[side]}``" for side in invalid) + return CLIError( + 2, + "diff_contract_invalid", + { + "invalid": invalid, + "errors": {side: exc.errors[side] for side in invalid}, + "message": ( + f"``fluid diff --baseline`` compares valid contracts only: {named}. " + f"Run {commands} for the full report." + ), + }, + ) diff --git a/fluid_build/cli/discover/_jdbc_introspect.py b/fluid_build/cli/discover/_jdbc_introspect.py index 39d04be79..146739408 100644 --- a/fluid_build/cli/discover/_jdbc_introspect.py +++ b/fluid_build/cli/discover/_jdbc_introspect.py @@ -541,7 +541,7 @@ def _attach_string_sqlite(args: Dict[str, str], alias: str) -> str: return f"ATTACH {quote_ansi_string_literal(path)} AS {alias} (TYPE sqlite)" -_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*\Z") def _validate_alias(alias: str) -> str: diff --git a/fluid_build/cli/doctor.py b/fluid_build/cli/doctor.py index 3417119ba..75f883020 100644 --- a/fluid_build/cli/doctor.py +++ b/fluid_build/cli/doctor.py @@ -33,7 +33,7 @@ from types import SimpleNamespace from typing import TYPE_CHECKING, Dict, List, Optional, Tuple -from fluid_build.cli.console import cprint +from fluid_build.cli.console import cprint, cprint_literal from ._common import CLIError from ._logging import info @@ -173,6 +173,12 @@ "1800s default", "Per-`tofu` invocation wall-clock cap", ), + ( + "FLUID_TOFU_S3_LOCKFILE", + "on (OpenTofu 1.10+)", + "Set =0 to skip S3-native state locking when the deploying role cannot " + "write the `.tflock` object", + ), ( "FLUID_FEDERATION_TIMEOUT_SECONDS", "30s default", @@ -567,6 +573,7 @@ def run(args, logger: logging.Logger) -> int: feature_checks_ok=feature_checks_ok, copilot_readiness=copilot_readiness, ) + _print_extras_move_notice() if not extended_requested: return 0 if feature_checks_ok and not needs_action else 1 @@ -725,6 +732,19 @@ def _print_doctor_summary( cprint() +#: Shown once per ``fluid doctor`` run. litellm and mcp leave the base +#: install in 0.28.0 (``[standard]`` restores both); keep the text in step +#: with the release note for that change. +EXTRAS_MOVE_NOTICE = "AI/MCP move to extras in 0.28.0: pip install 'data-product-forge[standard]'" + + +def _print_extras_move_notice() -> None: + """Print the one-line AI/MCP extras notice (a single call per run).""" + # Literal: rich would read "[standard]" as a markup tag and drop it. + cprint_literal(EXTRAS_MOVE_NOTICE) + cprint() + + def _print_doctor_next_steps( *, feature_checks_ok: bool, copilot_readiness: LlmReadinessCheck ) -> None: @@ -1182,30 +1202,51 @@ def _check_fluid_features() -> Tuple[bool, List[Dict[str, any]]]: # check is non-critical (ok=True) so a missing install doesn't fail # the suite. Surfaces both install state and the env-var status so # users can confirm at a glance whether routing is live. - try: - import litellm # type: ignore[import-untyped] - - litellm_version = getattr(litellm, "__version__", "(unknown)") - details = f"litellm {litellm_version} · routing every provider" - checks.append( - { - "check": "LiteLLM backend", - "category": "ai", - "status": "✅ Installed", - "ok": True, - "details": details, - } - ) - except ImportError: - checks.append( - { - "check": "LiteLLM backend", - "category": "ai", - "status": "⚠️ Not installed (optional)", - "ok": True, - "details": "Install with: pip install 'data-product-forge[litellm]'", - } - ) + # Detection is metadata-only: ``import litellm`` costs ~2,300 modules and + # makes litellm fetch its model-cost map over the network, which a health + # check must never do. + from importlib import metadata as _metadata + + from fluid_build._optional import is_available + + # Same metadata-only probe for the other optional runtimes the install hints + # point at, so ``fluid doctor`` shows the extra to install. Never imported: + # ``mcp`` and ``openlineage`` are large trees and have no business in a health check. + for label, module, dist, extra, note in ( + ("LiteLLM backend", "litellm", "litellm", "litellm", "routing every provider"), + ("MCP SDK", "mcp", "mcp", "mcp", "fluid mcp serve / MCP delegates"), + ( + "OpenLineage client", + "openlineage.client", + "openlineage-python", + "lineage", + "lineage emission", + ), + ): + if is_available(module): + try: + version = _metadata.version(dist) + except _metadata.PackageNotFoundError: + version = "(unknown)" + checks.append( + { + "check": label, + "category": "ai", + "status": "✅ Installed", + "ok": True, + "details": f"{dist} {version} · {note}", + } + ) + else: + checks.append( + { + "check": label, + "category": "ai", + "status": "⚠️ Not installed (optional)", + "ok": True, + "details": f"Install with: pip install 'data-product-forge[{extra}]'", + } + ) # Third-party LLM provider plugins (entry-point group fluid_build.llm_providers). # Reads entry-point NAMES only (never loads/executes plugin code) so a health diff --git a/fluid_build/cli/export.py b/fluid_build/cli/export.py index f5adc3d63..fe7bae0b6 100644 --- a/fluid_build/cli/export.py +++ b/fluid_build/cli/export.py @@ -132,7 +132,9 @@ def run(args, logger: logging.Logger) -> int: provider_name = args.provider if not provider_name: # Auto-detect from contract - provider_name = contract.get("provider", "aws") + from fluid_build.providers import resolve_retired_platform + + provider_name = resolve_retired_platform(contract.get("provider", "aws")) if args.verbose: info(logger, f"Auto-detected provider: {provider_name}") diff --git a/fluid_build/cli/forge_copilot_agent.py b/fluid_build/cli/forge_copilot_agent.py index 2834d8169..b265670c6 100644 --- a/fluid_build/cli/forge_copilot_agent.py +++ b/fluid_build/cli/forge_copilot_agent.py @@ -466,7 +466,9 @@ def create_project( if self.console: self.console.print(f"[red]❌ {exc.message}[/red]") for suggestion in exc.suggestions: - self.console.print(f"[dim]• {suggestion}[/dim]") + from rich.markup import escape as _rich_escape + + self.console.print(f"[dim]• {_rich_escape(str(suggestion))}[/dim]") else: console_error(exc.message) for suggestion in exc.suggestions: diff --git a/fluid_build/cli/forge_copilot_default_summarizer.py b/fluid_build/cli/forge_copilot_default_summarizer.py index fd4a40409..8fa0686c3 100644 --- a/fluid_build/cli/forge_copilot_default_summarizer.py +++ b/fluid_build/cli/forge_copilot_default_summarizer.py @@ -128,7 +128,7 @@ def summarize(blob: str) -> str: # for every provider. Replaces ~30 lines of per-provider # request building, header formatting, and httpx posting # that the legacy path used to do. - import litellm # core dep + import litellm # optional ([litellm]); an ImportError is handled below from fluid_build.cli.forge_copilot_llm_litellm import ( _LITELLM_PREFIX_BY_PROVIDER, diff --git a/fluid_build/cli/forge_copilot_prompts.py b/fluid_build/cli/forge_copilot_prompts.py index cb5dade65..319c3acd2 100644 --- a/fluid_build/cli/forge_copilot_prompts.py +++ b/fluid_build/cli/forge_copilot_prompts.py @@ -253,7 +253,7 @@ def get_active_domain() -> Optional[str]: # leading dot/underscore. This is the first line of defence against # ``--prompt-profile ../../etc``; ``_resolve_profile_dir`` adds a resolved-path # containment check as the second. -_PROFILE_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_PROFILE_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*\Z") class PromptProfileError(ValueError): diff --git a/fluid_build/cli/forge_db_tools.py b/fluid_build/cli/forge_db_tools.py index f672f1f93..71b599fe0 100644 --- a/fluid_build/cli/forge_db_tools.py +++ b/fluid_build/cli/forge_db_tools.py @@ -341,13 +341,12 @@ def _fetch_sample_rows(arguments: Dict[str, Any], env: Mapping[str, str]) -> Dic ref = f'"{alias}"."{schema}"."{table}"' if schema else f'"{alias}"."{table}"' sql = f"SELECT * FROM {ref} LIMIT {applied_limit}" + from fluid_build._optional import MissingOptionalDependency, require + try: - import duckdb # noqa: F401 - the install hint below - except ImportError: - return { - "error": "DuckDbNotInstalled", - "message": "fetch_sample_rows requires duckdb (pip install 'data-product-forge[local]').", - } + require("duckdb", "local", "fetch_sample_rows") + except MissingOptionalDependency as exc: + return {"error": "DuckDbNotInstalled", "message": str(exc)} from fluid_build.providers._duckdb_sandbox import DuckDBAllowlist, secure_duckdb_connect con = None diff --git a/fluid_build/cli/forge_domain_enrichment.py b/fluid_build/cli/forge_domain_enrichment.py index ba581b4cc..04d9620e3 100644 --- a/fluid_build/cli/forge_domain_enrichment.py +++ b/fluid_build/cli/forge_domain_enrichment.py @@ -47,7 +47,7 @@ # override — defence in depth against a ``../``-style traversal reaching a # fragment block in an out-of-tree YAML. Matches the profile-name guard in # ``forge_copilot_prompts._PROFILE_NAME_RE``. -_SAFE_DOMAIN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_SAFE_DOMAIN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*\Z") @lru_cache(maxsize=1) diff --git a/fluid_build/cli/forge_modes.py b/fluid_build/cli/forge_modes.py index 9d9f32cc9..007e29793 100644 --- a/fluid_build/cli/forge_modes.py +++ b/fluid_build/cli/forge_modes.py @@ -1139,9 +1139,9 @@ def _render_ai_copilot_error(console: Any, exc: BaseException) -> None: copilot = copilot_class() is_non_interactive = bool(get_cli_arg_fn(args, "non_interactive", False)) - # Phase 0.2 — detect-first welcome scan. Runs in <50ms (parallel - # probes) and renders a populated panel so the user sees what we - # already know before any prompt. Skipped silently in + # Phase 0.2 — detect-first welcome scan. Parallel probes under a + # wall-clock cap (see ``_welcome_scan``) render a populated panel so + # the user sees what we already know before any prompt. Skipped silently in # non-interactive runs and for return users (forge_count >= 5). # Set FLUID_FORGE_NO_WELCOME=1 to suppress for tests / CI. if ( diff --git a/fluid_build/cli/forge_prompt_overlays.py b/fluid_build/cli/forge_prompt_overlays.py index 5842f2ac7..6accf0781 100644 --- a/fluid_build/cli/forge_prompt_overlays.py +++ b/fluid_build/cli/forge_prompt_overlays.py @@ -96,7 +96,7 @@ # leading dot/underscore. First line of defence against # ``--prompt-overlay ../../etc``; :func:`_resolve_overlay_path` adds a # resolved-realpath containment check as the second. -_OVERLAY_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_OVERLAY_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*\Z") _ALLOWED_MODES = ("replace", "append", "prepend") diff --git a/fluid_build/cli/forge_validation.py b/fluid_build/cli/forge_validation.py index c6cb37867..966ee0058 100644 --- a/fluid_build/cli/forge_validation.py +++ b/fluid_build/cli/forge_validation.py @@ -46,7 +46,7 @@ def validate_project_name(name: str) -> Tuple[bool, Optional[str]]: if len(name) > 63: return False, "Project name must be 63 characters or less" - if not re.match(r"^[a-z][a-z0-9-]*[a-z0-9]$", name): + if not re.match(r"^[a-z][a-z0-9-]*[a-z0-9]\Z", name): return ( False, "Project name must start with a letter, end with letter/number, and contain only lowercase letters, numbers, and hyphens", @@ -147,7 +147,7 @@ def validate_template_name(template: str) -> Tuple[bool, Optional[str]]: if len(template) < 2: return False, "Template name must be at least 2 characters" - if not re.match(r"^[a-z][a-z0-9-]*$", template): + if not re.match(r"^[a-z][a-z0-9-]*\Z", template): return ( False, "Template name must start with a letter and contain only lowercase letters, numbers, and hyphens", diff --git a/fluid_build/cli/generate_iac.py b/fluid_build/cli/generate_iac.py index f08cb4535..3224c5912 100644 --- a/fluid_build/cli/generate_iac.py +++ b/fluid_build/cli/generate_iac.py @@ -32,6 +32,7 @@ IAC_PLUGINS, assemble_tofu_document, get_iac_plugin, + known_iac_platforms, provider_config, provider_match, render_tofu_json, @@ -39,12 +40,13 @@ from fluid_build.iac.base import UnsupportedBindingError from fluid_build.iac.module import write_dependency_lock from fluid_build.iac.packaging import resolve_packaging +from fluid_build.iac.registry import IacProviderChoices, iac_plugin_unavailable_reason from fluid_build.iac.source import contract_source, source_contract_file from ._common import CLIError, load_contract_with_overlay, resolve_env_templates_in_contract from ._logging import info -_PROVIDER_CHOICES = ["auto", *sorted(IAC_PLUGINS)] +_PROVIDER_CHOICES = IacProviderChoices() #: The AWS account the native planner is given when ``fluid generate iac`` runs #: without ``AWS_ACCOUNT_ID`` (see :func:`native_actions`). Shaped to pass the @@ -61,7 +63,8 @@ def register_subcommand(subparsers: argparse._SubParsersAction): description=( "Compile a FLUID contract into an OpenTofu `.tf.json` module.\n\n" "The module is emitted for review — apply it with `tofu`.\n" - "Supported clouds: " + ", ".join(sorted(IAC_PLUGINS)) + "." + "Supported clouds: " + ", ".join(sorted(IAC_PLUGINS)) + ", plus any installed " + "IaC plugin (entry-point group fluid_build.iac_providers)." ), epilog="""Examples: fluid generate iac contract.fluid.yaml @@ -123,7 +126,7 @@ def run(args, logger: logging.Logger) -> int: packaging = resolve_packaging(contract) logger.debug("packaging resolved: legacy=%s pool=%s", packaging.is_legacy, packaging.pool) provider = _resolve_provider(contract, getattr(args, "provider", "auto")) - plugin = get_iac_plugin(provider) + plugin = _require_plugin(provider) with contract_source(source_contract_file(contract_path)): # A module written for review is applied with a plain `tofu apply`, which would # skip what the plugin binds outside OpenTofu (Unity Catalog masks and row filters). @@ -324,7 +327,7 @@ def _resolve_provider(contract, requested: str) -> str: return requested found = _detect_clouds(contract) - supported = "/".join(sorted(IAC_PLUGINS)) + supported = "/".join(known_iac_platforms()) if not found: raise CLIError( @@ -347,7 +350,7 @@ def _resolve_provider(contract, requested: str) -> str: ) cloud = found[0] - if cloud not in IAC_PLUGINS: + if get_iac_plugin(cloud) is None: # ``local``/DuckDB is a real, detected target but runs in-process — it # has no OpenTofu module to emit. Say so explicitly instead of the # misleading "no supported cloud". @@ -365,6 +368,19 @@ def _resolve_provider(contract, requested: str) -> str: return cloud +def _require_plugin(provider: str): + """The IaC plugin for ``provider``, or a ``CLIError`` saying why there is none.""" + plugin = get_iac_plugin(provider) + if plugin is not None: + return plugin + why = iac_plugin_unavailable_reason(provider) or "no IaC plugin serves it" + raise CLIError( + 1, + "generate_iac_failed", + {"error": f"no OpenTofu plugin for provider {provider!r}: {why}"}, + ) + + def native_actions(contract, logger: logging.Logger, *, offline: bool = False) -> list: """Best-effort native ``provider.plan()`` actions for the contract. @@ -410,12 +426,37 @@ def native_actions(contract, logger: logging.Logger, *, offline: bool = False) - # outside its declared jurisdiction logged the violation and then # emitted the module anyway, exit 0. Absence of a check and a failed # check are different things; only the first is best-effort. - if _is_sovereignty_refusal(exc): + # The same holds for a plan its own provider's validator rejected + # (``PlanValidationError``): a dotted contract id made the AWS planner + # refuse, and swallowing that left a module with no schedule, exit 0. + if _is_sovereignty_refusal(exc) or _is_plan_validation_refusal(exc): raise logger.debug("shadow: native planner unavailable: %s", exc) return [] +def _is_plan_validation_refusal(exc: BaseException) -> bool: + """True when ``exc`` (or anything it was raised *from*) is a plan-validation refusal. + + Matched on type through the ``__cause__``/``__context__`` chain, never on + message text, as :func:`_is_sovereignty_refusal` is. + """ + from fluid_build.providers._plan_validation import PlanValidationError + + return _in_cause_chain(exc, (PlanValidationError,)) + + +def _in_cause_chain(exc: BaseException, types: tuple) -> bool: + seen: set = set() + cur: BaseException | None = exc + while cur is not None and id(cur) not in seen: + if isinstance(cur, types): + return True + seen.add(id(cur)) + cur = cur.__cause__ or cur.__context__ + return False + + def _is_sovereignty_refusal(exc: BaseException) -> bool: """True when ``exc`` (or anything it was raised *from*) is a sovereignty veto. @@ -432,15 +473,7 @@ def _is_sovereignty_refusal(exc: BaseException) -> bool: from fluid_build._errors import ResidencyViolationError from fluid_build.cli._errors import SovereigntyViolationError - vetoes = (SovereigntyViolationError, ResidencyViolationError) - seen: set = set() - cur: BaseException | None = exc - while cur is not None and id(cur) not in seen: - if isinstance(cur, vetoes): - return True - seen.add(id(cur)) - cur = cur.__cause__ or cur.__context__ - return False + return _in_cause_chain(exc, (SovereigntyViolationError, ResidencyViolationError)) def _drop_referenced_container_actions(contract, actions: list, logger: logging.Logger) -> list: diff --git a/fluid_build/cli/hosted_mcp.py b/fluid_build/cli/hosted_mcp.py index f3344136d..0d602403e 100644 --- a/fluid_build/cli/hosted_mcp.py +++ b/fluid_build/cli/hosted_mcp.py @@ -76,6 +76,7 @@ from typing import Any, AsyncIterator, Dict, List, Mapping, Optional, Tuple from fluid_build._mcp_compat import attr as _mcp_attr +from fluid_build._optional import MissingOptionalDependency # Reuse dbt-mcp's async bridge + result coercion verbatim (borrow, no drift). from fluid_build.cli.dbt_mcp import _result_to_payload, _run_async @@ -330,12 +331,10 @@ def _open_session(self): @asynccontextmanager async def _open_real_session(self) -> AsyncIterator[Any]: - try: - from mcp import ClientSessionGroup, StdioServerParameters - except ImportError as exc: # pragma: no cover - mcp is a base dep - raise RuntimeError( - "Hosted-MCP delegation requires the 'mcp' SDK (pip install 'data-product-forge[mcp]')." - ) from exc + from fluid_build._optional import require + + require("mcp", "mcp", "Hosted-MCP delegation") + from mcp import ClientSessionGroup, StdioServerParameters params = StdioServerParameters( command=self.command, @@ -402,7 +401,10 @@ def hosted_mcp_tool_definitions( except Exception as exc: # noqa: BLE001 - never break tool listing # Type-only log (no message interpolation) — the server's stderr may # carry connection-string-shaped text. - LOG.warning("%s tool discovery unavailable: %s", spec.name, type(exc).__name__) + if isinstance(exc, MissingOptionalDependency): + LOG.warning("%s tool discovery unavailable: %s", spec.name, exc) + else: + LOG.warning("%s tool discovery unavailable: %s", spec.name, type(exc).__name__) continue for name, description, schema in tools: safe_desc = demote_markers(f"[{spec.label}] {description}".strip()) @@ -434,6 +436,9 @@ def dispatch_hosted_mcp_tool( try: payload = HostedMcpClient(spec, env=env).call_tool(bare, arguments or {}) except Exception as exc: # noqa: BLE001 + if isinstance(exc, MissingOptionalDependency): + LOG.warning("hosted MCP tool %s failed: %s", name, exc) + return {"error": type(exc).__name__, "message": str(exc)} LOG.warning("hosted MCP tool %s failed: %s", name, type(exc).__name__) return { "error": type(exc).__name__, diff --git a/fluid_build/cli/import_workflow/dbt.py b/fluid_build/cli/import_workflow/dbt.py index 34edaad3e..422d5f7a2 100644 --- a/fluid_build/cli/import_workflow/dbt.py +++ b/fluid_build/cli/import_workflow/dbt.py @@ -64,7 +64,7 @@ _REF_RE = re.compile(r"ref\(\s*(?:['\"][^'\"]+['\"]\s*,\s*)?['\"]([^'\"]+)['\"]\s*\)") # FLUID identifier pattern (schema `$defs.identifier`). -_IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]*[A-Za-z0-9_]$|^[A-Za-z0-9_]$") +_IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]*[A-Za-z0-9_]\Z|^[A-Za-z0-9_]\Z") # dbt folder layer → medallion layer (mirrors engines/dbt/models.py). _LAYER_TO_MEDALLION = {"staging": "Bronze", "intermediate": "Silver", "marts": "Gold"} diff --git a/fluid_build/cli/market_catalogs/command_center.py b/fluid_build/cli/market_catalogs/command_center.py index 9c8a44cd7..09ef66be9 100644 --- a/fluid_build/cli/market_catalogs/command_center.py +++ b/fluid_build/cli/market_catalogs/command_center.py @@ -68,15 +68,26 @@ async def _connect_impl(self) -> bool: self.base_url = cc.get_catalog_url() self.cc_client = cc - # Initialize aiohttp session. Every request on it follows a + # Initialize the HTTP session. Every request on it follows a # redirect only within the Command Center's origin: the client's # probes passed the host gate for that origin, not for wherever # the Command Center redirects (the reporter's redirect rule). - import aiohttp - - from fluid_build.observability.reporter import foreign_redirect_trace_config - - self.session = aiohttp.ClientSession(trace_configs=[foreign_redirect_trace_config()]) + import httpx + + from fluid_build.observability.reporter import refuse_foreign_redirect + from fluid_build.util.http_client import new_async_client + + # aiohttp's defaults, kept: 10 redirects, 300s total / 30s connect, + # and no proxy-environment lookup. + self.session = new_async_client( + timeout=httpx.Timeout(300.0, connect=30.0), + response_hooks=[refuse_foreign_redirect], + max_redirects=10, + trust_env=False, + # aiohttp's default context: the system CA store, so a corporate + # CA installed only there is still trusted. + system_trust=True, + ) self.logger.info(f"Connected to Command Center catalog: {self.base_url}") return True @@ -94,18 +105,18 @@ async def _search_data_products_impl(self, filters: SearchFilters) -> List[DataP # Build query parameters params = {} - if filters.query: - params["query"] = filters.query - if filters.domains: - params["domains"] = ",".join(filters.domains) - if filters.owners: - params["owners"] = ",".join(filters.owners) + if filters.text_query: + params["query"] = filters.text_query + if filters.domain: + params["domains"] = filters.domain + if filters.owner: + params["owners"] = filters.owner if filters.tags: params["tags"] = ",".join(filters.tags) - if filters.layers: - params["layers"] = ",".join([layer.value for layer in filters.layers]) - if filters.statuses: - params["statuses"] = ",".join([status.value for status in filters.statuses]) + if filters.layer: + params["layers"] = filters.layer.value + if filters.status: + params["statuses"] = filters.status.value if filters.min_quality_score: params["min_quality"] = filters.min_quality_score if filters.limit: @@ -114,69 +125,80 @@ async def _search_data_products_impl(self, filters: SearchFilters) -> List[DataP params["offset"] = filters.offset # Make request - async with self.session.get(self.base_url, params=params) as response: - if response.status != 200: - self.logger.warning(f"Command Center search failed: {response.status}") - return [] - - data = await response.json() - products = [] - - # Map Command Center schema to DataProductMetadata - for item in data.get("items", []): - try: - # Parse layer - layer = None - if "layer" in item: - try: - layer = DataProductLayer(item["layer"]) - except ValueError: - pass - - # Parse status - status = DataProductStatus.ACTIVE # Default - if "status" in item: - try: - status = DataProductStatus(item["status"]) - except ValueError: - pass - - # Create metadata - product = DataProductMetadata( - id=item["id"], - name=item["name"], - description=item.get("description", ""), - domain=item.get("domain", "unknown"), - owner=( - item.get("owner", {}).get("name", "unknown") - if isinstance(item.get("owner"), dict) - else item.get("owner", "unknown") - ), - layer=layer, - status=status, - tags=item.get("tags", []), - quality_score=item.get("quality_score", 0.0), - created_at=item.get("created_at"), - updated_at=item.get("updated_at"), - data_location=item.get("data_location", ""), - schema_definition=item.get("schema", {}), - sla=item.get("sla", {}), - documentation_url=item.get("documentation_url"), - metadata={ - "source": "command_center", - "cc_url": f"{self.base_url}/{item['id']}", - "version": item.get("version", "unknown"), - **item.get("metadata", {}), - }, - ) - - products.append(product) - - except Exception as e: - self.logger.warning(f"Failed to parse product {item.get('id')}: {e}") - continue - - return products + from fluid_build.util.http_client import merge_query + + response = await self.session.get(merge_query(self.base_url, params)) + if response.status_code != 200: + self.logger.warning(f"Command Center search failed: {response.status_code}") + return [] + + data = response.json() + products = [] + + # Map Command Center schema to DataProductMetadata + for item in data.get("items", []): + try: + # Parse layer + layer = None + if "layer" in item: + try: + layer = DataProductLayer(item["layer"]) + except ValueError: + self.logger.debug( + "Command Center product %r has an unknown layer %r; left unset", + item.get("id"), + item["layer"], + ) + + # Parse status + status = DataProductStatus.ACTIVE # Default + if "status" in item: + try: + status = DataProductStatus(item["status"]) + except ValueError: + self.logger.debug( + "Command Center product %r has an unknown status %r; kept %s", + item.get("id"), + item["status"], + status.value, + ) + + # Create metadata + product = DataProductMetadata( + id=item["id"], + name=item["name"], + description=item.get("description", ""), + domain=item.get("domain", "unknown"), + owner=( + item.get("owner", {}).get("name", "unknown") + if isinstance(item.get("owner"), dict) + else item.get("owner", "unknown") + ), + layer=layer, + status=status, + tags=item.get("tags", []), + quality_score=item.get("quality_score", 0.0), + created_at=item.get("created_at"), + updated_at=item.get("updated_at"), + data_location=item.get("data_location", ""), + schema_definition=item.get("schema", {}), + sla=item.get("sla", {}), + documentation_url=item.get("documentation_url"), + metadata={ + "source": "command_center", + "cc_url": f"{self.base_url}/{item['id']}", + "version": item.get("version", "unknown"), + **item.get("metadata", {}), + }, + ) + + products.append(product) + + except Exception as e: + self.logger.warning(f"Failed to parse product {item.get('id')}: {e}") + continue + + return products except Exception as e: self.logger.error(f"Command Center search failed: {e}") @@ -187,11 +209,11 @@ async def _get_catalog_stats_impl(self) -> Dict[str, Any]: try: stats_url = f"{self.base_url}/stats" - async with self.session.get(stats_url) as response: - if response.status == 200: - return await response.json() - else: - return {"total_products": 0, "total_domains": 0, "avg_quality_score": 0.0} + response = await self.session.get(stats_url) + if response.status_code == 200: + return response.json() + else: + return {"total_products": 0, "total_domains": 0, "avg_quality_score": 0.0} except Exception as e: self.logger.warning(f"Failed to get Command Center stats: {e}") diff --git a/fluid_build/cli/market_catalogs/mcp_catalog.py b/fluid_build/cli/market_catalogs/mcp_catalog.py index 2957775fa..444296812 100644 --- a/fluid_build/cli/market_catalogs/mcp_catalog.py +++ b/fluid_build/cli/market_catalogs/mcp_catalog.py @@ -501,12 +501,10 @@ async def _open_real_session(self) -> AsyncIterator[Any]: # streamable-HTTP), runs the ``initialize`` handshake, and tears the # connection down on context exit. We don't re-implement any of that — # we only translate config into the right params object below. - try: - from mcp import ClientSessionGroup - except ImportError as exc: # pragma: no cover - mcp is a core dep - raise RuntimeError( - "MCP discovery requires the 'mcp' SDK (pip install 'data-product-forge[mcp]')." - ) from exc + from fluid_build._optional import require + + require("mcp", "mcp", "MCP discovery") + from mcp import ClientSessionGroup params = self._build_server_params() async with ClientSessionGroup() as group: diff --git a/fluid_build/cli/marketplace.py b/fluid_build/cli/marketplace.py index 67ed28e3a..fbde4f1ce 100644 --- a/fluid_build/cli/marketplace.py +++ b/fluid_build/cli/marketplace.py @@ -33,36 +33,12 @@ from pathlib import Path from typing import Any, Callable, Dict, List, Optional -# NOTE: ``requests`` (~107 modules incl urllib3/certifi/charset_normalizer) is -# resolved lazily — at import time it would land on the cold ``fluid --help`` -# path (this module is imported at registration). ``_resolve_requests`` performs -# the import on first use and caches it into the module global, and the PEP 562 -# ``__getattr__`` exposes ``requests`` as a module attribute so the -# ``patch("…cli.marketplace.requests")`` test seam keeps working. See the A++ -# Light CLI startup card. -_REQUESTS_UNRESOLVED = object() # sentinel: import not yet attempted - - -def _resolve_requests(): - """Import ``requests`` lazily, caching the module (or ``None``) in globals. - - Honors a test ``patch("…cli.marketplace.requests")``: once the module - attribute is set, that value is returned unchanged. - """ - current = globals().get("requests", _REQUESTS_UNRESOLVED) - if current is not _REQUESTS_UNRESOLVED: - return current - try: - import requests as _requests - except ImportError: - _requests = None # type: ignore[assignment] - globals()["requests"] = _requests - return _requests +# NOTE: ``httpx`` is imported lazily, inside the functions that make a request: +# this module is imported at registration, and ``httpx`` must stay off the cold +# ``fluid --help`` path. The HTTP calls go through ``fluid_build.util.http_client``. def __getattr__(name: str): - if name == "requests": - return _resolve_requests() if name == "get_command_center_client": from ._command_center import get_command_center_client @@ -85,10 +61,10 @@ def __getattr__(name: str): from ._common import CLIError from .console import cprint -# NOTE: ``get_command_center_client`` lives in ``cli._command_center``, which -# imports ``requests`` (~107 modules) at module scope. Importing it here eagerly -# would pull ``requests`` onto the cold ``fluid --help`` path (this module is -# imported at registration). It's resolved lazily via ``__getattr__`` below — and +# NOTE: ``get_command_center_client`` lives in ``cli._command_center``. +# Importing it here eagerly would pull the Command Center client onto the cold +# ``fluid --help`` path (this module is imported at registration). It's resolved +# lazily via ``__getattr__`` above — and # exposing it as a module attribute keeps the # ``patch("…cli.marketplace.get_command_center_client")`` test seam working. See # the A++ Light CLI startup card. @@ -110,19 +86,32 @@ def __getattr__(name: str): console = Console() if RICH_AVAILABLE else None -def _requests_exception_type(name: str) -> type[BaseException] | None: - """Return a requests exception class, tolerating patched test doubles.""" - requests = _resolve_requests() - exceptions = getattr(requests, "exceptions", None) - exc_type = getattr(exceptions, name, None) - if isinstance(exc_type, type) and issubclass(exc_type, BaseException): - return exc_type - return None +def _http_errors() -> tuple: + """What a failed marketplace request raises (``requests.RequestException``'s role).""" + import httpx + + return (httpx.HTTPError, httpx.InvalidURL) + + +def _response_errors() -> tuple: + """:func:`_http_errors` plus a body that is not JSON. + + ``requests`` wrapped a ``.json()`` decode failure in a ``RequestException``; + ``httpx`` lets the ``ValueError`` (``json.JSONDecodeError``) through. + """ + return _http_errors() + (ValueError,) + + +def _is_timeout(exc: BaseException) -> bool: + import httpx + + return isinstance(exc, httpx.TimeoutException) + +def _is_connection_error(exc: BaseException) -> bool: + import httpx -def _is_requests_exception(exc: BaseException, name: str) -> bool: - exc_type = _requests_exception_type(name) - return bool(exc_type and isinstance(exc, exc_type)) + return isinstance(exc, (httpx.NetworkError, httpx.RemoteProtocolError, httpx.ProxyError)) class NamedRegistryURL(str): @@ -139,19 +128,19 @@ class NamedRegistryURL(str): __slots__ = () -# The headers ``requests`` sends on every request of its own accord -# (``requests.utils.default_headers``, -# https://github.com/psf/requests/blob/main/src/requests/utils.py). Any other -# header may carry a credential, and on a hop to another host ``requests`` -# re-derives only ``Authorization`` (``Session.rebuild_auth``) and ``Cookie`` -# (``Session.resolve_redirects``): an ``X-API-Key`` would go on with it. So a -# request with any other header counts as carrying one. -_DEFAULT_REQUEST_HEADERS = frozenset({"user-agent", "accept-encoding", "accept", "connection"}) +# The headers ``httpx`` sends on every request of its own accord (``Host``, +# plus the client defaults in ``httpx._client``). Any other header may carry a +# credential, and on a hop to another origin ``httpx`` strips only +# ``Authorization``: an ``X-API-Key`` would go on with it. So a request with +# any other header counts as carrying one. +_DEFAULT_REQUEST_HEADERS = frozenset( + {"host", "user-agent", "accept-encoding", "accept", "connection"} +) def _carries_anything(request: Any) -> bool: - """Whether a prepared request has a body or a header beyond requests' defaults.""" - return request.body is not None or any( + """Whether a request has a body or a header beyond the client's defaults.""" + return bool(request.content) or any( name.lower() not in _DEFAULT_REQUEST_HEADERS for name in request.headers ) @@ -160,28 +149,20 @@ def _named_registry_redirect_hook() -> Callable[..., Any]: """A response hook for one call to a named registry: ``refuse_foreign_redirect`` if the call's first request carries a body or a header of its own. - A call whose first request has neither (only ``requests``' default + A call whose first request has neither (only the client's default headers) is followed wherever it goes, as main did: nothing in it is the operator's. One with a body (the instantiate POST, the operator's - parameters) or any other header (Authorization from ``auth`` or - ``~/.netrc``, X-API-Key, a cookie the caller passes) still stops at the - origin. - - Judged once, on the first request: ``requests`` runs a call's response - hooks on every hop, the first response's first. A later hop's prepared - request may carry headers ``requests`` adds of its own accord, none of - them the operator's for the next origin, and each recomputed for each - next URL: the ``Cookie`` it builds from the call's own jar, which holds - the cookies this redirect chain set (``Session.resolve_redirects``; - ``requests.get`` uses a fresh session); for an http:// URL, the - ``Proxy-Authorization`` of the operator's proxy - (``Session.rebuild_proxies``), which goes to the proxy, not the origin; - and an ``Authorization`` that ``~/.netrc`` binds to the hop's own host - (``Session.rebuild_auth``, which drops it when the host changes). - Judging every hop afresh read the first two as the operator's and - stopped a registry that set a cookie, or was reached through an - authenticated proxy, before it moved host. One hook per call: the - verdict belongs to that call's redirect chain. + parameters) or any other header (Authorization from ``~/.netrc``, + X-API-Key, a cookie the caller passes) still stops at the origin. + + Judged once, on the first request: ``httpx`` runs a call's response hooks + on every hop, the first response's first. A later hop's request may carry a + ``Cookie`` the client builds of its own accord from the cookies this + redirect chain set, none of them the operator's for the next origin; + judging every hop afresh read that as the operator's and stopped a registry + that set a cookie before it moved host. One hook per call (and one client + per call, as ``requests.get`` used a fresh session): the verdict belongs to + that call's redirect chain. """ from fluid_build.observability.reporter import refuse_foreign_redirect @@ -198,46 +179,45 @@ def hook(response: Any, *args: Any, **kwargs: Any) -> Any: def _marketplace_request( - request_func, url: str, logger: logging.Logger, *, named_registry: bool = False, **kwargs + method: str, url: str, logger: logging.Logger, *, named_registry: bool = False, **kwargs ): """Run a marketplace HTTP request with bounded timeout and one transient retry. - A redirect is followed only within the marketplace's origin (the - reporter's redirect rule, ``refuse_foreign_redirect``, as for the Command - Center client's probes): neither a search nor the instantiate POST, which - carries the operator's parameters, goes on to another origin. The refusal - is raised as a ``RequestException``, which every caller reports. + Redirects are followed (as ``requests`` did) but only within the + marketplace's origin (the reporter's redirect rule, + ``refuse_foreign_redirect``, as for the Command Center client's probes): + neither a search nor the instantiate POST, which carries the operator's + parameters, goes on to another origin. The refusal is raised as an + ``httpx.HTTPError``, which every caller reports. ``named_registry`` says ``url`` is in a registry the operator named (:class:`NamedRegistryURL`), not the Command Center: there a request with no body and no header of its own follows a redirect to another origin, as on main (:func:`_named_registry_redirect_hook`). """ + import httpx + from fluid_build.observability.reporter import ( CredentialRedirectRefused, refuse_foreign_redirect, ) + from fluid_build.util.http_client import merge_query, new_client timeout = kwargs.pop("timeout", MARKETPLACE_HTTP_TIMEOUT_SECONDS) - request_error = _requests_exception_type("RequestException") or Exception - transient_errors = tuple( - exc_type - for exc_type in ( - _requests_exception_type("Timeout"), - _requests_exception_type("ConnectionError"), - ) - if exc_type is not None - ) + # ``requests`` merged ``params`` into the URL's own query (``?tenant=acme``); + # ``httpx`` would replace it. + url = merge_query(url, kwargs.pop("params", None)) for attempt in range(MARKETPLACE_HTTP_RETRIES + 1): # A retry is a new redirect chain, judged afresh. hook = _named_registry_redirect_hook() if named_registry else refuse_foreign_redirect try: - return request_func(url, timeout=timeout, hooks={"response": hook}, **kwargs) + with new_client(timeout=timeout, response_hooks=[hook]) as client: + return client.request(method, url, **kwargs) except CredentialRedirectRefused as exc: - raise request_error(f"not following the redirect: {exc}") from exc - except request_error as exc: - is_transient = bool(transient_errors and isinstance(exc, transient_errors)) + raise httpx.HTTPError(f"not following the redirect: {exc}") from exc + except _http_errors() as exc: + is_transient = _is_timeout(exc) or _is_connection_error(exc) if not is_transient or attempt >= MARKETPLACE_HTTP_RETRIES: raise logger.warning("Marketplace request failed transiently; retrying once: %s", exc) @@ -246,24 +226,25 @@ def _marketplace_request( def _marketplace_get(url: str, logger: logging.Logger, **kwargs): - requests = _resolve_requests() - return _marketplace_request(requests.get, url, logger, **kwargs) + return _marketplace_request("GET", url, logger, **kwargs) def _marketplace_post(url: str, logger: logging.Logger, **kwargs): - requests = _resolve_requests() - return _marketplace_request(requests.post, url, logger, **kwargs) + return _marketplace_request("POST", url, logger, **kwargs) def _format_marketplace_error(exc: BaseException) -> str: - if _is_requests_exception(exc, "Timeout"): + if _is_timeout(exc): return ( "Marketplace API request timed out after " f"{MARKETPLACE_HTTP_TIMEOUT_SECONDS}s. Check the API URL or try again." ) - if _is_requests_exception(exc, "ConnectionError"): + if _is_connection_error(exc): return "Could not connect to the marketplace API. Check Command Center or FLUID_API_URL." + if isinstance(exc, ValueError): + return "Marketplace API returned a response that is not valid JSON." + response = getattr(exc, "response", None) status_code = getattr(response, "status_code", None) if status_code: @@ -538,7 +519,6 @@ def _bundled_blueprints_matching(args) -> list: def search_blueprints(args, logger: logging.Logger, api_url: str) -> int: """Search marketplace blueprints (bundled + registry).""" - requests = _resolve_requests() # ``--format json`` (mirrors the catalog discovery path in market.py): emit a # clean machine-parseable array on stdout and suppress every decorative print # so a script piping the output gets pure JSON, no rich-table contamination. @@ -582,7 +562,7 @@ def search_blueprints(args, logger: logging.Logger, api_url: str) -> int: data = response.json() registry = data.get("items", []) registry_total = data.get("total", len(registry)) - except requests.exceptions.RequestException as e: + except _response_errors() as e: message = _format_marketplace_error(e) # A bundled-only result is still useful; only hard-fail when there # is nothing at all to show. @@ -669,7 +649,6 @@ def search_blueprints(args, logger: logging.Logger, api_url: str) -> int: def show_blueprint_info(args, logger: logging.Logger, api_url: str) -> int: """Show detailed blueprint information.""" - requests = _resolve_requests() console.print(f"[cyan]ℹ️ Fetching blueprint: {args.blueprint_id}...[/cyan]\n") from fluid_build.cli._market_bundled_blueprints import get_bundled_blueprint, is_bundled @@ -690,7 +669,7 @@ def show_blueprint_info(args, logger: logging.Logger, api_url: str) -> int: ) response.raise_for_status() bp = response.json() - except requests.exceptions.RequestException as e: + except _response_errors() as e: message = _format_marketplace_error(e) console.print(f"[red]❌ {message}[/red]") logger.error("Failed to get blueprint info: %s", message, exc_info=True) @@ -787,7 +766,6 @@ def instantiate_blueprint(args, logger: logging.Logger, api_url: str) -> int: render_bundled_contract, ) - requests = _resolve_requests() console.print(f"[cyan]⚙️ Instantiating blueprint: {args.blueprint_id}...[/cyan]\n") bundled_bp = get_bundled_blueprint(args.blueprint_id) if is_bundled(args.blueprint_id) else None @@ -805,7 +783,7 @@ def instantiate_blueprint(args, logger: logging.Logger, api_url: str) -> int: ) response.raise_for_status() bp = response.json() - except requests.exceptions.RequestException as e: + except _response_errors() as e: message = _format_marketplace_error(e) console.print(f"[red]❌ Failed to fetch blueprint: {message}[/red]") logger.error("Failed to fetch blueprint: %s", message, exc_info=True) @@ -859,7 +837,7 @@ def instantiate_blueprint(args, logger: logging.Logger, api_url: str) -> int: result = response.json() contract = result["contract"] cost_estimate = result.get("cost_estimate", "N/A") - except requests.exceptions.RequestException as e: + except _response_errors() as e: message = _format_marketplace_error(e) console.print(f"[red]❌ Failed to instantiate blueprint: {message}[/red]") logger.error("Blueprint instantiation failed: %s", message, exc_info=True) diff --git a/fluid_build/cli/mcp/cli.py b/fluid_build/cli/mcp/cli.py index ff3f17e95..3563e281b 100644 --- a/fluid_build/cli/mcp/cli.py +++ b/fluid_build/cli/mcp/cli.py @@ -143,8 +143,10 @@ def run(args, logger: logging.Logger) -> int: return _render_mcp_guide() if action == "serve": policy = _build_policy_from_args(args) - _set_policy(policy) + # Build first: when the SDK is missing this raises, and the process must + # not be left with a policy installed for a server that never started. app = _build_fastmcp_app(policy) + _set_policy(policy) try: asyncio.run(app.run_stdio_async()) except KeyboardInterrupt: diff --git a/fluid_build/cli/plan.py b/fluid_build/cli/plan.py index dadfdd0db..7e84c35ac 100644 --- a/fluid_build/cli/plan.py +++ b/fluid_build/cli/plan.py @@ -701,6 +701,9 @@ def _gate_provider_flag(requested: Optional[str], logger: logging.Logger) -> Non registry.discover_providers(logger) name = requested.strip().lower().replace("-", "_") if name not in registry.PROVIDERS: + hint = registry.retired_provider_hint(name) + if hint: + raise CLIError(2, "provider_retired", {"error": hint}) raise CLIError( 2, "provider_unknown", {"requested": name, "available": sorted(registry.PROVIDERS)} ) diff --git a/fluid_build/cli/plugins_cmd.py b/fluid_build/cli/plugins_cmd.py index 243930800..712ab7e37 100644 --- a/fluid_build/cli/plugins_cmd.py +++ b/fluid_build/cli/plugins_cmd.py @@ -23,12 +23,13 @@ ALLOWED plugins to read their declared metadata. **This command does not give you a plugin-free process.** CLI bootstrap builds -the whole argparse tree before dispatch, and two roles feed argparse -``choices=`` from their registries — ``iac_provider`` (via -``cli/generate_iac.py``) and ``modeling_technique`` (via -``cli/forge_data_model.py``). Their entry points are therefore ``ep.load()``-ed -on **every** invocation, ``fluid --version`` and ``fluid plugins`` included. -The ``command`` role is genuinely lazy and is not imported until dispatched. +the whole argparse tree before dispatch, and the ``modeling_technique`` role +feeds argparse ``choices=`` from its registry (via ``cli/forge_data_model.py``), +so its entry points are ``ep.load()``-ed on **every** invocation, ``fluid +--version`` and ``fluid plugins`` included. The ``iac_provider`` role is lazy: +only its entry-point *names* are read, and a plugin is imported the first time a +contract's platform needs it (or before a destructive apply attributes a state). +The ``command`` role is also not imported until dispatched. To inspect an untrusted plugin set without executing it, block it first:: diff --git a/fluid_build/cli/policy_apply.py b/fluid_build/cli/policy_apply.py index 97d21174a..f54b91eeb 100644 --- a/fluid_build/cli/policy_apply.py +++ b/fluid_build/cli/policy_apply.py @@ -70,7 +70,10 @@ def _resolve_from_bindings(data: dict) -> tuple[str, str]: project = b.get("project", "") if provider and project: break - return provider, project + from fluid_build.providers import resolve_retired_platform + + # A platform the contract named, so a retired one resolves to its successor. + return resolve_retired_platform(provider) if provider else provider, project @_traced_stage("policy_apply") diff --git a/fluid_build/cli/security.py b/fluid_build/cli/security.py index 3e2f82f9d..d2add1e77 100644 --- a/fluid_build/cli/security.py +++ b/fluid_build/cli/security.py @@ -662,7 +662,7 @@ def validate_project_name(name: str) -> bool: return False # Check format: alphanumeric, hyphens, underscores only - if not re.match(r"^[a-zA-Z0-9_-]+$", name): + if not re.match(r"^[a-zA-Z0-9_-]+\Z", name): return False # Check length diff --git a/fluid_build/cli/test.py b/fluid_build/cli/test.py index 204690fda..3da19ba4d 100644 --- a/fluid_build/cli/test.py +++ b/fluid_build/cli/test.py @@ -103,7 +103,7 @@ def register(subparsers: argparse._SubParsersAction) -> None: - Metadata and governance completeness - Optional drift detection against historical results - Supports all FLUID providers: gcp, snowflake, aws, local. + Supports all FLUID providers: aws, databricks, gcp, local, snowflake. """, epilog=""" Examples: @@ -139,7 +139,9 @@ def register(subparsers: argparse._SubParsersAction) -> None: # --- optional --- p.add_argument("--env", help="Environment overlay (dev/test/prod)") - p.add_argument("--provider", help="Override provider platform (gcp, snowflake, aws, local)") + p.add_argument( + "--provider", help="Override provider platform (aws, databricks, gcp, local, snowflake)" + ) p.add_argument("--project", help="Override project/account ID") p.add_argument("--region", help="Override region/location") diff --git a/fluid_build/cli/verify.py b/fluid_build/cli/verify.py index 31c04a1a3..e0ce92ebf 100644 --- a/fluid_build/cli/verify.py +++ b/fluid_build/cli/verify.py @@ -1200,6 +1200,7 @@ def _render_masking_dimension(masking: Dict[str, Any], ok: str, heading: str) -> ("residency", "\n 🔍 Residency (workspace region)"), ("governance_tags", "\n 🔍 Governance tags (Unity Catalog)"), ("row_filter_masks", "\n 🔍 Row filter and column masks (Unity Catalog)"), + ("retention", "\n 🔍 Retention (Delta history)"), ("jobs", "\n 🔍 Jobs"), ) @@ -1217,6 +1218,11 @@ def _render_findings_dimensions(dimensions: Dict[str, Any]) -> None: if dimension.get("status") == "skip": cprint(f" ⚪ SKIPPED - {dimension.get('message', '')}", markup=False) continue + if dimension.get("status") == "warn": + cprint(f" ⚠️ WARN - {dimension.get('message', '')}", markup=False) + for warning in dimension.get("warnings") or []: + cprint(f" Fix: {warning}", markup=False) + continue cprint(f" ❌ FAIL - {dimension.get('message', '')}", markup=False) if dimension.get("fix"): cprint(f" Fix: {dimension['fix']}", markup=False) @@ -1227,6 +1233,8 @@ def _render_findings_dimensions(dimensions: Dict[str, Any]) -> None: f" Fix (run by an admin of the catalog): {finding['fix']}", markup=False, ) + for warning in dimension.get("warnings") or []: + cprint(f" ⚠️ {warning}", markup=False) #: What ``fluid apply`` provisions for a GCP-bound expose, as far as stage 9 diff --git a/fluid_build/copilot/agents/token_budget.py b/fluid_build/copilot/agents/token_budget.py index f66b6ac02..af880cc2d 100644 --- a/fluid_build/copilot/agents/token_budget.py +++ b/fluid_build/copilot/agents/token_budget.py @@ -288,7 +288,7 @@ def count_tokens(text: str, *, provider: str = "", model: str = "") -> int: return estimate_tokens(text) if provider and model: try: - import litellm # core dep + import litellm # optional ([litellm]); the except below falls back to the heuristic from fluid_build.llm.litellm_backend import ( _LITELLM_PREFIX_BY_PROVIDER, diff --git a/fluid_build/copilot/catalog/_patterns.py b/fluid_build/copilot/catalog/_patterns.py index 19edce90e..7c443ce30 100644 --- a/fluid_build/copilot/catalog/_patterns.py +++ b/fluid_build/copilot/catalog/_patterns.py @@ -99,7 +99,8 @@ # Conservative default identifier shape — covers Snowflake / BigQuery / # Unity / Glue without quoting weirdness. Catalogs with looser rules -# (e.g., DataHub URN strings) override the regex per-adapter. +# (e.g., DataHub URN strings) override the regex per-adapter. Matched +# with ``fullmatch``: ``$`` also matches before a trailing newline. _DEFAULT_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") @@ -143,7 +144,7 @@ def validate_and_quote_identifier( non-SQL-shaped identifiers. """ rule = pattern or _DEFAULT_IDENTIFIER - if not rule.match(value or ""): + if not rule.fullmatch(value or ""): raise CatalogConfigError( message=f"Invalid {kind} identifier: {value!r}", suggestions=[ diff --git a/fluid_build/copilot/catalog/bigquery.py b/fluid_build/copilot/catalog/bigquery.py index bfaba7e61..aa9de5af3 100644 --- a/fluid_build/copilot/catalog/bigquery.py +++ b/fluid_build/copilot/catalog/bigquery.py @@ -86,6 +86,8 @@ # # The pattern table here keeps both rules visible — adding a new # identifier kind (e.g., a connection name) just adds one entry. +# Match these with ``fullmatch``: their ``$`` also matches before a +# trailing newline, so ``.match`` lets ``"orders\n"`` through. _BQ_IDENT_PATTERNS = { # Allow hyphens for project IDs. "project": re.compile(r"^[A-Za-z][A-Za-z0-9-]*$"), @@ -113,7 +115,7 @@ def _quote_ident(value: str, *, kind: str) -> str: def _validate_ident(value: str, *, kind: str) -> str: """Return a BigQuery identifier after strict kind-specific validation.""" pattern = _BQ_IDENT_PATTERNS.get(kind) - if pattern is None or not pattern.match(value or ""): + if pattern is None or not pattern.fullmatch(value or ""): raise CatalogConfigError( message=f"Invalid BigQuery {kind} identifier: {value!r}", suggestions=[ diff --git a/fluid_build/copilot/missions/spec.py b/fluid_build/copilot/missions/spec.py index 879c019fc..64c435977 100644 --- a/fluid_build/copilot/missions/spec.py +++ b/fluid_build/copilot/missions/spec.py @@ -70,7 +70,7 @@ _BUDGET_KEYS = frozenset({"max_usd", "max_iterations", "max_wall_seconds"}) #: One dotted-path segment: a key, optionally fanned out over an array. -_SEGMENT_RE = re.compile(r"^([A-Za-z_][A-Za-z0-9_-]*)(\[\*\])?$") +_SEGMENT_RE = re.compile(r"^([A-Za-z_][A-Za-z0-9_-]*)(\[\*\])?\Z") _MISSION_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$") @@ -172,7 +172,7 @@ def validate_predicate_path(path: str, *, label: str) -> None: for segment in path.split("."): if not _SEGMENT_RE.match(segment): raise MissionSpecError( - f"{label}: invalid path segment '{segment}' in '{path}'. The predicate " + f"{label}: invalid path segment {segment!r} in {path!r}. The predicate " "grammar is frozen: dotted keys with optional [*] array fan-out " "(e.g. exposes[*].contract.dq.rules)." ) diff --git a/fluid_build/credentials/protected_env.py b/fluid_build/credentials/protected_env.py index 15af94a06..9c425457b 100644 --- a/fluid_build/credentials/protected_env.py +++ b/fluid_build/credentials/protected_env.py @@ -197,6 +197,8 @@ "FLUID_DIAG_OUT_DIR", "FLUID_PRICES_JSON", "FLUID_STATE_BACKEND", + # =0 turns S3 state locking off: the operator's call, never a checkout's + "FLUID_TOFU_S3_LOCKFILE", "FLUID_STORE_BACKEND", "FLUID_STORE_DSN", "FLUID_STORE_PATH", diff --git a/fluid_build/engines/dbt/_test_mapping.py b/fluid_build/engines/dbt/_test_mapping.py index ade15fb1d..434657f48 100644 --- a/fluid_build/engines/dbt/_test_mapping.py +++ b/fluid_build/engines/dbt/_test_mapping.py @@ -76,6 +76,10 @@ import re from typing import Any, Mapping, Sequence +# Key recognition lives in ``util.contract`` so ``fluid diff`` judges a key the way this +# engine builds one without importing every engine; re-exported for ``_tm.*`` callers. +from ...util.contract import column_is_key, is_truthy # noqa: F401 + # ── Package prefixes / canonical test names ─────────────────────────────── # The single numeric range/between dialect (see module docstring). RANGE_TEST_NAME = "dbt_expectations.expect_column_values_to_be_between" @@ -275,7 +279,6 @@ def sentinel_test(kind: str, column: str | None = None) -> str: # ``maximum`` / FK spellings below are accepted best-effort because contracts # express quality intent inline in practice. -_PK_KEYS = ("primary", "primaryKey", "primary_key", "pk", "isPrimary") _ENUM_KEYS = ("enum", "acceptedValues", "accepted_values") _FK_KEYS = ("foreign_key", "foreignKey", "references", "relationship_to") @@ -298,32 +301,6 @@ def sentinel_test(kind: str, column: str | None = None) -> str: _REFERENCES_RE = re.compile(r"^\s*references\s+(\S+?)\.(\w+)\s*$", re.IGNORECASE) -def is_truthy(value: Any) -> bool: - """Loose truthiness for YAML/JSON booleans-as-strings (``"true"``/``True``).""" - if isinstance(value, bool): - return value - if isinstance(value, str): - return value.strip().lower() in ("true", "1", "yes") - return bool(value) - - -def column_is_key(col: Mapping[str, Any]) -> bool: - """True when the column is declared a uniqueness key / primary key.""" - if is_truthy(col.get("unique")): - return True - if any(is_truthy(col.get(k)) for k in _PK_KEYS): - return True - if str(col.get("semanticType") or "").strip().lower() in ("identifier", "primary_key"): - return True - labels = col.get("labels") - if isinstance(labels, Mapping): - if is_truthy(labels.get("unique")): - return True - if str(labels.get("constraint") or "").strip().lower() in ("primary_key", "unique"): - return True - return False - - def validation_rules(col: Mapping[str, Any]) -> list[Mapping[str, Any]]: """The column's ``validationRules[]`` entries (schema-canonical).""" raw = col.get("validationRules") diff --git a/fluid_build/forge/core/bundle.py b/fluid_build/forge/core/bundle.py index 3758ed067..ca4e757af 100644 --- a/fluid_build/forge/core/bundle.py +++ b/fluid_build/forge/core/bundle.py @@ -492,7 +492,7 @@ def bundle_overlay_warnings(source: Optional[Mapping[str, Any]]) -> List[str]: #: path. It selects the overlay file whose ``id`` the recorded source #: contract is checked with, so a value like ``../../x`` must not be able to #: pick an arbitrary file for that check. -_ENV_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$") +_ENV_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*\Z") def bundle_source_contract( diff --git a/fluid_build/forge/core/validators.py b/fluid_build/forge/core/validators.py index 83472ccf4..2bab29bca 100644 --- a/fluid_build/forge/core/validators.py +++ b/fluid_build/forge/core/validators.py @@ -390,9 +390,14 @@ def validate_json_schema( ) ] schema = schema_manager.get_schema(version) - try: - import jsonschema # type: ignore - except ImportError: # pragma: no cover — jsonschema is a hard dep upstream + from fluid_build.schema_manager import ( + JSONSCHEMA_AVAILABLE, + format_schema_error, + schema_validator, + ) + + # The flag schema_validator needs: a package that is found can still fail to import. + if not JSONSCHEMA_AVAILABLE: return [ ValidationIssue( file=path, @@ -405,9 +410,7 @@ def validate_json_schema( # Honour the schema's declared $schema; Draft 7 silently ignores # newer keywords, which drops the constraints they express. - from fluid_build.schema_manager import format_schema_error - - validator = jsonschema.validators.validator_for(schema)(schema) + validator = schema_validator(schema) errors = sorted(validator.iter_errors(doc), key=lambda e: list(e.absolute_path)) for err in errors: loc = "/".join(str(p) for p in err.absolute_path) or "" diff --git a/fluid_build/forge/federation.py b/fluid_build/forge/federation.py index 76df5e109..4a1c7086f 100644 --- a/fluid_build/forge/federation.py +++ b/fluid_build/forge/federation.py @@ -236,6 +236,7 @@ def _federation_git_timeout() -> float: # (``~/.cache/fluid/federation-git/`` and the digest-cache file). # Constrain it to a conservative slug so a tampered manifest row can't # smuggle ``../`` traversal or absolute-path segments into those sinks. +# Matched with ``fullmatch``: ``$`` also matches before a trailing newline. _WORKSPACE_ID_RE = re.compile(r"^[A-Za-z0-9_.-]{1,64}$") @@ -589,7 +590,7 @@ def from_dict(cls, d: Mapping[str, Any]) -> "FederatedWorkspace": # anything outside a conservative slug so a tampered manifest # row cannot smuggle ``../`` traversal or absolute segments into # those path sinks. - if not _WORKSPACE_ID_RE.match(ws_id): + if not _WORKSPACE_ID_RE.fullmatch(ws_id): raise ValueError( f"federation workspace id {ws_id!r} is invalid: must match " f"{_WORKSPACE_ID_RE.pattern} (alphanumerics, dot, dash, " diff --git a/fluid_build/iac/__init__.py b/fluid_build/iac/__init__.py index 8dc0d8393..cd0975b69 100644 --- a/fluid_build/iac/__init__.py +++ b/fluid_build/iac/__init__.py @@ -35,6 +35,7 @@ IAC_PLUGINS, discover_iac_entrypoints, get_iac_plugin, + known_iac_platforms, register_iac_plugin, ) from .shadow import LogicalResource, ShadowReport, shadow_compare @@ -48,6 +49,7 @@ "register_iac_plugin", "get_iac_plugin", "discover_iac_entrypoints", + "known_iac_platforms", "assemble_tofu_document", "build_module", "provider_config", @@ -77,7 +79,8 @@ register_iac_plugin("gcp", GcpIacPlugin()) register_iac_plugin("snowflake", SnowflakeIacPlugin()) -# External clouds (pip-installed) — discovered after the built-ins so a -# third-party package can add (or deliberately override) a cloud with zero -# edits to forge-cli core. -discover_iac_entrypoints() +# External clouds (pip-installed, entry-point group ``fluid_build.iac_providers``) +# are NOT discovered here: ``get_iac_plugin(name)`` loads one by name, the first +# time a contract's platform needs it. Importing this package, and so +# ``fluid --help``, never imports third-party code. A plugin can add a platform; +# it cannot replace a built-in (``registry.reserved_iac_names``). diff --git a/fluid_build/iac/access.py b/fluid_build/iac/access.py index 7acc74957..cde8f1e01 100644 --- a/fluid_build/iac/access.py +++ b/fluid_build/iac/access.py @@ -52,8 +52,9 @@ from __future__ import annotations +import re from dataclasses import dataclass -from typing import Any, Iterable, Mapping, Optional, Tuple +from typing import Any, FrozenSet, Iterable, Mapping, Optional, Tuple #: Principal type prefixes the ``accessPolicy`` schema documents. USER = "user" @@ -73,6 +74,54 @@ _SERVICE_ACCOUNT_SUFFIX = ".gserviceaccount.com" +# The JSONPath scopes a grant's ``resources`` may carry that an emitter can evaluate: one +# expose by id (``$.exposes[?(@.exposeId=='orders')]``, either quote, spaces allowed), or +# every expose (``$.exposes[*]``, ``$.exposes``). Anything else is kept as written and +# reported as unevaluable, never read as "every expose". +_ONE_EXPOSE_RE = re.compile( + r"""\$\.exposes\s*\[\s*\?\s*\(\s*@\.exposeId\s*==\s*(['"])([^'"]+)\1\s*\)\s*\]""" +) +_EVERY_EXPOSE_RE = re.compile(r"\$\.exposes(?:\s*\[\s*\*\s*\])?") + + +@dataclass(frozen=True) +class GrantScope: + """Which exposes a grant's ``resources`` name. + + A grant without ``resources`` covers every expose (``all_exposes``). ``unevaluable`` + holds each resource that is not one of the shapes above, as written: a grant with any + is one an emitter cannot place, and :meth:`covers` is then false for every expose. + """ + + all_exposes: bool = True + expose_ids: FrozenSet[str] = frozenset() + unevaluable: Tuple[str, ...] = () + + def covers(self, expose_id: Any) -> bool: + if self.unevaluable: + return False + return self.all_exposes or str(expose_id) in self.expose_ids + + +def parse_grant_scope(resources: Iterable[str]) -> GrantScope: + """The :class:`GrantScope` of a grant's ``resources``.""" + resources = tuple(resources) + if not resources: + return GrantScope() + every = False + ids: set[str] = set() + unevaluable: list[str] = [] + for resource in resources: + text = resource.strip() + if _EVERY_EXPOSE_RE.fullmatch(text): + every = True + elif one := _ONE_EXPOSE_RE.fullmatch(text): + ids.add(one.group(2)) + else: + unevaluable.append(resource) + return GrantScope(all_exposes=every, expose_ids=frozenset(ids), unevaluable=tuple(unevaluable)) + + @dataclass(frozen=True) class AccessGrant: """One normalized access grant, provider-agnostic. @@ -89,6 +138,11 @@ class AccessGrant: permissions: Tuple[str, ...] resources: Tuple[str, ...] = () + @property + def scope(self) -> GrantScope: + """The exposes ``resources`` names (every expose when it is empty).""" + return parse_grant_scope(self.resources) + @property def is_service_account(self) -> bool: return self.principal_type == SERVICE_ACCOUNT @@ -149,7 +203,9 @@ def _from_access_policy(contract: Mapping[str, Any]) -> Tuple[AccessGrant, ...]: if not split: continue principal, principal_type = split - resources = tuple(str(r) for r in (entry.get("resources") or ()) if str(r).strip()) + # A blank entry is kept: dropped, it would leave an empty list, which means every + # expose; kept, ``parse_grant_scope`` reports it as unevaluable. + resources = tuple(str(r) for r in (entry.get("resources") or ())) grants.append( AccessGrant( principal=principal, @@ -257,9 +313,11 @@ def role_grants( "AccessGrant", "DOMAIN", "GROUP", + "GrantScope", "SERVICE_ACCOUNT", "USER", "normalize_access_grants", + "parse_grant_scope", "role_grants", "uses_legacy_policy_surface", ] diff --git a/fluid_build/iac/base.py b/fluid_build/iac/base.py index 084623211..d23aafdec 100644 --- a/fluid_build/iac/base.py +++ b/fluid_build/iac/base.py @@ -50,8 +50,10 @@ class IacProviderPlugin(Protocol): * In-tree: add a module under ``iac/providers/`` and ``register_iac_plugin`` it in ``iac/__init__.py``. * Out-of-tree (pip-installed): ship an entry-point under - ``fluid_build.iac_providers``; ``registry.discover_iac_entrypoints`` (run - on import, via the unified plugin manager) registers it automatically. + ``fluid_build.iac_providers``. ``registry.get_iac_plugin`` loads it by + name the first time a contract's platform needs it (never at import), + through the unified plugin manager's allow/block policy, the + ``requires_cli`` gate and the reserved built-in names. This mirrors dbt's per-platform adapter pattern (``dbt-bigquery``, ``dbt-snowflake``), which scales that way to 40+ adapters. diff --git a/fluid_build/iac/column_types.py b/fluid_build/iac/column_types.py new file mode 100644 index 000000000..d4ee94fce --- /dev/null +++ b/fluid_build/iac/column_types.py @@ -0,0 +1,153 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""What a column type becomes on a cloud table, and what that costs. + +The AWS (Glue) and GCP (BigQuery) emitters each translate a contract's column +types into their own. A type with no faithful target used to be written anyway: +Glue turned 47 of the 79 types the schema admits into ``string``, and BigQuery +emitted ``ARRAY`` (no such column type: an array is a REPEATED column of an +element type) or a ``STRUCT`` with no fields, which it rejects at apply. Neither +said so, and ``fluid validate --strict`` passed. + +Each emitter now classifies a type as one of: + +* ``exact``: the target type holds every value the declared type does. +* ``lossy``: the target has no such type, a fallback is written (``json`` becomes + ``string`` on Glue) and ``fluid validate`` warns, so ``--strict`` fails. +* ``refused``: there is no form worth writing; validate, ``generate iac`` and + ``apply`` all refuse with an :class:`UnsupportedBindingError` of kind + ``column-type``. + +The classification lives with each emitter's type table (``_hive_outcome`` in +``providers/aws.py``, ``_bq_outcome`` in ``providers/gcp.py``). This module holds +what both share, and :func:`column_type_findings` is the one function the emitter +and the validate-time gate both call, so they cannot disagree. +""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, Iterable, List, Mapping, NamedTuple, Optional, Tuple + +from .base import UnsupportedBindingError + +EXACT = "exact" +LOSSY = "lossy" +REFUSED = "refused" + + +class TypeOutcome(NamedTuple): + """One declared type's translation: ``target`` is ``None`` when refused.""" + + target: Optional[str] + status: str + why: str = "" + fix: str = "" + + +#: Types that carry element or field types, which the contract schema cannot +#: declare yet: ``$defs.column`` is closed, and ``array(string)`` is only a type +#: pattern. No cloud table takes them without that detail. +STRUCTURED = frozenset({"array", "struct", "map", "object", "record", "row"}) + +#: The fix for a refused structured column, whichever cloud refused it. +NESTED_FIX = "Declare json or string for now; typed nested columns are planned." + + +def expand(groups: Mapping[str, str]) -> Dict[str, str]: + """``{"INT64": "int, bigint"}`` as ``{"int": "INT64", "bigint": "INT64"}``: a type + table written one target at a time, read one declared type at a time.""" + return {name.strip(): target for target, names in groups.items() for name in names.split(",")} + + +def exact(target: str) -> TypeOutcome: + return TypeOutcome(target, EXACT) + + +def lossy(target: str, why: str, fix: str) -> TypeOutcome: + return TypeOutcome(target, LOSSY, why, fix) + + +def refused(why: str, fix: str) -> TypeOutcome: + return TypeOutcome(None, REFUSED, why, fix) + + +def split_type(raw: Any) -> Tuple[str, str]: + """``("decimal", "(10,2)")`` for ``"Decimal (10,2)"``; ``("array", "(string)")`` + for ``"ARRAY(string)"``. The base is lower-cased with its words single-spaced + (``timestamp with time zone``); the rest is whatever follows it, spaces dropped. + A missing type is ``string``, as the emitters have always read it.""" + text = " ".join(str(raw if raw is not None else "string").strip().lower().split()) + if not text: + text = "string" + cuts = [i for i in (text.find("("), text.find("<")) if i >= 0] + cut = min(cuts) if cuts else len(text) + return text[:cut].strip(), text[cut:].replace(" ", "") + + +def exposes_of(contract: Any) -> List[Any]: + """The contract's ``exposes`` when it is a list, else none: a schema-invalid contract + is the schema check's to report, and the type gate skips what it cannot read.""" + exposes = contract.get("exposes") if isinstance(contract, Mapping) else None + return exposes if isinstance(exposes, list) else [] + + +def schema_of(exposure: Mapping[str, Any]) -> List[Any]: + """The expose's ``contract.schema`` columns when they are a list, else none.""" + section = exposure.get("contract") + schema = section.get("schema") if isinstance(section, Mapping) else None + return schema if isinstance(schema, list) else [] + + +def column_type_findings( + expose: str, + schema: Iterable[Any], + outcome_for: Callable[[Any], TypeOutcome], + target: str, +) -> Tuple[Optional[UnsupportedBindingError], List[str]]: + """``(refusal, warnings)`` for one expose's columns on ``target`` (a table kind). + + One refusal covers every column the target cannot hold, so the author fixes + them in one pass; each lossy column is its own warning. + """ + parts: List[str] = [] + fixes: Dict[str, List[str]] = {} + warnings: List[str] = [] + for column in schema or []: + if not isinstance(column, Mapping): + continue + name = column.get("name") + declared = str(column.get("type") if column.get("type") is not None else "string") + outcome = outcome_for(column.get("type")) + if outcome.status == REFUSED: + parts.append(f"{name!r} is declared {declared!r}, {outcome.why}") + fixes.setdefault(outcome.fix, []).append(repr(name)) + elif outcome.status == LOSSY: + warnings.append( + f"expose '{expose}': column {name!r} is declared {declared!r}, {outcome.why}; " + f"it is written as {outcome.target!r}. {outcome.fix}" + ) + if not parts: + return None, warnings + refusal = UnsupportedBindingError( + "column-type", + f"expose '{expose}': {target} cannot hold its columns as declared: " + + "; ".join(parts) + + ".", + tuple( + f"{'Column' if len(names) == 1 else 'Columns'} {', '.join(names)}: {fix}" + for fix, names in fixes.items() + ), + ) + return refusal, warnings diff --git a/fluid_build/iac/cutover.py b/fluid_build/iac/cutover.py index 800f8271e..daffff072 100644 --- a/fluid_build/iac/cutover.py +++ b/fluid_build/iac/cutover.py @@ -47,23 +47,27 @@ def default_engine(provider: str) -> str: — it is the strangler-fig cutover switch, and a cloud with an emitter but no entry in the set is deliberately still on its native path. - An **out-of-tree** cloud, registered through the ``fluid_build.iac_providers`` - entry-point group, cannot appear in that frozenset without a core edit — - which is precisely the "zero edits to forge-cli core" promise - ``registry.py`` makes. Such a plugin exists only to emit an OpenTofu - module, so it routes to the OpenTofu engine automatically. Before this, - ``default_engine("")`` returned ``native`` and the IaC path - the plugin was written for was never selected. + An **out-of-tree** cloud, served by an entry point in the + ``fluid_build.iac_providers`` group, cannot appear in that frozenset without + a core edit. Such a plugin exists only to emit an OpenTofu module, so a + platform it serves routes to the OpenTofu engine. This is the point where + the plugin is loaded (``get_iac_plugin``): only a platform that is not + in-tree and not ``local`` triggers an entry-point lookup, and a plugin the + allow/block policy or the version gate refused routes ``native``, so it + is reported as unsupported rather than half-run. """ if provider in OPENTOFU_DEFAULT_PROVIDERS: return "opentofu" # Local import: ``iac/__init__`` pulls every provider plugin, and this # module is imported on the apply cold path. try: - from .registry import IAC_ENTRYPOINT_PLUGINS + from .registry import IAC_ENTRYPOINT_PLUGINS, get_iac_plugin - if provider in IAC_ENTRYPOINT_PLUGINS: - return "opentofu" + # Only a plugin LOADED from the entry-point group counts: an in-tree + # cloud missing from the frozenset is deliberately still native. + if provider and get_iac_plugin(provider) is not None: + if provider.strip().lower().replace("-", "_") in IAC_ENTRYPOINT_PLUGINS: + return "opentofu" except Exception: # noqa: BLE001 — registry unavailable → the safe default pass return "native" diff --git a/fluid_build/iac/governance_validation.py b/fluid_build/iac/governance_validation.py index 322c437f8..0bd0ca790 100644 --- a/fluid_build/iac/governance_validation.py +++ b/fluid_build/iac/governance_validation.py @@ -118,9 +118,15 @@ def validate_governance(contract: Mapping[str, Any]) -> Tuple[List[str], List[st """ from .providers import gcp as _gcp from .providers import gcp_governance as _gov + from .providers.aws import validate_aws_column_types errors: List[str] = [] warnings: List[str] = [] + # Column types a Glue table cannot hold: refused by the emitter, written as + # ``string`` with a warning here, or exact. + type_errors, type_warnings = validate_aws_column_types(contract) + errors.extend(type_errors) + warnings.extend(type_warnings) grants = normalize_access_grants(contract) unenforced: List[str] = [] # AWS Iceberg tables in another catalog: Lake Formation cannot reach them, diff --git a/fluid_build/iac/provider_match.py b/fluid_build/iac/provider_match.py index 7952111d7..15cee6558 100644 --- a/fluid_build/iac/provider_match.py +++ b/fluid_build/iac/provider_match.py @@ -128,11 +128,19 @@ def canonical_cloud(token: object) -> str: """Map a raw contract platform/provider token to a canonical cloud name. Returns the canonical name (``aws``/``gcp``/``snowflake``/``confluent``/ - ``databricks``/``local``) or ``""`` when the token is empty or unrecognised. + ``databricks``/``local``), the name of an installed IaC plugin platform + (an entry point in ``fluid_build.iac_providers``; names only, nothing is + imported), or ``""`` when the token is empty or unrecognised. """ if not isinstance(token, str): return "" - return PROVIDER_ALIASES.get(token.strip().lower().replace("-", "_"), "") + key = token.strip().lower().replace("-", "_") + cloud = PROVIDER_ALIASES.get(key, "") + if cloud or not key: + return cloud + from .registry import is_iac_entrypoint_platform + + return key if is_iac_entrypoint_platform(key) else "" def is_cloud(binding: object, cloud: str) -> bool: diff --git a/fluid_build/iac/providers/aws.py b/fluid_build/iac/providers/aws.py index 17a6c877b..f6965f2b3 100644 --- a/fluid_build/iac/providers/aws.py +++ b/fluid_build/iac/providers/aws.py @@ -79,7 +79,7 @@ ) from ...providers._sql_safety import quote_string_literal, validate_ident from ...providers.aws.util import warehouse as _warehouse -from .. import column_access +from .. import column_access, column_types from ..base import UnsupportedBindingError from ..governance_labels import governance_labels from ..importer import ImportBlock @@ -103,7 +103,7 @@ # binding carries (a jurisdiction such as "EU", a Google region, an unresolved # "{{ env.AWS_REGION }}") must not become the provider's region; it used to be # ignored, and still is. -_AWS_REGION_RE = re.compile(r"^[a-z]{2}(-[a-z]+)+-\d+$") +_AWS_REGION_RE = re.compile(r"^[a-z]{2}(-[a-z]+)+-\d+\Z") def recorded_regions(resources: Iterable[Mapping[str, Any]]) -> set: @@ -187,7 +187,21 @@ def _resolve_catalog_id() -> str: identity = boto3.client("sts").get_caller_identity() value = str(identity.get("Account") or "") - except Exception: # noqa: BLE001 — best-effort; emit no import on failure + except ImportError: + # Loud, not silent: dropping the import block means a Glue database or + # table that already exists is planned as a create and then conflicts. + LOG.warning( + "boto3 is not installed, so the AWS account id could not be looked up and the " + "Glue catalog import blocks are omitted. Install it, or set AWS_ACCOUNT_ID:\n\n" + " pip install 'data-product-forge[aws]'" + ) + value = "" + except Exception as exc: # noqa: BLE001 — best-effort; emit no import on failure + LOG.warning( + "Could not resolve the AWS account id (%s), so the Glue catalog import blocks " + "are omitted. Set AWS_ACCOUNT_ID or configure AWS credentials.", + type(exc).__name__, + ) value = "" _CATALOG_ID_CACHE["value"] = value return value @@ -195,37 +209,230 @@ def _resolve_catalog_id() -> str: _CATALOG_ID_CACHE: Dict[str, str] = {} -# FLUID column type → Hive/Glue column type. -_HIVE_TYPES = { - "string": "string", - "str": "string", - "text": "string", - "integer": "int", - "int": "int", - "int32": "int", - "bigint": "bigint", - "int64": "bigint", - "long": "bigint", - "float": "float", - "float32": "float", - "double": "double", - "float64": "double", - "boolean": "boolean", - "bool": "boolean", - "date": "date", - "timestamp": "timestamp", - "datetime": "timestamp", - "binary": "binary", - "bytes": "binary", +# FLUID column type → Hive/Glue column type, keyed on the lower-cased base name +# (``varchar(255)`` is ``varchar``). Only a type Hive holds without loss is here; +# ``_hive_outcome`` classifies every other one. +# +# Where the choice is not obvious: Hive has no zone-aware timestamp, so the +# ``timestamptz`` family is ``timestamp``, which stores the UTC instant (the +# offset is not kept); ``money`` is SQL Server's ``decimal(19,4)``; ``year`` is +# MySQL's, a four-digit number, so ``smallint``; ``bit`` is the SQL Server +# 0/1 flag. Text, UUIDs and enums are ``string``: Hive has no better form, and +# every engine reading the table treats them as text. +_HIVE_TYPES = column_types.expand( + { + "string": "string, str, text, varchar, varchar2, nvarchar, char, nchar, character, clob, " + "uuid, uniqueidentifier, guid, enum", + "tinyint": "tinyint", + "smallint": "smallint, int2, int16, year", + "int": "int, integer, int4, int32, mediumint, serial", + "bigint": "bigint, int8, int64, long, longint, bigserial", + "float": "float, float4, float32, real", + "double": "double, double precision, float8, float64", + "boolean": "boolean, bool, bit", + "date": "date", + "timestamp": "timestamp, datetime, datetime2, smalldatetime, timestamptz, timestamp_tz, " + "timestamp_ltz, timestamp_ntz, timestampntz, timestamp with time zone, " + "timestamp without time zone", + "binary": "binary, bytes, varbinary, blob, bytea, raw", + "decimal(19,4)": "money", + } +) + +#: Types Hive has no form for but that keep their value as text. Written as +#: ``string``, and ``fluid validate`` warns. +_HIVE_AS_STRING = { + "json": "which Hive has no json type for", + "jsonb": "which Hive has no jsonb type for", + "variant": "which Hive has no variant type for", + "super": "which Hive has no super type for", + "time": "which Hive has no time-of-day type for", + "interval": "which Hive has no interval column type for", + "bignumeric": "whose digits beyond 38 no Hive decimal holds", } +#: Types with no Hive form and no fallback worth writing: which encoding a spatial +#: value or a sketch takes is the writer's, and a guess would be wrong half the time. +_HIVE_REFUSED = { + **{ + t: ( + "which has no Hive column type (a spatial value is stored as WKT text or WKB bytes, " + "and only the writer knows which)", + "Declare string (WKT) or binary (WKB), whichever the data holds.", + ) + for t in ("geography", "geometry", "geom", "point") + }, + "hll": ( + "which is an engine-specific sketch with no Hive column type", + "Declare binary to store the sketch's bytes.", + ), +} + +_DECIMAL_FAMILY = frozenset({"decimal", "numeric", "dec", "number"}) +# ASCII digits only: ``\d`` alone also matches Arabic-Indic and other Unicode digits. +_DECIMAL_ARGS_RE = re.compile(r"^\((\d+)(?:,(\d+))?\)$", re.ASCII) +_HIVE_DECIMAL_DEFAULT = "decimal(38,9)" +_HIVE_TYPE_FIX = ( + "Declare a FLUID column type such as string, integer, bigint, decimal(12,2), double, " + "boolean, date or timestamp." +) + + +def _hive_decimal(rest: str) -> column_types.TypeOutcome: + """``decimal(p,s)`` passes through as written; a bare one widens to a safe default.""" + if not rest: + return column_types.exact(_HIVE_DECIMAL_DEFAULT) + match = _DECIMAL_ARGS_RE.match(rest) + if match is not None: + precision, scale = int(match.group(1)), int(match.group(2) or 0) + if 1 <= precision <= 38 and scale <= precision: + return column_types.exact(f"decimal{rest}") + return column_types.refused( + f"whose precision and scale are not a Hive decimal's ({rest})", + "Declare decimal(p,s) with p from 1 to 38 and s no larger than p, or string.", + ) + + +def _hive_outcome(raw: Any) -> column_types.TypeOutcome: + """What ``raw`` becomes in a Glue table: its Hive type, exact, lossy or refused.""" + base, rest = column_types.split_type(raw) + if base in column_types.STRUCTURED: + return column_types.refused( + "which Glue holds only with its element or field types, and the contract schema " + "cannot declare them yet", + column_types.NESTED_FIX, + ) + if not rest.isascii(): + return column_types.refused( + f"whose parameters ({rest}) are not ASCII", "Declare the type with ASCII digits." + ) + if base in _DECIMAL_FAMILY: + return _hive_decimal(rest) + if base == "bit" and rest not in ("", "(1)"): + return column_types.refused( + "which is a bit string, and Hive has none", "Declare binary or an integer type." + ) + if base in _HIVE_REFUSED: + return column_types.refused(*_HIVE_REFUSED[base]) + if base in _HIVE_AS_STRING: + return column_types.lossy( + "string", + _HIVE_AS_STRING[base], + "Declare string to store the text on purpose; this warning then goes away.", + ) + hit = _HIVE_TYPES.get(base) + if hit is None: + return column_types.refused("which the Glue emitter does not map", _HIVE_TYPE_FIX) + if base == "float" and rest: + # ``float(53)`` is double precision, as in PostgreSQL and SQL Server. + digits = rest.strip("()") + if digits.isascii() and digits.isdigit() and int(digits) > 24: + return column_types.exact("double") + return column_types.exact(hit) + def _hive_type(raw: Any) -> str: - t = str(raw or "string").strip().lower() - if t.startswith(("decimal", "numeric")): - # decimal(10,2) passes through; a bare type widens to a safe default. - return t.replace("numeric", "decimal") if "(" in t else "decimal(38,9)" - return _HIVE_TYPES.get(t, "string") + """The Hive type ``raw`` is written as. Total, for ``fluid verify`` and ``fluid diff`` + to compare a live table with: a refused type reads as ``string`` here, and is + refused where the module is emitted (:func:`_refuse_glue_column_types`).""" + return _hive_outcome(raw).target or "string" + + +def _glue_column_type_findings( + expose: str, schema: Iterable[Any] +) -> Tuple[Optional[UnsupportedBindingError], List[str]]: + return column_types.column_type_findings(expose, schema, _hive_outcome, "AWS Glue") + + +def _glue_table_declared(binding: Mapping[str, Any]) -> bool: + """Does :func:`_emit_glue` write a table for ``binding``? Its three gates, for the + validate-time check to apply the column types where the emitter does.""" + loc = binding.get("location") + if not isinstance(loc, Mapping): + return False + fmt = binding.get("format") or "parquet" + return bool(loc.get("database") and loc.get("table") and _glue_cataloged(binding, fmt)) + + +def validate_aws_column_types(contract: Mapping[str, Any]) -> Tuple[List[str], List[str]]: + """``(errors, warnings)`` for the column types of the contract's Glue tables: the + refusal :func:`_emit_glue` raises, and a warning for each type it writes as a + fallback. ``fluid validate`` runs this so ``generate iac`` finds nothing new.""" + errors: List[str] = [] + warnings: List[str] = [] + for index, exposure in enumerate(column_types.exposes_of(contract)): + binding = exposure.get("binding") if isinstance(exposure, Mapping) else None + if not isinstance(binding, Mapping) or not is_cloud(binding, "aws"): + continue + if not _glue_table_declared(binding): + continue + schema = column_types.schema_of(exposure) + refusal, lossy = _glue_column_type_findings(_expose_id(exposure) or str(index), schema) + if refusal is not None: + errors.append(f"{refusal} {' '.join(refusal.remediation)}".strip()) + warnings.extend(lossy) + return errors, warnings + + +def _plan_glue_columns(state: Any) -> Dict[str, str]: + """``{column: type}`` of a planned ``aws_glue_catalog_table`` (``tofu show -json`` + renders its one ``storage_descriptor`` block as a one-item list).""" + if not isinstance(state, Mapping): + return {} + descriptor = state.get("storage_descriptor") + if isinstance(descriptor, list): + descriptor = descriptor[0] if descriptor else None + if not isinstance(descriptor, Mapping): + return {} + return { + str(c.get("name")): str(c.get("type")) + for c in descriptor.get("columns") or [] + if isinstance(c, Mapping) + } + + +def glue_column_type_changes(plan: Mapping[str, Any]) -> List[str]: + """One line per Glue table the plan updates in place with other column types. + + A column type change plans as a plain ``~`` and leaves the data files alone, but + Parquet written under the old type (Athena INSERT or CTAS, dbt: ``string``) holds + values the new type cannot read, and Athena then fails every read of those files. + So each line is data the apply puts at risk, and the engine's data-loss gate + (``plan_data_changes``) holds the apply until ``--allow-data-loss``. A difference + in spacing alone (``decimal(10, 2)`` against ``decimal(10,2)``) is no change. + Created and removed tables, and updates that keep every column's type, give none. + """ + lines: List[str] = [] + for change in plan.get("resource_changes") or []: + if not isinstance(change, Mapping) or change.get("type") != "aws_glue_catalog_table": + continue + detail = change.get("change") + if not isinstance(detail, Mapping) or list(detail.get("actions") or []) != ["update"]: + continue + before, after = detail.get("before"), detail.get("after") + old, new = _plan_glue_columns(before), _plan_glue_columns(after) + changed = [ + f"{name}: {old[name]} -> {new[name]}" + for name in old + if name in new and _unspaced(old[name]) != _unspaced(new[name]) + ] + if not changed: + continue + table = ".".join( + str(before.get(key) or "") + for key in ("database_name", "name") + if isinstance(before, Mapping) + ) + lines.append( + f"Glue table {table}: column types change in place ({'; '.join(changed)}); files " + "written under the old type may stop reading until they are rewritten, and " + "reverting the column type restores reads" + ) + return lines + + +def _unspaced(glue_type: str) -> str: + return "".join(glue_type.split()) def _columns(schema: List[Mapping[str, Any]]) -> List[Dict[str, str]]: @@ -463,6 +670,7 @@ def emit( contract=contract, placement=placement, labels=gov_labels, + expose=_expose_id(exposure) or str(index), ) _emit_s3(resources, loc, cid, tags, placement=placement) _emit_kinesis(resources, loc, cid, tags) @@ -727,6 +935,11 @@ def recorded_regions(self, resources: Iterable[Mapping[str, Any]]) -> set: """The regions existing state places resources in; see :func:`recorded_regions`.""" return recorded_regions(resources) + def plan_data_changes(self, plan: Mapping[str, Any]) -> List[str]: + """The Glue tables the saved plan gives other column types in place; each needs + ``--allow-data-loss``. See :func:`glue_column_type_changes`.""" + return glue_column_type_changes(plan) + def provider_block_for(self, contract: Mapping[str, Any]) -> Dict[str, Any]: """``provider_block``, plus the region the contract's AWS bindings name. @@ -741,6 +954,29 @@ def provider_block_for(self, contract: Mapping[str, Any]) -> Dict[str, Any]: provider v6). """ cfg = dict(self.provider_block()) + for index, expose in enumerate(contract.get("exposes") or []): + binding = expose.get("binding") or {} + raw = (binding.get("location") or {}).get("region") + # A region code with whitespace around it (a YAML ``|`` block ends in a + # newline) is refused. Dropping it leaves the provider on the + # environment's region; pinning it stripped applies a region the + # sovereignty policy, which reads the value as written, never checked. + if ( + is_cloud(binding, "aws") + and isinstance(raw, str) + and raw != raw.strip() + and _AWS_REGION_RE.match(raw.strip()) + ): + raise UnsupportedBindingError( + "aws-region-whitespace", + f"exposes[{expose.get('exposeId') or index}] binding.location.region is " + f"{raw!r}: a region code with whitespace around it. The sovereignty policy " + "checks the region as written, so it is refused rather than pinned stripped.", + ( + f"Write the region alone: region: {raw.strip()} (a YAML '|' block ends " + "the value in a newline).", + ), + ) named = { str(((e.get("binding") or {}).get("location") or {}).get("region")) for e in contract.get("exposes") or [] @@ -926,6 +1162,7 @@ def _emit_glue( contract: Optional[Mapping[str, Any]] = None, placement: _Placement = _LEGACY_PLACEMENT, labels: Optional[Mapping[str, str]] = None, + expose: str = "?", ) -> None: database = loc.get("database") if not database: @@ -961,6 +1198,11 @@ def _emit_glue( # had write-once-on-create semantics and never reconciled the # parameters again. Mirrors how the Glue Terraform Registry # examples model catalog metadata + descriptions in one resource. + # A column Glue cannot hold is refused, not written as ``string``: the same + # findings ``fluid validate`` reports (:func:`validate_aws_column_types`). + refusal, _lossy = _glue_column_type_findings(expose, schema) + if refusal is not None: + raise refusal storage: Dict[str, Any] = {"columns": _columns(schema)} # Athena reads a Glue table through the Hive input format and SerDe the # table declares. Without them every query failed with HIVE_UNSUPPORTED_FORMAT diff --git a/fluid_build/iac/providers/databricks.py b/fluid_build/iac/providers/databricks.py index 73185f8d5..84626e51f 100644 --- a/fluid_build/iac/providers/databricks.py +++ b/fluid_build/iac/providers/databricks.py @@ -31,7 +31,10 @@ set: Delta drops and renames columns only under column mapping. * ``accessPolicy.grants[]`` and ``policy.authz.readers`` / ``writers`` become one ``databricks_grant`` per (table, principal), plus ``USE_SCHEMA`` on the - schema. ``databricks_grant`` is authoritative for its principal on its + schema. A grant's ``resources`` places it on the tables of the exposes it names + (``$.exposes[?(@.exposeId=='')]``, or ``$.exposes[*]`` / ``$.exposes`` for + all; ``iac/access.py::parse_grant_scope``); any other JSONPath, or an exposeId + the contract does not have, is refused, never widened. ``databricks_grant`` is authoritative for its principal on its securable, so a principal named twice is merged into one resource rather than two that undo each other. It also means the product takes over each named principal's privileges on its schema: on a schema that existed before @@ -72,6 +75,10 @@ databricks``) becomes a serverless ``databricks_job`` with a ``dbt_task``, and a local project its ``databricks_workspace_file`` uploads (``databricks_jobs.py``). +* ``lifecycle.expire`` is accepted on a table that is rebuilt in full on every run (a dbt + ``table`` model, or forge-cli's DuckDB acquisition) when ``lifecycle.retention`` covers the + table's Delta history, so ``VACUUM`` removes older copies; anything else is refused + (``databricks_retention.py``). ``fluid verify`` reads the live properties. * A table a dbt build writes, locally or as a job, is rebuilt by dbt's ``table`` materialization with CREATE OR REPLACE, so the module creates it and then leaves its definition to dbt: ``lifecycle.ignore_changes`` on what that @@ -123,6 +130,7 @@ from __future__ import annotations +import json import logging import os import re @@ -130,7 +138,12 @@ from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple from ...providers._sql_safety import validate_ident, validate_sql_type_param_payload -from ...util.contract import is_databricks_job_build +from ...util.binding_paths import ENV_PLACEHOLDER_RE +from ...util.contract import ( + builds_writing_expose, + is_databricks_job_build, + is_duckdb_acquisition_build, +) from ..access import normalize_access_grants from ..base import UnsupportedBindingError from ..governance_labels import governance_labels @@ -155,6 +168,7 @@ dbt_written_exposes, run_as_allowlist, ) +from .databricks_retention import Retention, check_expire _LOG = logging.getLogger(__name__) @@ -259,6 +273,8 @@ class _Table: region: str = "" #: The column masks and row filter Unity Catalog enforces on it, if the expose has any. enforcement: Optional[TableEnforcement] = None + #: The ``lifecycle.expire`` it promises and this module accepted (``databricks_retention.py``). + retention: Optional[Retention] = None @property def full_name(self) -> str: @@ -434,20 +450,141 @@ def _packaging_refusal(where: str) -> UnsupportedBindingError: ) +def _describe_writer(build: Mapping[str, Any]) -> str: + engine = build.get("engine") + parts = [f"engine {engine!r}" if engine else "no engine"] + if build.get("pattern"): + parts.append(f"pattern {build.get('pattern')!r}") + return f"build {build.get('id')!r} ({', '.join(parts)})" + + +def masking_unapplied_reason(exposure: Mapping[str, Any], builds: Any) -> Optional[str]: + """Why no mask would be applied to this expose's rows, or ``None`` when one is. + + ``policy.privacy.masking`` is applied by the DuckDB acquisition runner at landing, before + the rows leave the machine (``build_runners/_masking.py``), so it holds on a Databricks + table only when every build that writes the table is such a build + (``is_duckdb_acquisition_build``). A dbt model, another engine, or no build at all writes + rows nothing masked. + """ + writers = builds_writing_expose(exposure.get("exposeId"), builds) + if not writers: + return "no build writes it, so nothing would apply the mask." + unmasked = [build for build in writers if not is_duckdb_acquisition_build(build)] + if not unmasked: + return None + return ( + f"{', '.join(_describe_writer(build) for build in unmasked)} " + f"{'writes' if len(unmasked) == 1 else 'write'} it and applies no mask." + ) + + +def _refuse_unapplied_masking( + where: str, + exposure: Mapping[str, Any], + builds: Sequence[Mapping[str, Any]], + refusals: List[UnsupportedBindingError], +) -> None: + """``policy.privacy.masking`` on an expose no DuckDB acquisition build alone writes.""" + policy = exposure.get("policy") + privacy = policy.get("privacy") if isinstance(policy, Mapping) else None + if not isinstance(privacy, Mapping) or not _set(privacy.get("masking")): + return + unapplied = masking_unapplied_reason(exposure, builds) + if unapplied is None: + return + refusals.append( + _refusal( + "governance-unmapped", + f"{where} declares policy.privacy.masking, which only forge-cli's DuckDB " + "acquisition runner applies, at landing, before the rows leave the machine. " + f"{unapplied[:1].upper()}{unapplied[1:]} Applied anyway, the table would hold the " + "values the contract promises to mask.", + "Write this expose with a DuckDB acquisition build (pattern: acquisition, " + "engine: duckdb), which masks the columns before it loads the table. Or remove " + "policy.privacy.masking and enforce access with policy.authz.columnRestrictions, " + "which become Unity Catalog column masks.", + ) + ) + + +def _refuse_unlandable( + where: str, + exposure: Mapping[str, Any], + builds: Sequence[Mapping[str, Any]], + refusals: List[UnsupportedBindingError], + *, + validating: bool = False, +) -> None: + """What the DuckDB acquisition runner would refuse at run time for this expose, refused + here: by then ``fluid apply`` has created the table. Its source mode and sink format + (``build_runners/_databricks_load.unsupported_reason``) and, when it masks at landing, + each masking rule (``build_runners/_masking.masking_rules``). A value that is still an + ``{{ env.X }}`` placeholder is left alone only while validating (``fluid validate`` reads + the contract unsubstituted); the emitter reads it substituted, and one still unresolved + there (its variable unset) is refused.""" + from ...build_runners._databricks_load import unsupported_reason + from ...build_runners._masking import MaskingPolicyError, masking_rules + + writers = [ + b + for b in builds_writing_expose(exposure.get("exposeId"), builds) + if is_duckdb_acquisition_build(b) + ] + for build in writers: + properties = build.get("properties") + properties = properties if isinstance(properties, Mapping) else {} + source = properties.get("source") + sink = properties.get("sink") + mode = source.get("mode") if isinstance(source, Mapping) else None + sink_format = (sink.get("format") if isinstance(sink, Mapping) else None) or "parquet" + placeholder = any( + isinstance(v, str) and ENV_PLACEHOLDER_RE.search(v) for v in (mode, sink_format) + ) + if validating and placeholder: + continue + reason = unsupported_reason(str(mode or ""), str(sink_format).lower(), 1) + if reason: + refusals.append( + _refusal( + "landing-unsupported", + f"{where}: build {build.get('id')!r} lands this expose with forge-cli's " + f"DuckDB acquisition runner, and {reason}. The build would be refused at " + "run time, after fluid apply had created the table.", + "Set properties.source.mode: full_refresh and properties.sink.format: " + "parquet on the build (one stream), or write the expose another way.", + ) + ) + if writers and not masking_unapplied_reason(exposure, builds): + try: + masking_rules(exposure) + except MaskingPolicyError as exc: + refusals.append( + _refusal( + "masking-invalid", + f"{where} policy.privacy.masking: {exc} The DuckDB acquisition runner " + "would refuse to land anything.", + "Correct the rule as the message says, or remove it from " + "policy.privacy.masking.", + ) + ) + + def _refuse_governance( where: str, binding: Mapping[str, Any], exposure: Mapping[str, Any], refusals: List[UnsupportedBindingError], + builds: Sequence[Mapping[str, Any]] = (), ) -> None: """Governance the module would apply with exit 0 but never enforce.""" policy = exposure.get("policy") or {} authz = (policy.get("authz") or {}) if isinstance(policy, Mapping) else {} privacy = (policy.get("privacy") or {}) if isinstance(policy, Mapping) else {} + _refuse_unapplied_masking(where, exposure, builds, refusals) unmapped = [ name for name, value in ( - ("policy.privacy.masking", privacy.get("masking")), ("policy.privacy.rowLevelPolicy", privacy.get("rowLevelPolicy")), ("binding.governance", binding.get("governance")), ) @@ -460,8 +597,8 @@ def _refuse_governance( f"{where} declares {', '.join(unmapped)}, which the Databricks emitter does not " "map yet. Applied anyway, the table would be readable without the mask, filter " "or restriction the contract promises.", - "Remove them from this expose until they are mapped (landing-time masking and " - "ABAC policies are not), and enforce access with grants. policy.authz." + "Remove them from this expose until they are mapped (ABAC policies are not), " + "and enforce access with grants. policy.authz." "columnRestrictions and policy.authz.rowFilters are mapped: they become Unity " "Catalog column masks and a row filter.", ) @@ -475,17 +612,6 @@ def _refuse_governance( "Remove binding.encryption; configure customer-managed keys on the workspace.", ) ) - lifecycle = exposure.get("lifecycle") or {} - if isinstance(lifecycle, Mapping) and lifecycle.get("expire"): - refusals.append( - _refusal( - "retention-unmapped", - f"{where} sets lifecycle.expire, and nothing on Databricks would expire the " - "data.", - "Remove lifecycle.expire from the databricks binding, or schedule the delete " - "in a build.", - ) - ) def _refuse_shape( @@ -671,7 +797,9 @@ def _resolve_table( exposure: Mapping[str, Any], refusals: List[UnsupportedBindingError], *, + contract: Mapping[str, Any], validating: bool, + builds: Sequence[Mapping[str, Any]] = (), ) -> Optional[_Table]: """One Databricks expose → :class:`_Table`, or ``None`` with its refusals.""" binding = exposure.get("binding") or {} @@ -685,7 +813,12 @@ def _resolve_table( columns = _columns(where, exposure, refusals, validating=validating) partitions = _partitions(where, loc, columns, refusals, validating=validating) region = _region(where, loc, refusals) - _refuse_governance(where, binding, exposure, refusals) + _refuse_governance(where, binding, exposure, refusals, builds) + _refuse_unlandable(where, exposure, builds, refusals, validating=validating) + retention, retention_refusals = check_expire( + where, exposure, names.get("table") or "", contract, validating=validating + ) + refusals.extend(retention_refusals) enforcement: Optional[TableEnforcement] = None if len(refusals) == before: try: @@ -715,6 +848,7 @@ def _resolve_table( exposure=exposure, region=region, enforcement=enforcement, + retention=retention, ) @@ -860,17 +994,38 @@ def _principal_identities( def _refuse_grant_shapes( contract: Mapping[str, Any], grants: Sequence[Any], refusals: List[UnsupportedBindingError] ) -> None: - """Grants a Unity Catalog grant cannot express: scoped by resources, or conditional.""" + """Grants a Unity Catalog grant cannot express: scoped by resources it cannot evaluate or + by an expose the contract does not have, or conditional.""" + expose_ids = { + str(e.get("exposeId")) for e in contract.get("exposes") or () if isinstance(e, Mapping) + } + fix = ( + "Scope the grant with $.exposes[?(@.exposeId=='')] for one expose (repeat it for " + "several) or $.exposes[*] for every expose, or drop resources (every table of the " + "product)." + ) for grant in grants: - if grant.resources: + scope = grant.scope + if scope.unevaluable: refusals.append( _refusal( "grant-scope-unmapped", - f"accessPolicy grant to {grant_key(grant)!r} is scoped by resources, which " - "the Databricks emitter does not evaluate: the grant would reach no " - "Databricks table.", - "Drop resources (the grant then covers every table of the product), or " - "use the expose's policy.authz.readers / writers.", + f"accessPolicy grant to {grant_key(grant)!r} is scoped by " + f"{', '.join(repr(r) for r in scope.unevaluable)}, which the Databricks " + "emitter cannot evaluate, so it cannot tell which tables the grant is for. " + "It is refused rather than placed on more tables than it names.", + fix, + ) + ) + unknown = sorted(scope.expose_ids - expose_ids) + if unknown: + refusals.append( + _refusal( + "grant-scope-unmapped", + f"accessPolicy grant to {grant_key(grant)!r} is scoped to exposeId " + f"{', '.join(repr(i) for i in unknown)}, which no expose of this contract " + "has.", + "Correct the exposeId in resources, or remove that resource. " + fix, ) ) for entry in (contract.get("accessPolicy") or {}).get("grants") or (): @@ -891,7 +1046,7 @@ def _target_privileges( """``{principal: privileges}`` on one expose's table, from accessPolicy and its authz.""" named: List[Tuple[Tuple[str, ...], set]] = [] for grant in grants: - if grant.resources: + if not grant.scope.covers(target.exposure.get("exposeId")): continue identities = _principal_identities( grant_key(grant), target, refusals, fallback=grant.principal @@ -1032,7 +1187,8 @@ def _compile( if _set(contract.get("packaging")): refusals.append(_packaging_refusal("the contract")) - targets = _resolve_tables(compiled, exposures, validating=validating) + builds = [b for b in contract.get("builds") or [] if isinstance(b, Mapping)] + targets = _resolve_tables(compiled, exposures, contract, validating=validating, builds=builds) warehouse = _warehouse(env, refusals) _refuse_duplicate_tables(compiled.tables, refusals) _refuse_region_conflict(compiled.tables, refusals) @@ -1071,12 +1227,24 @@ def _compile( def _resolve_tables( - compiled: _Compiled, exposures: Sequence[Tuple[int, Mapping[str, Any]]], *, validating: bool + compiled: _Compiled, + exposures: Sequence[Tuple[int, Mapping[str, Any]]], + contract: Mapping[str, Any], + *, + validating: bool, + builds: Sequence[Mapping[str, Any]] = (), ) -> List[_GrantTarget]: """Resolve each Databricks expose into ``compiled.tables``; a grant target for every one.""" targets: List[_GrantTarget] = [] for index, exposure in exposures: - table = _resolve_table(index, exposure, compiled.refusals, validating=validating) + table = _resolve_table( + index, + exposure, + compiled.refusals, + contract=contract, + validating=validating, + builds=builds, + ) if table is not None: compiled.tables.append(table) targets.append( @@ -1182,6 +1350,40 @@ def _emit_table( compiled.imports.append(ImportBlock(f"databricks_sql_table.{table_key}", table.full_name)) +#: The residency precondition's own message (``_region_precondition``), and the diagnostic +#: ``values`` entry that carries the live region (``statement: is "us-west-2"``). In ``tofu +#: -json`` output the text is one JSON string; in human output it is wrapped, hence ``\s+``. +_RESIDENCY_RE = re.compile( + r"The\s+contract\s+declares\s+binding\.location\.region\s+([a-z0-9-]+),\s+but\s+the\s+" + r"workspace\s+DATABRICKS_HOST\s+names\s+is\s+in\s+(\S+?)\.?\s+Residency" +) +_RESIDENCY_START_RE = re.compile( + r"The\s+contract\s+declares\s+binding\.location\.region\s+([a-z0-9-]+)," +) +_LIVE_VALUE_RE = re.compile(r'is\s+\\?"([a-z0-9-]+)\\?"') + + +def _residency_failure(output: str) -> Optional[UnsupportedBindingError]: + """One clean line for a plan that fails the residency precondition: the declared region, + the live one and the fix. ``None`` when the output is not that failure.""" + start = _RESIDENCY_START_RE.search(output) + if start is None: + return None + declared = start.group(1) + full = _RESIDENCY_RE.search(output) + live = full.group(2) if full else "" + if not re.fullmatch(r"[a-z0-9-]+", live): # "(unknown)", a value not yet known, or cut off + value = _LIVE_VALUE_RE.search(output) + live = value.group(1) if value else "" + seen = f"in {live}" if live else "in a region the plan output does not show" + return _refusal( + "residency-region", + f"the contract declares binding.location.region {declared}, but the workspace " + f"DATABRICKS_HOST names is {seen}. Point DATABRICKS_HOST at a workspace in {declared}, " + "or correct binding.location.region.", + ) + + def _region_precondition(compiled: _Compiled, cid: str, region: str) -> Dict[str, Any]: """The plan-time check that the workspace is in the declared ``region``. @@ -1562,7 +1764,11 @@ def emit( if compiled.refusals: raise compiled.refusals[0] _enforce_sovereignty(contract, compiled.tables) - for job, _tables in _jobs_to_pause(compiled, contract): + # Decided once, here: ``plan_notices`` of the same run says what this paused, so a + # transient read error cannot pause a job without the notice (or the reverse). + to_pause = _jobs_to_pause(compiled, contract) + self._pause_decision = (_contract_key(contract), [(j.name, t) for j, t in to_pause]) + for job, _tables in to_pause: _pause(compiled, job) return compiled.resources @@ -1723,9 +1929,15 @@ def plan_notices(self, contract: Mapping[str, Any]) -> List[str]: it creates paused until the tables they write are protected.""" compiled = _compile(contract, os.environ) notices = _use_catalog_notices(compiled) - for job, tables in _jobs_to_pause(compiled, contract): + decided = getattr(self, "_pause_decision", None) + self._pause_decision = None # used once: the next run decides again + if decided is not None and decided[0] is not None and decided[0] == _contract_key(contract): + paused = list(decided[1]) + else: + paused = [(j.name, t) for j, t in _jobs_to_pause(compiled, contract)] + for name, tables in paused: notices.append( - f"job {job.name!r} is created paused (pause_status PAUSED): it writes " + f"job {name!r} is created paused (pause_status PAUSED): it writes " f"{', '.join(sorted(tables))}, whose masks and row filter are not bound yet, and " "a run before that would fill an unprotected table. The next fluid apply, once " "they are bound, starts its schedule; an explicit run-now works meanwhile." @@ -1752,6 +1964,9 @@ def explain_plan_failure( "DATABRICKS_HOST with DATABRICKS_CLIENT_ID and DATABRICKS_CLIENT_SECRET (a " "service principal's OAuth secret), then re-run 'fluid apply'.", ) + residency = _residency_failure(output or "") + if residency is not None: + return residency if _UNREACHABLE_RE.search(output or ""): return _refusal( "workspace-unreachable", @@ -1842,6 +2057,16 @@ def _column_details( return out +def _contract_key(contract: Mapping[str, Any]) -> Optional[str]: + """A stable text of the contract, to tie a decision to the contract it was made for; + ``None`` for one JSON cannot write (mixed-type or date keys, a cycle), and then no + decision is ever reused for it: :meth:`plan_notices` works it out again.""" + try: + return json.dumps(contract, sort_keys=True, default=str) + except (TypeError, ValueError): + return None + + def _jobs_to_pause( compiled: _Compiled, contract: Mapping[str, Any] ) -> List[Tuple[DbtJob, Dict[str, str]]]: @@ -2031,6 +2256,10 @@ class ManagedTable: #: The whole expectation, with each function's definition: what ``fluid verify`` checks #: a bound function's owner, body, parameters and return type against. enforcement: Optional[TableEnforcement] = None + #: The accepted ``lifecycle.expire``: its retention as written and in seconds, which the + #: table's Delta history must not exceed. ``None`` when the expose sets no expire. + retention_period: Optional[str] = None + retention_seconds: Optional[int] = None def managed_objects( @@ -2076,6 +2305,8 @@ def managed_objects( for mask in (table.enforcement.masks if table.enforcement else ()) ), enforcement=table.enforcement, + retention_period=table.retention.period if table.retention else None, + retention_seconds=table.retention.seconds if table.retention else None, ) for table in compiled.tables ] diff --git a/fluid_build/iac/providers/databricks_enforce.py b/fluid_build/iac/providers/databricks_enforce.py index 83fbc8f23..23dbc72d4 100644 --- a/fluid_build/iac/providers/databricks_enforce.py +++ b/fluid_build/iac/providers/databricks_enforce.py @@ -26,7 +26,9 @@ * :func:`post_apply` binds whatever is still missing (the tables the apply just created, empty at that moment), and drops what the contract no longer asks for: a mask or filter whose function carries this module's name prefix. A filter an admin set by hand is left alone. - Dropping widens access, so it waits for the apply to succeed and is printed as such. + Dropping widens access, so it waits for the apply to succeed and is printed as such. After an + apply that succeeded it also drops this module's superseded functions that no table in the + schema uses and this run's identity owns (:func:`drop_orphan_functions`). Both read the live bindings first (``GET /api/2.1/unity-catalog/tables/{name}``: ``row_filter.function_name`` / ``input_column_names`` and ``columns[].mask.function_name``, @@ -47,6 +49,7 @@ from __future__ import annotations import os +import re import time from dataclasses import dataclass, field from typing import Any, Callable, Collection, Dict, List, Mapping, Optional, Sequence, Tuple @@ -144,7 +147,7 @@ def default_client() -> Any: return WorkspaceClient(**config) -def _scrub(text: Any) -> str: +def scrub(text: Any) -> str: """One short line with every credential value in the environment removed.""" from ...observability.secret_redactor import redact_secret_text from .databricks_auth import credential_values @@ -155,7 +158,7 @@ def _scrub(text: Any) -> str: return line if len(line) <= _MAX_MESSAGE else line[: _MAX_MESSAGE - 3] + "..." -def _is_not_found(exc: BaseException) -> bool: +def is_not_found(exc: BaseException) -> bool: for attr in ("status_code", "status", "http_status"): if getattr(exc, attr, None) == 404: return True @@ -177,7 +180,7 @@ def read_table(client: Any, full_name: str) -> Optional[Dict[str, Any]]: "GET", f"/api/2.1/unity-catalog/tables/{quote(full_name, safe='.')}" ) except Exception as exc: # noqa: BLE001 — a 404 is an answer, anything else is raised on - if _is_not_found(exc): + if is_not_found(exc): return None raise return out if isinstance(out, dict) else {} @@ -190,7 +193,7 @@ def read_function(client: Any, full_name: str) -> Optional[Dict[str, Any]]: "GET", f"/api/2.1/unity-catalog/functions/{quote(full_name, safe='.')}" ) except Exception as exc: # noqa: BLE001 — a 404 is an answer, anything else is raised on - if _is_not_found(exc): + if is_not_found(exc): return None raise return out if isinstance(out, dict) else {} @@ -218,6 +221,12 @@ def _type(text: Any) -> str: return "".join(str(text or "").split()).lower() +def owner_matches(owner: Any, owners: Collection[str]) -> bool: + """Whether a function's ``owner`` is one of ``owners`` (case-insensitive): the one comparison + the read-back of a function and the orphan drop share.""" + return str(owner or "").lower() in {o.lower() for o in owners} + + def function_problem( info: Optional[Mapping[str, Any]], expected: Function, owners: Collection[str] ) -> Optional[str]: @@ -233,7 +242,7 @@ def function_problem( return f"{expected.full_name} does not exist" problems: List[str] = [] owner = str(info.get("owner") or "") - if owner.lower() not in {o.lower() for o in owners}: + if not owner_matches(owner, owners): problems.append(f"its owner is {owner or 'unknown'}, not {' or '.join(sorted(owners))}") body_language = str(info.get("routine_body") or "SQL").upper() if body_language != "SQL": @@ -281,7 +290,12 @@ def __init__( def run(self, statement: str) -> None: """Run one statement to ``SUCCEEDED``; :class:`StatementFailed` otherwise.""" + self.execute(statement) + + def execute(self, statement: str) -> Mapping[str, Any]: + """:meth:`run`, and the finished statement's document (its ``result`` and ``manifest``).""" do = self._client.api_client.do + doc: Any = None try: doc = do( "POST", @@ -303,13 +317,20 @@ def run(self, statement: str) -> None: except StatementFailed: raise except Exception as exc: # noqa: BLE001 — every transport failure is the statement's - raise StatementFailed(f"{type(exc).__name__}: {_scrub(exc)}") from exc + raise StatementFailed(f"{type(exc).__name__}: {scrub(exc)}") from exc + except BaseException: + # Ctrl-C or SystemExit while the statement runs: it would go on running on the + # warehouse after this process has gone. + if isinstance(doc, Mapping) and doc.get("statement_id"): + self._cancel(doc) + raise state = self._state(doc) if state != "SUCCEEDED": error = (doc.get("status") or {}).get("error") or {} raise StatementFailed( - f"{state}: {_scrub(error.get('message') or 'no message from the server')}" + f"{state}: {scrub(error.get('message') or 'no message from the server')}" ) + return doc @staticmethod def _state(doc: Any) -> str: @@ -504,7 +525,7 @@ def run(self, step: Step) -> None: raise except Exception as exc: # noqa: BLE001 raise FunctionMismatch( - f"could not read {step.function.full_name}: {type(exc).__name__}: {_scrub(exc)}" + f"could not read {step.function.full_name}: {type(exc).__name__}: {scrub(exc)}" ) from exc if problem is not None: raise FunctionMismatch( @@ -559,7 +580,7 @@ def _client_for( if required: raise EnforcementError( f"could not create the Databricks client for the masks and row filters: " - f"{type(exc).__name__}: {_scrub(exc)}" + f"{type(exc).__name__}: {scrub(exc)}" ) from exc say( " note: masks and row filters an earlier apply bound were not checked for removal " @@ -574,7 +595,7 @@ def _read(client: Any, target: Target) -> Optional[Dict[str, Any]]: except Exception as exc: # noqa: BLE001 raise EnforcementError( f"could not read {target.full_name} from Unity Catalog to see which masks and row " - f"filters it has: {type(exc).__name__}: {_scrub(exc)}", + f"filters it has: {type(exc).__name__}: {scrub(exc)}", ("Check DATABRICKS_HOST and the credential, then re-run fluid apply.",), ) from exc @@ -741,6 +762,136 @@ def post_apply( "first (replacing a deny-all filter) and grants again afterwards.", ), ) + if not after_failed_apply: + drop_orphan_functions(runner, targets, say) + + +def _list_named(client: Any, path: str, query: Mapping[str, str], key: str) -> List[Dict[str, Any]]: + """Every entry of a paged Unity Catalog list (``key`` holds the page's rows).""" + out: List[Dict[str, Any]] = [] + token: Optional[str] = None + for _ in range(100): + page = client.api_client.do( + "GET", path, query={**query, **({"page_token": token} if token else {})} + ) + out += [r for r in (page.get(key) or []) if isinstance(r, Mapping)] + token = page.get("next_page_token") or None + if not token: + return out + raise StatementFailed("the list never ended") + + +def _bare(name: Any) -> str: + return str(name or "").replace("`", "").lower() + + +def _orphans( + names: Sequence[str], table: str, columns: Collection[str], keep: Collection[str] +) -> List[str]: + """The functions among ``names`` (bare, lower case) that carry this table's fluid_ name + pattern and are not in ``keep``: ``fluid_rf_
_<8 hex>``, the deny-all + ``fluid_rf_
_deny_all_<8 hex>`` and ``fluid_mask_
__<8 hex>`` for a column + the table has.""" + t = re.escape(table.lower()) + shapes = [ + f"{ROW_FILTER_PREFIX}{t}_[0-9a-f]{{8}}", + f"{ROW_FILTER_PREFIX}{t}_deny_all_[0-9a-f]{{8}}", + ] + shapes += [f"{MASK_PREFIX}{t}_{re.escape(c.lower())}_[0-9a-f]{{8}}" for c in columns] + rx = re.compile("|".join(f"(?:{x})" for x in shapes)) + return [n for n in names if n.startswith("fluid_") and rx.fullmatch(n) and n not in keep] + + +def drop_orphan_functions(session: Session, targets: Sequence[Target], say: Say) -> None: + """Drop this module's functions that a changed contract left behind, after a successful apply. + + A function is dropped only when its name is this table's fluid_ pattern + (:func:`_orphans`), it is not one the contract expects now, and no table in its schema has it + as a row filter or column mask (every table of the schema is read first). If the schema + cannot be read completely, nothing is dropped in it. Each drop is printed. + """ + schemas: Dict[Tuple[str, str], List[Target]] = {} + for target in targets: + parts = target.full_name.split(".") + if len(parts) == 3: + schemas.setdefault((parts[0], parts[1]), []).append(target) + expected = set() + for target in targets: + e = target.enforcement + if e is None: + continue + expected |= {_bare(m.function.full_name) for m in e.masks} + if e.row_filter is not None: + expected.add(_bare(e.row_filter.function.full_name)) + if e.deny_all is not None: + expected.add(_bare(e.deny_all.function.full_name)) + client = session.client + for (catalog, schema), members in sorted(schemas.items()): + where = f"{catalog}.{schema}" + try: + query = {"catalog_name": catalog, "schema_name": schema} + bound: set = set() + columns: Dict[str, List[str]] = {} + # include_browse: a table the run may only browse is listed too, and reading it + # fails, so nothing is dropped rather than a binding being missed + tables_query = {**query, "include_browse": "true"} + for row in _list_named(client, "/api/2.1/unity-catalog/tables", tables_query, "tables"): + full = str(row.get("full_name") or f"{where}.{row.get('name')}") + live = read_table(client, full) + if live is None: + continue + bound.add(_bare((live.get("row_filter") or {}).get("function_name"))) + live_cols = _live_columns(live) + bound |= { + _bare((c.get("mask") or {}).get("function_name")) for c in live_cols.values() + } + columns[full.rsplit(".", 1)[-1].lower()] = [ + str(c["name"]) for c in live_cols.values() + ] + listed = _list_named(client, "/api/2.1/unity-catalog/functions", query, "functions") + except Exception as exc: # noqa: BLE001 — unknown binding: drop nothing + say( + f" note: orphan functions in {where} not dropped: its tables or functions could " + f"not be read ({type(exc).__name__}: {scrub(exc)})." + ) + continue + names = [ + str(r.get("name") or str(r.get("full_name") or "").rsplit(".", 1)[-1]).lower() + for r in listed + ] + mine = f"{where}.".lower() + keep = {n[len(mine) :] for n in expected | bound if n.startswith(mine)} + settled: List[str] = [] + for target in members: + table = target.full_name.split(".")[2] + for name in _orphans(names, table, columns.get(table.lower(), ()), keep): + if name not in settled and _drop_if_owned(session, f"{where}.{name}", say): + settled.append(name) + + +def _drop_if_owned(session: Session, full: str, say: Say) -> bool: + """Drop one orphan function when the deploy identity owns it, printing what happened. + + ``True`` when it is settled: dropped, or left (and named) because another identity owns it + or the owner cannot be read, so it is not tried again for another table. ``False`` when the + ``DROP`` itself failed. + """ + try: + owner = str((read_function(session.client, full) or {}).get("owner") or "") + me = session.identity + except Exception: # noqa: BLE001 — unknown owner: leave it + owner, me = "", "" + if not me or not owner_matches(owner, {me}): + why = f"owned by {owner}, not by the deploy identity" if owner else "owner unknown" + say(f" left function {full}: {why}") + return True + try: + session.runner.run(f"DROP FUNCTION IF EXISTS {quote_table(full)}") + except StatementFailed as exc: + say(f" note: superseded function {full} was not dropped: {exc}") + return False + say(f" dropped superseded function {full}") + return True @dataclass @@ -761,6 +912,13 @@ class FailSafe: checked: bool = False #: Whether the table's grants could be listed (else only the restricted identities were revoked). listed: bool = True + #: Whether any effective-permission entry said where a privilege comes from + #: (``inherited_from_*``); if none did, "nothing inherited is open" cannot be told from + #: "the listing does not say". + inheritance_visible: bool = False + #: The table's owner could not be read, so the lock could not skip it (only this run's own + #: identity was skipped). + owner_unknown: bool = False _READ = frozenset({"SELECT", "ALL_PRIVILEGES"}) @@ -913,8 +1071,16 @@ def _lock( result: FailSafe, say: Say, ) -> None: - """Revoke SELECT (and ALL PRIVILEGES) on the table from every grantee but its owner.""" + """Revoke SELECT (and ALL PRIVILEGES) on the table from every grantee but its owner and the + identity this run is: revoking from the deploy identity would lock the run out of the table + it has to repair.""" owner = str((live or {}).get("owner") or "") + result.owner_unknown = not owner + try: + me = session.identity + except Exception: # noqa: BLE001 — unreadable: nothing to skip but the owner + me = "" + skip = {p.lower() for p in (owner, me) if p} grantees: Dict[str, set] = {} try: for entry in _list_pages( @@ -931,7 +1097,7 @@ def _lock( ) grantees = {p: {"SELECT"} for p in enforcement.restricted} for principal, held in sorted(grantees.items()): - if owner and principal.lower() == owner.lower(): + if principal.lower() in skip: continue statements = [] if "SELECT" in held: @@ -947,6 +1113,16 @@ def _lock( else: result.revoked.append(statement) say(f" revoked: {statement}") + restricted = {p.lower() for p in enforcement.restricted} + for held_by in dict.fromkeys(h for h in (owner, me) if h): # once when the run owns it + if held_by.lower() in restricted: + what = "owns" if held_by == owner else "is the identity of this run, and has" + result.still_open.append( + f"{held_by} {what} {table} and the contract restricts it; an owner reads its " + "table whatever is revoked, so this lock cannot close that path. Run: ALTER TABLE " + f"{'.'.join(quote_ident(p) for p in table.split('.'))} OWNER TO " + "``" + ) _open_paths(session, table, owner, result) @@ -961,6 +1137,12 @@ def _open_paths(session: Session, table: str, owner: str, result: FailSafe) -> N result.checked = False return result.checked = True + result.inheritance_visible = any( + str(k).startswith("inherited_from") + for entry in entries + for detail in _privileges(entry).values() + for k in detail + ) for entry in entries: principal = str(entry.get("principal") or "") if not principal or (owner and principal.lower() == owner.lower()): @@ -992,10 +1174,15 @@ def _describe(target: Target, fs: FailSafe) -> str: done = f"The deny-all row filter could not be bound ({fs.deny_all_problem}). " + ( "SELECT (and ALL PRIVILEGES where held) was revoked on the table from " + ", ".join(sorted({s.rsplit(" FROM ", 1)[1] for s in fs.revoked})) - + " (every grantee but its owner). " + + " (every grantee but its owner and this run's own identity). " if fs.revoked else "Nothing was revoked. " ) + if fs.owner_unknown: + done += ( + "The table's owner could not be read, so it could not be skipped: if the owner held " + "SELECT through a grant, it was revoked too. " + ) if not fs.listed: done += "Its grants could not be listed, so only the restricted identities were revoked. " if fs.unrevoked: @@ -1004,8 +1191,13 @@ def _describe(target: Target, fs: FailSafe) -> str: ) if fs.still_open: done += "Still open: " + " | ".join(fs.still_open) + ". " - elif fs.checked: + elif fs.checked and fs.inheritance_visible: done += "No grant inherited from the schema or the catalog was found open. " + elif fs.checked: + done += ( + "The permission listing did not say where any privilege comes from, so it could not " + "determine whether a schema or catalog grant is open: check them. " + ) else: done += "Grants inherited from the schema or the catalog could not be checked: check them. " return done.rstrip() diff --git a/fluid_build/iac/providers/databricks_jobs.py b/fluid_build/iac/providers/databricks_jobs.py index d78abac85..3efa289a0 100644 --- a/fluid_build/iac/providers/databricks_jobs.py +++ b/fluid_build/iac/providers/databricks_jobs.py @@ -123,9 +123,9 @@ def _under_home(prefix: str, suffix: str) -> TofuExpr: "dev.azure.com": "azureDevOpsServices", } _GIT_REFS = ("branch", "tag", "commit") -_GIT_REF_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]*$") -_COMMIT_RE = re.compile(r"^[0-9a-fA-F]{7,40}$") -_REPO_PATH_RE = re.compile(r"^[A-Za-z0-9._/-]+$") +_GIT_REF_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]*\Z") +_COMMIT_RE = re.compile(r"^[0-9a-fA-F]{7,40}\Z") +_REPO_PATH_RE = re.compile(r"^[A-Za-z0-9._/-]+\Z") #: ``1h``, ``30m``, ``45s``, ``2d``, a bare number of seconds, or ISO 8601 ``PT1H30M``. _DURATION_RE = re.compile(r"^(\d+)\s*([smhd]?)$") diff --git a/fluid_build/iac/providers/databricks_retention.py b/fluid_build/iac/providers/databricks_retention.py new file mode 100644 index 000000000..86205c726 --- /dev/null +++ b/fluid_build/iac/providers/databricks_retention.py @@ -0,0 +1,887 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``lifecycle.expire`` on a Databricks table: limit the Delta history. + +Nothing on Databricks deletes rows by age. What can make a retention promise +true is the table's shape: a table rebuilt **in full** on every run holds no row +copy older than the last run, except in its Delta history (the old data files +Delta keeps for time travel). ``VACUUM`` removes those files once they are older +than the table's retention, and predictive optimization runs ``VACUUM`` on Unity +Catalog managed tables. So ``expire: true`` is accepted when + +1. every build that writes the expose rewrites the whole table: + + * a dbt build whose model for the table is materialized as ``table`` + (``CREATE OR REPLACE`` each run). ``incremental`` and ``snapshot`` keep old + rows, a ``view`` stores none, and anything dbt cannot be shown to have + resolved to ``table`` is refused, not guessed; + * forge-cli's own DuckDB acquisition. The landing it does into Databricks is + an ``INSERT OVERWRITE`` of the whole table, and this module relies on that: + an append there would keep every run's rows and make this acceptance false; + +2. the contract's ``lifecycle.retention`` is at least the table's history + retention, the larger of ``delta.logRetentionDuration`` (default 30 days) and + ``delta.deletedFileRetentionDuration`` (default 7 days) + (https://docs.databricks.com/aws/en/delta/history, + https://docs.delta.io/latest/delta-batch.html#data-retention). A retention + under the 30-day default is refused for now: this module does not set the + table's properties, since dbt owns a dbt-written table's definition. + +``fluid verify`` is the gate on the live table (``cli/_verify_databricks.py``). + +dbt's configuration is resolved statically, as dbt resolves it +(https://docs.getdbt.com/reference/model-configs, +https://docs.getdbt.com/reference/dbt-jinja-functions/config): the ``config()`` +call in the model's SQL beats the model's ``config:`` in a properties YAML, which +beats ``dbt_project.yml`` (``models:``, the most specific path winning); the +default materialization is ``view``. The model is the one whose relation name +(its ``alias``, else its file name) is the expose's table. A value that is not a +plain literal (``var()``, a conditional ``config()``) is *undeterminable* and +refused, with the fix. +""" + +from __future__ import annotations + +import ast +import fnmatch +import os +import re +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath +from typing import Any, Dict, Iterator, List, Mapping, Optional, Sequence, Tuple + +from ...util.binding_paths import ENV_PLACEHOLDER_RE +from ...util.contract import builds_writing_expose, is_duckdb_acquisition_build +from ..base import UnsupportedBindingError +from ..source import current_contract_source + +DAY = 86400 + +#: Delta's table properties for how long history is kept, and their defaults +#: (Databricks "Work with Delta Lake table history"). +LOG_RETENTION = "delta.logRetentionDuration" +DELETED_FILE_RETENTION = "delta.deletedFileRetentionDuration" +DEFAULT_LOG_SECONDS = 30 * DAY +DEFAULT_DELETED_FILE_SECONDS = 7 * DAY + +#: The most a project read here may hold: the same bounds the job upload keeps. +MAX_FILE_BYTES = 2 * 1024 * 1024 +MAX_FILES = 1000 + + +def _refusal(kind: str, message: str, *remediation: str) -> UnsupportedBindingError: + return UnsupportedBindingError(f"databricks-retention-{kind}", message, tuple(remediation)) + + +# ── Durations ─────────────────────────────────────────────────────────────── + +#: The schema's ``isoDuration`` pattern (``fluid-schema-0.7.7.json``), as groups. The ``T`` +#: needs a time part after it (``PT``, ``P1DT`` are not durations), digits are bounded, +#: and a fraction, a sign or a lower-case letter is not accepted. +_ISO_RE = re.compile( + r"^P(?!$)(?:(\d{1,12})Y)?(?:(\d{1,12})M)?(?:(\d{1,12})W)?(?:(\d{1,12})D)?" + r"(?:T(?=\d)(?:(\d{1,12})H)?(?:(\d{1,12})M)?(?:(\d{1,12})S)?)?$" +) + +#: A year and a month are not fixed lengths. The *shortest* is taken, so a contract's +#: retention is never read as longer than it is, and the check that it covers the +#: history never passes one that does not: P1Y = 365 days (not 366), P1M = 28 days (not 30). +_YEAR_DAYS = 365 +_MONTH_DAYS = 28 + + +def iso_duration_seconds(text: Any) -> int: + """An ISO 8601 duration in seconds, years and months at their shortest. + + Raises ``ValueError`` for anything the schema's ``isoDuration`` does not allow. + """ + if not isinstance(text, str) or not (match := _ISO_RE.match(text.strip())): + raise ValueError(f"{text!r} is not an ISO 8601 duration") + years, months, weeks, days, hours, minutes, seconds = (int(g or 0) for g in match.groups()) + return ( + (years * _YEAR_DAYS + months * _MONTH_DAYS + weeks * 7 + days) * DAY + + hours * 3600 + + minutes * 60 + + seconds + ) + + +_INTERVAL_PART = re.compile( + r"(\d{1,12})\s*(week|day|hour|minute|second|millisecond|microsecond)s?\b", re.IGNORECASE +) +_UNIT_SECONDS = { + "week": 7 * DAY, + "day": DAY, + "hour": 3600, + "minute": 60, + "second": 1, + "millisecond": 0.001, + "microsecond": 0.000001, +} + + +def interval_seconds(text: Any) -> int: + """A Delta interval property (``interval 30 days``, ``interval 1 week 12 hours``) in + seconds, rounded up. Delta takes no month or year here; ``ValueError`` for anything + else, so an unreadable property is never taken as a short one.""" + if not isinstance(text, str): + raise ValueError(f"{text!r} is not an interval") + body = re.sub(r"^\s*interval\b", "", text.strip(), flags=re.IGNORECASE).strip() + parts = _INTERVAL_PART.findall(body) + if not parts or _INTERVAL_PART.sub("", body).strip(): + raise ValueError(f"{text!r} is not a Delta interval such as 'interval 30 days'") + total = sum(int(n) * _UNIT_SECONDS[unit.lower()] for n, unit in parts) + return int(-(-total // 1)) + + +def history_seconds(properties: Mapping[str, Any]) -> int: + """The Delta history retention of a table with these properties: the larger of the log + retention and the deleted-file retention, each at its default when unset. + + Property names are matched case-insensitively, as Delta does. ``ValueError`` when one + is set to something that is not an interval. + """ + lowered = {str(k).lower(): v for k, v in properties.items()} + log = lowered.get(LOG_RETENTION.lower()) + deleted = lowered.get(DELETED_FILE_RETENTION.lower()) + return max( + DEFAULT_LOG_SECONDS if log is None else interval_seconds(log), + DEFAULT_DELETED_FILE_SECONDS if deleted is None else interval_seconds(deleted), + ) + + +def days_text(seconds: int) -> str: + """``30 days``, ``36 hours``: how a message names a span.""" + if seconds % DAY == 0: + days = seconds // DAY + return f"{days} day{'s' if days != 1 else ''}" + if seconds % 3600 == 0: + return f"{seconds // 3600} hours" + return f"{seconds} seconds" + + +# ── dbt: which materialization does the model get? ────────────────────────── + + +class DbtUnresolved(Exception): + """The model's materialization cannot be worked out; the message says why.""" + + +class _Unknown: + """A config value that is not a plain literal (``var()``, a name, a ``**``).""" + + def __repr__(self) -> str: # pragma: no cover - debugging only + return "" + + +UNKNOWN = _Unknown() + + +@dataclass(frozen=True) +class DbtModel: + """What dbt would do for the model that writes a table.""" + + name: str + materialized: str + #: The Delta ``tblproperties`` it sets, as written (strings). + tblproperties: Mapping[str, Any] = field(default_factory=dict) + #: dbt's fully qualified name (project, folders below the model path, name), and the + #: file's path below the project, which dbt's selectors match. + fqn: Tuple[str, ...] = () + path: str = "" + + +def _jinja(value: Any) -> bool: + return isinstance(value, str) and ("{{" in value or "{%" in value) + + +def _yaml(path: Path) -> Any: + import yaml + + try: + if path.stat().st_size > MAX_FILE_BYTES: + raise DbtUnresolved(f"{path.name} is larger than {MAX_FILE_BYTES} bytes") + return yaml.safe_load(path.read_text(encoding="utf-8")) + except DbtUnresolved: + raise + except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: + raise DbtUnresolved(f"{path.name} could not be read ({type(exc).__name__})") from None + + +def _literal(node: ast.AST) -> Any: + """A Python or Jinja literal (``true`` and ``none`` too); ``UNKNOWN`` otherwise.""" + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Name) and node.id.lower() in ("true", "false", "none"): + return {"true": True, "false": False, "none": None}[node.id.lower()] + if isinstance(node, (ast.List, ast.Tuple)): + items = [_literal(item) for item in node.elts] + return UNKNOWN if any(item is UNKNOWN for item in items) else items + if isinstance(node, ast.Dict) and all(key is not None for key in node.keys): + out = {} + for key, value in zip(node.keys, node.values, strict=True): + k, v = _literal(key), _literal(value) # type: ignore[arg-type] + if k is UNKNOWN or not isinstance(k, str): + return UNKNOWN + out[k] = v + return out + return UNKNOWN + + +_RAW_OPEN_RE = re.compile(r"\{%-?\s*raw\s*-?%\}") +_RAW_CLOSE_RE = re.compile(r"\{%-?\s*endraw\s*-?%\}") + + +# The scans below match what the lazy regexes ``\{#.*?#\}``, ``{% raw %}.*?{% endraw %}`` and +# ``\{\{(.*?)\}\}|\{%(.*?)%\}`` would, in linear time: an opener with no closer after it means no +# later opener of that kind has one either, so the scan stops looking for that kind instead of +# re-reading the rest of the file from every opener (quadratic on a file of unclosed ``{{``). +def _without_comments_and_raw(text: str) -> str: + out: List[str] = [] + pos = 0 + while (start := text.find("{#", pos)) >= 0 and (end := text.find("#}", start + 2)) >= 0: + out.append(text[pos:start]) + pos = end + 2 + text = "".join(out) + text[pos:] + out, pos = [], 0 + while (opened := _RAW_OPEN_RE.search(text, pos)) and ( + closed := _RAW_CLOSE_RE.search(text, opened.end()) + ): + out.append(text[pos : opened.start()]) + pos = closed.end() + return "".join(out) + text[pos:] + + +def _tags(text: str) -> Iterator[Tuple[Optional[str], Optional[str]]]: + """``(expression, None)`` for each ``{{ ... }}`` and ``(None, statement)`` for each + ``{% ... %}``, in order.""" + closer = {"{{": "}}", "{%": "%}"} + nxt: Dict[str, int] = {} + pos = 0 + while True: + for opener in list(closer): + if nxt.get(opener, -1) < pos: + nxt[opener] = text.find(opener, pos) + if nxt[opener] < 0: + del closer[opener] + if not closer: + return + start, opener = min((nxt[o], o) for o in closer) + end = text.find(closer[opener], start + 2) + if end < 0: + del closer[opener] # no closer after this opener, so none after a later one + continue + inner = text[start + 2 : end] + yield (inner, None) if opener == "{{" else (None, inner) + pos = end + 2 + + +_BLOCKS = frozenset({"if", "for", "macro", "call", "filter", "with", "block"}) +_ENDS = frozenset({"endif", "endfor", "endmacro", "endcall", "endfilter", "endwith", "endblock"}) + + +def sql_config(text: str) -> Dict[str, Any]: + """The configs a model's SQL sets with ``{{ config(...) }}``, later calls over earlier. + + Only a call outside every ``{% if %}`` / ``{% for %}`` / ``{% macro %}`` block, whose + keyword values are literals, is read. Raises :class:`DbtUnresolved` for a ``config()`` + inside a block, or one with a ``**`` expansion or positional argument. A keyword whose + value is not a literal is :data:`UNKNOWN`. + """ + text = _without_comments_and_raw(text) + found: Dict[str, Any] = {} + depth = 0 + for expression, statement in _tags(text): + if statement is not None: + words = statement.strip(" \t\r\n-+").split(None, 1) + word = words[0] if words else "" + if word in _BLOCKS or (word == "set" and "=" not in statement): + depth += 1 + elif word in _ENDS or word == "endset": + depth = max(0, depth - 1) + continue + inner = expression.strip(" \t\r\n-+") + if not re.match(r"config\s*\(", inner): + continue + if depth: + raise DbtUnresolved("its config() is inside a Jinja block, so it may not apply") + try: + call = ast.parse(inner, mode="eval").body + except SyntaxError: + raise DbtUnresolved("its config() call could not be parsed") from None + if ( + not isinstance(call, ast.Call) + or not isinstance(call.func, ast.Name) + or call.func.id != "config" + or call.args + or any(keyword.arg is None for keyword in call.keywords) + ): + raise DbtUnresolved("its config() call is not plain keyword arguments") + for keyword in call.keywords: + found[str(keyword.arg)] = _literal(keyword.value) + return found + + +def _tree_configs(node: Any) -> Dict[str, Any]: + """The configs set directly on one level of ``dbt_project.yml``'s ``models:`` tree: + a ``+key``, or a key whose value is not a mapping (a folder holds a mapping).""" + out: Dict[str, Any] = {} + if not isinstance(node, Mapping): + return out + for key, value in node.items(): + key = str(key) + if key.startswith("+"): + out[key[1:].replace("-", "_")] = UNKNOWN if _jinja(value) else value + elif not isinstance(value, Mapping): + out[key.replace("-", "_")] = UNKNOWN if _jinja(value) else value + return out + + +def _project_configs(models: Any, project: str, fqn: Sequence[str]) -> Dict[str, Any]: + """``dbt_project.yml``'s configs for the node at ``fqn`` (folders, then the model), + the most specific last: root ``+`` configs, the project's, then each folder's.""" + merged = dict(_tree_configs(models)) + node = models.get(project) if isinstance(models, Mapping) else None + for level in [None, *fqn]: + if level is not None: + node = node.get(level) if isinstance(node, Mapping) else None + merged.update(_tree_configs(node)) + if not isinstance(node, Mapping): + break + return merged + + +def _model_paths(config: Mapping[str, Any]) -> List[PurePosixPath]: + raw = config.get("model-paths", config.get("source-paths", ["models"])) + out = [] + for entry in [raw] if isinstance(raw, str) else raw if isinstance(raw, list) else []: + rel = PurePosixPath(str(entry)) + if rel.is_absolute() or ".." in rel.parts or not rel.parts: + raise DbtUnresolved(f"model-paths names {str(entry)!r}, outside the project") + out.append(rel) + return out + + +def _walk_models( + project: Path, roots: Sequence[PurePosixPath] +) -> Iterator[Tuple[Path, Tuple[str, ...]]]: + """``(file, folders below its model path)`` for every file under the model paths, + never through a link.""" + count = 0 + for rel in roots: + base = project / rel + if base.is_symlink() or not base.is_dir(): + continue + if not base.resolve().is_relative_to(project.resolve()): + continue # a folder on the way is a link out of the project + for here, dirs, names in os.walk(base, followlinks=False): + dirs[:] = sorted( + d for d in dirs if not d.startswith(".") and not (Path(here) / d).is_symlink() + ) + for name in sorted(names): + path = Path(here) / name + if name.startswith(".") or path.is_symlink() or not path.is_file(): + continue + count += 1 + if count > MAX_FILES: + raise DbtUnresolved(f"the model paths hold more than {MAX_FILES} files") + yield path, Path(here).relative_to(base).parts + + +def resolve_model(project: Path, relation: str) -> DbtModel: + """The dbt model of ``project`` that creates the relation ``relation``, with the + materialization and Delta properties dbt resolves for it. + + Raises :class:`DbtUnresolved` when there is no such model, more than one, or a value + that decides the outcome is not a plain literal. + """ + config = _yaml(project / "dbt_project.yml") + if not isinstance(config, Mapping) or not isinstance(config.get("name"), str): + raise DbtUnresolved("dbt_project.yml has no project name") + name = config["name"] + models = config.get("models") or {} + sql: Dict[str, Tuple[Path, Tuple[str, ...]]] = {} + properties: Dict[str, Mapping[str, Any]] = {} + for path, folders in _walk_models(project, _model_paths(config)): + if path.suffix == ".sql": + if path.stem in sql: + raise DbtUnresolved(f"two models are named {path.stem!r}") + sql[path.stem] = (path, folders) + elif path.suffix == ".py": + sql.setdefault(path.stem, (path, folders)) + elif path.suffix in (".yml", ".yaml"): + document = _yaml(path) + for entry in (document.get("models") if isinstance(document, Mapping) else None) or []: + if isinstance(entry, Mapping) and isinstance(entry.get("name"), str): + if entry.get("versions"): + raise DbtUnresolved(f"model {entry['name']!r} is versioned") + block = entry.get("config") + properties[entry["name"]] = block if isinstance(block, Mapping) else {} + + wanted = relation.lower() + candidates: Dict[str, Dict[str, Any]] = {} + for model, (path, folders) in sql.items(): + merged = _project_configs(models, name, (*folders, model)) + merged.update( + { + str(k).replace("-", "_"): (UNKNOWN if _jinja(v) else v) + for k, v in properties.get(model, {}).items() + } + ) + text = "" + if path.suffix == ".sql": + try: + if path.stat().st_size > MAX_FILE_BYTES: + raise DbtUnresolved(f"{path.name} is larger than {MAX_FILE_BYTES} bytes") + text = path.read_text(encoding="utf-8") + merged.update(sql_config(text)) + except (OSError, UnicodeDecodeError): + raise DbtUnresolved(f"{path.name} could not be read") from None + except DbtUnresolved as exc: + if "alias" in text: + raise DbtUnresolved( + f"model {model!r}: {exc}, so it cannot be ruled out as the writer of " + f"{relation!r}" + ) from None + merged["_sql_error"] = str(exc) + alias = merged.get("alias") + if alias is UNKNOWN: + raise DbtUnresolved( + f"the alias of model {model!r} is not a plain value, so it cannot be ruled " + f"out as the writer of {relation!r}" + ) + if str(alias if alias else model).lower() == wanted: + candidates[model] = merged + if not candidates: + raise DbtUnresolved(f"no model in the project creates a relation named {relation!r}") + if len(candidates) > 1: + raise DbtUnresolved(f"models {', '.join(sorted(candidates))} all create {relation!r}") + model, merged = next(iter(candidates.items())) + if sql[model][0].suffix != ".sql": + raise DbtUnresolved(f"model {model!r} is a Python model") + if merged.get("_sql_error"): + raise DbtUnresolved(f"model {model!r}: {merged['_sql_error']}") + if merged.get("enabled") is UNKNOWN or merged.get("enabled") in (False, "false", "False"): + raise DbtUnresolved(f"model {model!r} is disabled or its enabled is not a plain value") + for key in ("schema", "database", "catalog"): + if merged.get(key) not in (None, ""): + raise DbtUnresolved( + f"model {model!r} sets {key}, so its relation may not be in the expose's " + f"{'schema' if key == 'schema' else 'catalog'}" + ) + materialized = merged.get("materialized", "view") + if not isinstance(materialized, str): + raise DbtUnresolved(f"model {model!r}: materialized is not a plain value") + props = merged.get("tblproperties") or {} + if not isinstance(props, Mapping): + raise DbtUnresolved(f"model {model!r}: tblproperties is not a plain mapping") + path, folders = sql[model] + return DbtModel( + model, + materialized, + {str(k): v for k, v in props.items()}, + fqn=(name, *folders, model), + path=path.relative_to(project).as_posix(), + ) + + +# ── dbt: does the build select the model? ─────────────────────────────────── + +#: Selection methods this module does not evaluate (a tag, a config, the state of a +#: previous run): a selector using one cannot be said to include the model. +_METHOD_RE = re.compile(r"^([A-Za-z_.]+):") +_GRAPH_RE = re.compile(r"^(@)?(\d*\+)?(.*?)(\+\d*)?$") + + +def _fqn_selects(fqn: List[str], selector: str) -> bool: + """dbt's fqn method for one node, as dbt-core 1.11 has it (``dbt/graph/selector_methods.py`` + ``is_selected_node``, called with the fqn and with the fqn less its project by + ``QualifiedNameSelectorMethod.node_is_match``; Apache-2.0): the model's own name exactly, + else the selector's dot-separated parts as an exact prefix of the fqn up to the first part + with a wildcard, and from there one ``fnmatch`` of the rest. Case-sensitive, the stricter + reading. A versioned model is refused before this is asked.""" + if fqn[-1] == selector: + return True + flat = [item for segment in fqn for item in segment.split(".")] + parts = selector.split(".") + if len(flat) < len(parts): + return False + for i, part in enumerate(parts): + if any(w in part for w in ("*", "?", "[", "]")): + return fnmatch.fnmatchcase(".".join(flat[i:]), ".".join(parts[i:])) + if flat[i] != part: + return False + return True + + +def _part_selects(part: str, model: DbtModel) -> Optional[bool]: + """Whether one selector (no space, no comma) includes ``model``: ``True``, ``False``, or + ``None`` when it cannot be told without dbt's graph or run state (a graph operator on + another node, a tag or config method). + + Only what is decidable from the project's files is: a node's own name or fully qualified + name (prefixes and ``*`` included), and ``path:``. A graph operator (``+``, ``N+``, ``@``) + includes the named node, and nodes around it that only dbt's graph knows. + """ + if part == "*": + return True + method = _METHOD_RE.match(part) + value = part + if method: + name = method.group(1).lower() + value = part[method.end() :] + if name not in ("fqn", "path", "model"): + return None + else: + name = "path" if "/" in part or part.endswith(".sql") else "fqn" + graph = _GRAPH_RE.match(value) + operators = bool(graph and (graph.group(1) or graph.group(2) or graph.group(4))) + value = graph.group(3) if graph else value + if name == "path": + wanted = PurePosixPath(value.rstrip("/")) + have = PurePosixPath(model.path) + hit = len(wanted.parts) <= len(have.parts) and all( + fnmatch.fnmatchcase(h, w) for h, w in zip(have.parts, wanted.parts, strict=False) + ) + else: + hit = any(_fqn_selects(list(fqn), value) for fqn in (model.fqn, model.fqn[1:]) if fqn) + if hit: + return True + return None if operators else False + + +def selects_model(selectors: Sequence[str], model: DbtModel) -> Optional[bool]: + """Whether ``dbt build --select `` builds ``model``: ``True``, ``False``, or + ``None`` when it cannot be told. No selector builds every model. Space-separated + selectors are a union, a comma joins an intersection (dbt's selection syntax, + https://docs.getdbt.com/reference/node-selection/syntax).""" + tokens = [t for item in selectors for t in str(item).split()] + if not tokens: + return True + unknown = False + for token in tokens: + results = [_part_selects(part, model) for part in token.split(",") if part] + if not results or any(r is False for r in results): + continue + if all(r is True for r in results): + return True + unknown = True + return None if unknown else False + + +# ── the decision ──────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class Retention: + """An accepted ``lifecycle.expire``: what ``fluid verify`` holds the live table to.""" + + #: ``lifecycle.retention`` as written, and in seconds (years and months at their shortest). + period: str + seconds: int + + +def _is_dbt(build: Mapping[str, Any]) -> bool: + engine = str(build.get("engine") or "").strip().lower() + return engine == "dbt" or engine.startswith("dbt-") + + +def _fix_materialized() -> str: + return "Declare materialized='table' on the model ({{ config(materialized='table') }})." + + +def _templated(value: Any) -> bool: + """Whether ``value`` still holds an ``{{ env.X }}`` placeholder: ``fluid validate`` reads + the contract before the environment is substituted, the emitter after.""" + return isinstance(value, str) and ENV_PLACEHOLDER_RE.search(value) is not None + + +def _dbt_model( + build: Mapping[str, Any], table: str, where: str, validating: bool +) -> Tuple[Optional[DbtModel], Optional[UnsupportedBindingError]]: + """The dbt model of ``build`` for ``table``; ``(None, None)`` when ``validating`` with + no contract file to find the project from, as the job emitter does, or with a table + name that is still an environment placeholder (the emitter resolves it).""" + if validating and _templated(table): + return None, None + label = f"{where}: build {build.get('id')!r}" + repository = str(build.get("repository") or "").strip() + if "://" in repository or repository.startswith("git@"): + return None, _refusal( + "dbt-unresolved", + f"{label} runs a dbt project from a Git repository ({repository.split('?')[0]}), " + "which forge-cli cannot read here, so it cannot tell whether the model for " + f"{table!r} is rebuilt in full.", + "Point repository at a local dbt project directory (forge-cli reads it, and " + "uploads it to the workspace), and " + _fix_materialized(), + ) + source = current_contract_source() + if source is None: + if validating: + return None, None + return None, _refusal( + "dbt-unresolved", + f"{label}: the contract's location is not known here, so its dbt project cannot " + "be read to see how the model is materialized.", + "Run it through fluid apply / fluid generate iac with the contract's path.", + ) + from ...build_runners.dbt.runner import resolve_dbt_project_path + + project = resolve_dbt_project_path(Path(source), dict(build)) + if project is None: + return None, _refusal( + "dbt-unresolved", + f"{label}: repository {repository!r} holds no dbt_project.yml, so the model's " + "materialization cannot be read.", + "Point repository at the dbt project directory, relative to the contract.", + ) + try: + return resolve_model(project, table), None + except DbtUnresolved as exc: + return None, _refusal( + "dbt-unresolved", + f"{label}: cannot tell whether the model for {table!r} is rebuilt in full: {exc}.", + _fix_materialized(), + ) + + +def _selection_refusal( + build: Mapping[str, Any], model: DbtModel, table: str, label: str +) -> Optional[UnsupportedBindingError]: + """A refusal unless the build's dbt selection includes the resolved model: a model the + build does not run is not rebuilt, and its table keeps whatever it last held.""" + from ...build_runners.dbt.runner import build_selectors + + selectors = build_selectors(dict(build)) + chosen = selects_model(selectors, model) + if chosen: + return None + shown = " ".join(selectors) + how = ( + "does not select it" + if chosen is False + else "selects it only through graph operators, tags, configs or state that forge-cli " + "cannot evaluate here" + ) + return _refusal( + "not-selected", + f"{label}: the table {table!r} is written by model {model.name!r}, but the build's " + f"selection ({shown!r}, from properties.model or properties.select) {how}, so a run " + "need not rebuild the table and rows older than the retention could stay.", + f"Select the model by name: properties.select: [{model.name!r}] (or " + f"properties.model: {model.name}).", + ) + + +def _acquisition_mode_refusal( + build: Mapping[str, Any], label: str, validating: bool = False +) -> Optional[UnsupportedBindingError]: + """A refusal unless the DuckDB acquisition's ``properties.source.mode`` is one the + Databricks landing runs as a full overwrite of the table (an environment placeholder + is left to the emitter, which reads it substituted).""" + from ...build_runners._databricks_load import LOAD_MODES + + properties = build.get("properties") + source = properties.get("source") if isinstance(properties, Mapping) else None + mode = source.get("mode") if isinstance(source, Mapping) else None + mode = getattr(mode, "value", mode) + if mode in LOAD_MODES or (validating and _templated(mode)): + return None + fix = ( + f"Set properties.source.mode to {LOAD_MODES[0]!r} (the landing overwrites the whole " + "table), or remove lifecycle.expire." + ) + if not mode: + return _refusal( + "unmapped", + f"{label} sets no properties.source.mode, so nothing says it rewrites the whole " + "table, and rows older than the retention could stay.", + fix, + ) + return _refusal( + "incremental", + f"{label}: the acquisition's source mode is {mode!r}, and only " + f"{', '.join(repr(m) for m in LOAD_MODES)} rewrites the whole table on Databricks, so " + "rows older than the retention could stay.", + fix, + ) + + +def check_expire( + where: str, + exposure: Mapping[str, Any], + table: str, + contract: Mapping[str, Any], + *, + validating: bool = False, +) -> Tuple[Optional[Retention], List[UnsupportedBindingError]]: + """The retention ``exposure`` promises and every reason this module refuses it. + + ``(None, [])`` when it sets no ``lifecycle.expire``. ``table`` is the expose's table + name, the relation dbt must create. + """ + lifecycle = exposure.get("lifecycle") + if not isinstance(lifecycle, Mapping) or not lifecycle.get("expire"): + return None, [] + refusals: List[UnsupportedBindingError] = [] + period = lifecycle.get("retention") + seconds: Optional[int] = None + if validating and _templated(period): + pass # the emitter checks it once the environment is substituted + elif not period: + refusals.append( + _refusal( + "period", + f"{where} sets lifecycle.expire but no lifecycle.retention, so there is no " + "period to keep the data for.", + "Set lifecycle.retention (an ISO 8601 duration such as P365D), or drop expire.", + ) + ) + else: + try: + seconds = iso_duration_seconds(period) + except ValueError: + refusals.append( + _refusal( + "period", + f"{where} lifecycle.retention is {period!r}, which is not an ISO 8601 " + "duration.", + "Write it as an ISO 8601 duration, such as P30D, P12W or P6Y.", + ) + ) + tblproperties: Mapping[str, Any] = {} + builds = [b for b in contract.get("builds") or [] if isinstance(b, Mapping)] + writers = builds_writing_expose(exposure.get("exposeId"), builds) + if not writers: + refusals.append( + _refusal( + "unmapped", + f"{where} sets lifecycle.expire, and no build writes it, so nothing says the " + "table is rebuilt in full: rows it holds could be any age, and nothing on " + "Databricks would expire them.", + "Name the build that rewrites the whole table in builds[].outputs, or remove " + "lifecycle.expire.", + ) + ) + for build in writers: + label = f"{where}: build {build.get('id')!r}" + if is_duckdb_acquisition_build(build): + # forge-cli's acquisition lands into Databricks as an INSERT OVERWRITE of the + # whole table, and only in the modes the landing accepts (it refuses any other + # before the build runs); acceptance rests on both. If the landing ever appends, + # this must refuse. + refusal = _acquisition_mode_refusal(build, label, validating) + if refusal is not None: + refusals.append(refusal) + continue + if not _is_dbt(build): + refusals.append( + _refusal( + "unmapped", + f"{label} (engine {build.get('engine')!r}) writes it, and forge-cli cannot " + "tell that it rewrites the whole table, so rows older than the retention " + "could stay.", + "Use a dbt build with materialized='table', or a DuckDB acquisition, or " + "remove lifecycle.expire.", + ) + ) + continue + model, refusal = _dbt_model(build, table, where, validating) + if refusal is not None: + refusals.append(refusal) + elif model is not None and (selection := _selection_refusal(build, model, table, label)): + refusals.append(selection) + elif model is not None: + kind = model.materialized + if kind in ("incremental", "snapshot"): + refusals.append( + _refusal( + "incremental", + f"{label}: model {model.name!r} is materialized as {kind!r}, which " + "keeps rows from earlier runs, so no run leaves the table without a " + "row older than the retention. Delta history limits cannot expire " + "rows.", + "Rebuild it in full (materialized='table'), or drop lifecycle.expire " + "and delete old rows in the model.", + ) + ) + elif kind != "table": + refusals.append( + _refusal( + "not-table", + f"{label}: model {model.name!r} is materialized as {kind!r} (dbt's " + "default is 'view'), not 'table', so a run does not replace the " + "stored table.", + _fix_materialized(), + ) + ) + else: + tblproperties = {**tblproperties, **model.tblproperties} + effective = DEFAULT_LOG_SECONDS + try: + effective = history_seconds(tblproperties) + except ValueError as exc: + refusals.append( + _refusal( + "dbt-unresolved", + f"{where}: the model's Delta retention properties cannot be read ({exc}).", + f"Write {LOG_RETENTION} and {DELETED_FILE_RETENTION} as 'interval N days'.", + ) + ) + if seconds is not None and seconds < effective: + + def _shown(key: str, default: int) -> str: + value = tblproperties.get(key) + return f"{value!s} (declared)" if value else f"{days_text(default)} (Delta's default)" + + refusals.append( + _refusal( + "too-short", + f"{where} lifecycle.retention is {period} ({days_text(seconds)}), shorter than " + f"the table's Delta history of {days_text(effective)} ({LOG_RETENTION} " + f"{_shown(LOG_RETENTION, DEFAULT_LOG_SECONDS)}, {DELETED_FILE_RETENTION} " + f"{_shown(DELETED_FILE_RETENTION, DEFAULT_DELETED_FILE_SECONDS)}). VACUUM keeps " + "old data files that long, so a copy older than the retention would remain. " + "forge-cli does not shorten the history yet.", + f"Set lifecycle.retention to at least P{-(-effective // DAY)}D, or remove " + "lifecycle.expire.", + ) + ) + if refusals: + return None, refusals + if seconds is None: + # validating with a placeholder retention: nothing to promise yet; the emitter + # reads it substituted + return None, [] + return Retention(period=str(period), seconds=seconds), [] + + +__all__ = [ + "DEFAULT_DELETED_FILE_SECONDS", + "DEFAULT_LOG_SECONDS", + "DELETED_FILE_RETENTION", + "LOG_RETENTION", + "DbtModel", + "DbtUnresolved", + "Retention", + "check_expire", + "days_text", + "history_seconds", + "interval_seconds", + "iso_duration_seconds", + "selects_model", + "resolve_model", + "sql_config", +] diff --git a/fluid_build/iac/providers/gcp.py b/fluid_build/iac/providers/gcp.py index 197088c49..5334eebd7 100644 --- a/fluid_build/iac/providers/gcp.py +++ b/fluid_build/iac/providers/gcp.py @@ -42,9 +42,11 @@ import hashlib import json +import re from dataclasses import dataclass from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple +from .. import column_types from ..access import ( GROUP, AccessGrant, @@ -52,6 +54,7 @@ normalize_access_grants, role_grants, ) +from ..base import UnsupportedBindingError from ..importer import ImportBlock from ..naming import TofuExpr, safe_ident, tofu_ref from ..packaging import ( @@ -65,28 +68,44 @@ from ..versions import required_providers from . import gcp_governance as _gov -# FLUID column type → BigQuery type (best-effort; unknown types upper-cased). -_BQ_TYPES = { - "string": "STRING", - "str": "STRING", - "text": "STRING", - "integer": "INT64", - "int": "INT64", - "int64": "INT64", - "bigint": "INT64", - "float": "FLOAT64", - "float64": "FLOAT64", - "double": "FLOAT64", - "numeric": "NUMERIC", - "decimal": "NUMERIC", - "boolean": "BOOL", - "bool": "BOOL", - "timestamp": "TIMESTAMP", - "datetime": "DATETIME", - "date": "DATE", - "time": "TIME", - "bytes": "BYTES", - "json": "JSON", +# FLUID column type → BigQuery type, keyed on the lower-cased base name +# (``varchar(255)`` is ``varchar``). Every entry is a type a BigQuery table schema +# takes; ``_bq_outcome`` classifies the rest. A ``number`` or ``decimal`` is NUMERIC +# whatever precision it declares (precision and scale are not emitted yet), and +# ``_bq_outcome`` warns when the declared range is wider than a NUMERIC holds. +_BQ_TYPES = column_types.expand( + { + "STRING": "string, str, text, varchar, varchar2, nvarchar, char, nchar, character, clob, " + "uuid, uniqueidentifier, guid, enum", + "INT64": "integer, int, int2, int4, int8, int16, int32, int64, tinyint, smallint, " + "mediumint, bigint, long, longint, serial, bigserial, year", + "FLOAT64": "float, float4, float8, float32, float64, double, real", + "NUMERIC": "numeric, decimal, dec, number, money", + "BIGNUMERIC": "bignumeric", + "BOOL": "boolean, bool, bit", + "TIMESTAMP": "timestamp, timestamptz, timestamp_tz, timestamp_ltz", + "DATETIME": "datetime, datetime2, smalldatetime, timestamp_ntz, timestampntz", + "DATE": "date", + "TIME": "time", + "INTERVAL": "interval", + "BYTES": "bytes, binary, varbinary, blob, bytea, raw, hll", + "JSON": "json, jsonb, variant, super", + "GEOGRAPHY": "geography, geometry, geom, point", + } +) + +#: What a BigQuery NUMERIC holds, and the types that take a precision and scale: the +#: scale is not emitted, so a declared range beyond it overflows at load. +_BQ_NUMERIC_INTEGER_DIGITS = 29 +_BQ_NUMERIC_SCALE = 9 +_BQ_DECIMAL_FAMILY = frozenset({"decimal", "numeric", "dec", "number"}) +_BQ_DECIMAL_ARGS_RE = re.compile(r"^\((\d+)(?:,(\d+))?\)$", re.ASCII) + +#: Types whose BigQuery type holds a different thing. ``GEOMETRY`` is planar and +#: ``GEOGRAPHY`` is a sphere, so the same coordinates mean other shapes and distances. +_BQ_LOSSY = { + "geometry": "which is planar, and BigQuery has only the spherical GEOGRAPHY", + "geom": "which is planar, and BigQuery has only the spherical GEOGRAPHY", } # FLUID permission → BigQuery dataset access role. BigQuery dataset @@ -136,24 +155,94 @@ } -def _bq_type(raw: Any) -> str: - base = " ".join(str(raw or "STRING").strip().lower().split("(", 1)[0].split()) +def _bq_outcome(raw: Any) -> column_types.TypeOutcome: + """What ``raw`` becomes in a BigQuery table schema: exact, lossy or refused.""" + base, rest = column_types.split_type(raw) + if base in column_types.STRUCTURED: + return column_types.refused( + "which BigQuery holds only as a REPEATED column of an element type or a RECORD " + "with fields, and the contract schema cannot declare them yet", + column_types.NESTED_FIX, + ) + if not rest.isascii(): + return column_types.refused( + f"whose parameters ({rest}) are not ASCII", "Declare the type with ASCII digits." + ) + if base == "bit" and rest not in ("", "(1)"): + return column_types.refused( + "which is a bit string, and BigQuery has none", + "Declare bytes or an integer type.", + ) hit = _BQ_TYPES.get(base) or _BQ_MULTIWORD_TYPES.get(base) - if hit is not None: - return hit - # A source-native spelling (VARCHAR, SMALLINT, UUID, TIMESTAMPTZ ...) used to - # be upper-cased verbatim, which BigQuery rejects: a contract read from - # Postgres emitted ``"type": "VARCHAR"``. The ODCS physical-type table covers - # the FLUID column-type enum and agrees with every entry above, so this only - # changes the types that were invalid before. - from ...providers.odcs.mappers.types import fluid_to_physical - - physical = fluid_to_physical(base, "gcp") - if physical: - return physical + if hit is None: + return column_types.refused( + "which the BigQuery emitter does not map", + "Declare a FLUID column type such as string, integer, bigint, numeric, double, " + "boolean, date, timestamp or json.", + ) + if base in _BQ_DECIMAL_FAMILY and rest: + match = _BQ_DECIMAL_ARGS_RE.match(rest) + if match is not None: + precision, scale = int(match.group(1)), int(match.group(2) or 0) + if scale > _BQ_NUMERIC_SCALE or precision - scale > _BQ_NUMERIC_INTEGER_DIGITS: + return column_types.lossy( + hit, + f"whose range a BigQuery NUMERIC ({_BQ_NUMERIC_INTEGER_DIGITS} integer and " + f"{_BQ_NUMERIC_SCALE} fractional digits) cannot hold", + "Declare bignumeric if the values need that range.", + ) + if base in _BQ_LOSSY: + return column_types.lossy( + hit, _BQ_LOSSY[base], "Declare geography if the coordinates are on the sphere." + ) + return column_types.exact(hit) + + +def _bq_type(raw: Any) -> str: + """The BigQuery type ``raw`` is written as. Total, for ``fluid verify``, ``fluid diff`` + and the Dataform engine: a refused type reads as its upper-cased name here, and is + refused where the module is emitted (:func:`_refuse_bq_column_types`).""" + outcome = _bq_outcome(raw) + if outcome.target is not None: + return outcome.target return str(raw).upper() if raw else "STRING" +def _bq_column_type_findings( + expose: str, schema: Iterable[Any] +) -> Tuple[Optional[UnsupportedBindingError], List[str]]: + return column_types.column_type_findings(expose, schema, _bq_outcome, "BigQuery") + + +def _refuse_bq_column_types(expose: str, schema: Iterable[Any]) -> None: + """Raise for a column BigQuery cannot hold, which it would reject at apply, after + the dataset was created, or (an array) take without its element type.""" + refusal, _lossy = _bq_column_type_findings(expose, schema) + if refusal is not None: + raise refusal + + +def validate_bigquery_column_types(contract: Mapping[str, Any]) -> Tuple[List[str], List[str]]: + """``(errors, warnings)`` for the column types of the contract's BigQuery tables: + the refusal :func:`_emit_bigquery` raises, and a warning for each type it writes as + a different one. A view carries no schema. ``fluid validate`` runs this, so + ``generate iac`` finds nothing new.""" + errors: List[str] = [] + warnings: List[str] = [] + for exposure in column_types.exposes_of(contract): + binding = exposure.get("binding") if isinstance(exposure, Mapping) else None + if not isinstance(binding, Mapping) or not _gov.gcp_owned(binding): + continue + if resolve_gcp_target(binding) != BIGQUERY_TABLE: + continue + schema = column_types.schema_of(exposure) + refusal, lossy = _bq_column_type_findings(_expose_id(exposure) or "?", schema) + if refusal is not None: + errors.append(f"{refusal} {' '.join(refusal.remediation)}".strip()) + warnings.extend(lossy) + return errors, warnings + + def _bq_fields(schema: List[Mapping[str, Any]]) -> List[Dict[str, Any]]: """FLUID contract schema → BigQuery schema fields.""" return [ @@ -1167,6 +1256,10 @@ def _emit_bigquery( placement: _Placement = _LEGACY_PLACEMENT, governance: Optional[_BqGovernance] = None, ) -> None: + if not is_view: + # Refused before anything is written: the same findings ``fluid validate`` + # reports (:func:`validate_bigquery_column_types`). + _refuse_bq_column_types(_expose_id(exposure) or "?", schema) gov = governance or _NO_GOVERNANCE dataset = loc.get("dataset") or "default" table = _bq_table_name(exposure, loc) @@ -1455,7 +1548,8 @@ def validate_gcp_binding(contract: Mapping[str, Any]) -> Tuple[List[str], List[s binding (no platform) is a different contract error, and the JSON-schema check already names it. Iceberg exposes are left to ``iac.iceberg_validation``, which owns a more specific message for the - same input. Returns ``(errors, warnings)``. + same input. It also reports the column types of each BigQuery table + (:func:`validate_bigquery_column_types`). Returns ``(errors, warnings)``. """ errors: List[str] = [] warnings: List[str] = [] @@ -1496,7 +1590,10 @@ def validate_gcp_binding(contract: Mapping[str, Any]) -> Tuple[List[str], List[s errors.append(message) else: warnings.append(message) - return errors, warnings + # The column types of each BigQuery table: what the emitter refuses, or writes + # as another type. + type_errors, type_warnings = validate_bigquery_column_types(contract) + return errors + type_errors, warnings + type_warnings def _bq_table_name(exposure: Mapping[str, Any], loc: Mapping[str, Any]) -> str: diff --git a/fluid_build/iac/registry.py b/fluid_build/iac/registry.py index 8550bf7e3..33425393a 100644 --- a/fluid_build/iac/registry.py +++ b/fluid_build/iac/registry.py @@ -14,106 +14,498 @@ """IaC plugin registry. -Keyed by cloud name — a distinct axis from ``providers.register_provider`` -(the apply-provider registry). Mirrors that registry's shape so the -pattern is familiar. +Keyed by cloud name, a distinct axis from ``providers.register_provider`` +(the apply-provider registry). In-tree clouds register on import (see ``iac/__init__.py``). External packages -register a NEW cloud the same way a provider/validator plugin does — via an -entry-point under ``fluid_build.iac_providers``. Discovery routes through the -unified :mod:`fluid_build.plugin_manager` (shared allow/block policy + -per-plugin fail-isolation), and :func:`iac.cutover.default_engine` reads this -registry, so a registered plugin cloud routes to the OpenTofu apply engine -with no core edit. +supply a cloud (an OpenTofu emitter) through an entry point in the +``fluid_build.iac_providers`` group, the same naming as ``fluid_build.providers``:: -**What still requires a core edit** (this module previously claimed "zero -edits to forge-cli core", which was not true end-to-end): + [project.entry-points."fluid_build.iac_providers"] + onprem = "my_pkg.iac:OnPremIacPlugin" + +Design, borrowed rather than invented: + +* **Names without importing** are read with stdlib + ``importlib.metadata.entry_points(group=...)`` (``.name`` / ``.dist``), via + :func:`fluid_build.plugin_manager._entry_points`; nothing is ``ep.load()``-ed. +* **One plugin is loaded, by name, when a platform needs it**, as stevedore's + ``DriverManager`` does (https://docs.openstack.org/stevedore/latest/user/ + essays/pbr_and_stevedore.html, ``stevedore.driver.DriverManager``). Nothing + scans or loads this group at import time, so ``fluid --help`` and + ``fluid providers`` never run third-party code. :func:`get_iac_plugin` is the + load point. +* **Policy** is the unified plugin manager's: the allow/block lists (names match + case-insensitively, ``-`` == ``_``) and the reserved names are checked before + ``ep.load()``; the plugin's ``get_plugin_info().requires_cli`` is read from the + loaded plugin, so it is checked right after the import, with the provider + registry's own gate (``providers._check_sdk_compat``: advisory warning by + default, rejection under ``FLUID_PLUGIN_STRICT_COMPAT=1``). +* **Reserved names**, as ``providers._claims_retired_name`` does for retired + provider names: an entry point that claims a built-in platform (``aws``, + ``gcp``, ``snowflake``, ``confluent``, ``databricks``, any alias + ``iac.provider_match`` resolves, or a retired provider name) is skipped + BEFORE ``ep.load()`` with a WARNING naming the distribution and the opt-in. A + plugin can add a platform; it replaces one only when the operator grants it + with a distribution-qualified ``FLUID_PLUGINS_ALLOWLIST`` entry, + ``:`` (``plugin_manager.override_granted``, + which reads the distribution's real name, never its display string). A grant + needs the name to pass the bare allowlist too: a bare allowlist that omits it + cancels the grant, with a WARNING. A granted override replaces the emitter + plugin only, not the in-tree helpers that modules import directly. +* **Duplicate names.** Several distributions shipping one reserved name: each + ungranted claim is warned about, and the one granted claim loads. Several + shipping one non-reserved name: all are refused (fail closed) with one WARNING + naming them; the operator uses ``FLUID_PLUGINS_BLOCKLIST`` or uninstalls. + +**What still requires a core edit**: * ``binding.platform`` / ``builds[].execution.runtime.platform`` is a **closed - enum** in every shipped ``fluid-schema-*.json``. A contract naming a plugin - cloud is rejected at validation, before the provider is ever consulted. - Opening the enum is a schema-evolution decision, not a registry change. -* The global ``fluid --provider`` flag's ``choices=`` list is hardcoded in - ``cli/__init__.py``. Deriving it from this registry would force plugin - discovery — i.e. third-party ``ep.load()`` — on every ``fluid --help``, - which is the attack surface the lazy ``command`` role deliberately avoids. - -So an out-of-tree cloud is reachable through ``--provider`` only for -contracts whose declared platform the schema already admits. + enum** in every shipped ``fluid-schema-*.json``. A contract naming a platform + the schema does not list is rejected at validation. A plugin therefore serves + a platform the schema admits and no built-in emitter serves (``kubernetes``, + ``postgres``, ``kafka`` ...). Opening the enum is a schema decision. +* ``fluid --provider`` is validated against the apply-provider registry + (``cli/__init__.py::_validate_global_args``), so for now it accepts built-ins + only. A plugin platform is reached by auto-detection from the contract's + ``binding.platform``. """ from __future__ import annotations import logging -from typing import Dict, Optional, Set +import threading +from typing import Any, Dict, FrozenSet, List, Optional, Set, Tuple from .base import IacProviderPlugin +_log = logging.getLogger(__name__) + IAC_PLUGINS: Dict[str, IacProviderPlugin] = {} -#: Names registered from the ``fluid_build.iac_providers`` entry-point group, -#: i.e. clouds that live OUTSIDE this repo. ``iac.cutover.default_engine`` -#: reads it: an out-of-tree cloud cannot be listed in -#: ``OPENTOFU_DEFAULT_PROVIDERS`` without the core edit the plugin system -#: exists to avoid, so its presence here is what routes it to the OpenTofu -#: engine. In-tree clouds are NOT recorded — the frozenset stays their single -#: cutover switch. +#: Names loaded from the ``fluid_build.iac_providers`` entry-point group, i.e. +#: clouds that live OUTSIDE this repo. In-tree clouds are NOT recorded; the +#: ``cutover.OPENTOFU_DEFAULT_PROVIDERS`` frozenset stays their cutover switch. IAC_ENTRYPOINT_PLUGINS: Set[str] = set() +#: The in-tree registrations, snapshotted as they register (not while an entry +#: point is loading), so ``reset_iac_plugin_cache`` can restore a built-in a +#: granted override replaced. +_INTREE: Dict[str, IacProviderPlugin] = {} +_LOADING = False + +#: Entry-point names whose load was attempted and refused or failed, so a +#: platform that needs the plugin repeatedly does not re-import, and re-warn +#: about, a broken one. ``reset_iac_plugin_cache`` clears it. +_REFUSED: Set[str] = set() +_LOCK = threading.RLock() + + +def _norm(name: str) -> str: + """The spelling ``provider_match.canonical_cloud`` and ``providers`` normalise to.""" + return str(name).strip().lower().replace("-", "_") + def register_iac_plugin(name: str, plugin: IacProviderPlugin) -> None: """Register an IaC plugin under a cloud name (``gcp``/``aws``/...). The operator allow/block policy is enforced here, at the registration chokepoint, so it holds for the in-tree clouds registered on import as - well as for entry-point plugins (which ``discover_iac_entrypoints`` - already gated). Without this, blocking ``snowflake`` removed it from the - provider registry but left the IaC plugin registered — and - ``fluid apply`` reaches the warehouse through *this* registry, so the - block would still have been inert on the path that emits DDL. + well as for entry-point plugins. Without this, blocking ``snowflake`` + removed it from the provider registry but left the IaC plugin registered, + and ``fluid apply`` reaches the warehouse through *this* registry. """ from fluid_build.plugin_manager import is_allowed if not is_allowed(name): - logging.getLogger(__name__).debug("iac plugin %r skipped by allow/block policy", name) + _log.debug("iac plugin %r skipped by allow/block policy", name) return + # Not a boundary: a loaded plugin runs at host trust and can call this (or + # write IAC_PLUGINS) directly. Reserved names are enforced BEFORE the import. IAC_PLUGINS[name] = plugin + if not _LOADING: + _INTREE[name] = plugin -def get_iac_plugin(name: str) -> Optional[IacProviderPlugin]: - """Return the registered plugin for ``name``, or ``None`` if unknown.""" - return IAC_PLUGINS.get(name) +def reserved_iac_names() -> FrozenSet[str]: + """Platform names an entry-point plugin may not claim (normalised spellings). + Everything the in-tree resolution already gives a meaning to: the cutover + set, every ``provider_match`` alias (``s3``, ``bigquery``, ``local`` ...), + the in-tree plugins registered on import, and the retired provider names. + Static, so a built-in the operator BLOCKED stays reserved. + """ + from .cutover import OPENTOFU_DEFAULT_PROVIDERS + from .provider_match import PROVIDER_ALIASES -def discover_iac_entrypoints(logger: Optional[logging.Logger] = None) -> None: - """Register external IaC cloud plugins from the ``fluid_build.iac_providers`` group. + names: Set[str] = set(OPENTOFU_DEFAULT_PROVIDERS) | set(PROVIDER_ALIASES) + names |= {_norm(n) for n in IAC_PLUGINS if n not in IAC_ENTRYPOINT_PLUGINS} + from fluid_build.providers import RETIRED_PROVIDERS + + names |= set(RETIRED_PROVIDERS) + return frozenset(names) + + +def _claims_reserved_name(ep: Any) -> bool: + """True (after a WARNING naming the plugin) when ``ep`` claims a reserved name + without the operator's grant. + + Called BEFORE ``ep.load()``, so a package squatting a built-in platform never + gets to run its import-time code. The distribution is in the message itself, + repr'd so control characters are escaped: the CLI's formatters print the + message only, and this line is how an operator learns what to uninstall, or + how to opt in. A distribution-qualified ``FLUID_PLUGINS_ALLOWLIST`` entry + (``plugin_manager.override_granted``) lifts the reservation for that one + entry point; it is then logged at INFO and loaded. A retired name has no + built-in to replace, so no grant lifts it (as in ``providers``). + """ + key = _norm(ep.name) + if key not in reserved_iac_names(): + return False + from fluid_build.plugin_manager import ( + entry_point_distribution, + granted_distribution, + override_granted, + ) + from fluid_build.providers import RETIRED_PROVIDERS + + who = entry_point_distribution(ep) or "" + if key in RETIRED_PROVIDERS: + _log.warning( + "iac_provider_name_reserved: plugin %r (entry point %r) tried to register the retired " + "platform %r; it was skipped without being imported", + who, + ep.name, + key, + extra={"evt": "iac_provider_name_reserved", "_name": key, "plugin": who}, + ) + return True + if override_granted(ep): + _log.info( + "iac_provider_override_granted: plugin %r replaces the built-in IaC platform %r " + "(granted in FLUID_PLUGINS_ALLOWLIST)", + who, + key, + extra={"evt": "iac_provider_override_granted", "_name": key, "plugin": who}, + ) + return False + dist = granted_distribution(ep) + hint = ( + f"; to allow this override add {f'{dist}:{ep.name}'!r} to FLUID_PLUGINS_ALLOWLIST" + if dist is not None + else "" + ) + _log.warning( + "iac_provider_name_reserved: plugin %r (entry point %r) tried to register the built-in " + "IaC platform %r; it was skipped without being imported%s", + who, + ep.name, + key, + hint, + extra={"evt": "iac_provider_name_reserved", "_name": key, "plugin": who}, + ) + return True + + +#: ``(scanner, index)``: one metadata scan per process (about 8 ms), reused by +#: the per-exposure platform lookups. Keyed on the scanner so a test that swaps +#: ``plugin_manager._entry_points`` gets a fresh read. +_INDEX: Optional[Tuple[Any, Dict[str, List[Any]]]] = None - A third-party package declares:: - [project.entry-points."fluid_build.iac_providers"] - mycloud = "my_pkg.iac:MyCloudIacPlugin" +def _entrypoints_by_name() -> Dict[str, List[Any]]: + """``{normalised name: [entry points]}`` for the group. Reads names; never loads. - where the referenced object is an :class:`IacProviderPlugin` (a class is - instantiated; an instance is used as-is). Per-plugin fail-isolation: a plugin - that fails to load/instantiate is skipped (logged by type only) and never - drops the others. Runs after the in-tree built-ins, so an external plugin may - add a new cloud — and intentionally override a built-in of the same name. + A list per name: two distributions can ship the same one. """ - log = logger or logging.getLogger(__name__) - # Imported lazily to keep the IaC package decoupled from the plugin manager - # at import time during early CLI bootstrap. - from fluid_build.plugin_manager import ROLE_GROUPS, iter_plugins + global _INDEX # noqa: PLW0603 + from fluid_build import plugin_manager - for name, obj in iter_plugins(ROLE_GROUPS["iac_provider"], logger=log): + scanner = plugin_manager._entry_points + cached = _INDEX + if cached is None or cached[0] is not scanner: + group = plugin_manager.ROLE_GROUPS["iac_provider"] + index: Dict[str, List[Any]] = {} + for ep in scanner(group): + index.setdefault(_norm(ep.name), []).append(ep) + cached = (scanner, index) + _INDEX = cached # read back above on the next call + return cached[1] + + +def _who(ep: Any) -> str: + from fluid_build.plugin_manager import entry_point_distribution + + return entry_point_distribution(ep) or "" + + +def _cancelled_grant_reason(key: str) -> str: + """Why a grant for ``key`` is cancelled when policy refuses the name.""" + from fluid_build.plugin_manager import _BLOCKLIST_ENV, _env_names, _policy_key + + if key in {_policy_key(n) for n in _env_names(_BLOCKLIST_ENV)}: + return f"{key!r} is in FLUID_PLUGINS_BLOCKLIST" + return f"the bare names in FLUID_PLUGINS_ALLOWLIST do not include {key!r}" + + +def _resolve(key: str, eps: List[Any], logger: logging.Logger) -> Optional[Any]: + """The pre-load gates for one name: policy, reserved names, duplicates. + + Returns the one entry point that may load, else ``None`` (after a WARNING + where an operator needs to act). Nothing is imported here. + """ + from fluid_build.plugin_manager import is_allowed, override_granted + + if not is_allowed(key): + logger.debug("iac plugin %r skipped by allow/block policy", key) + for ep in eps: + if override_granted(ep): + logger.warning( + "iac_provider_grant_cancelled: the grant for plugin %r (entry point %r) is " + "cancelled because %s; the platform %r stays unserved by it", + _who(ep), + ep.name, + _cancelled_grant_reason(key), + key, + extra={"evt": "iac_provider_grant_cancelled", "_name": key}, + ) + return None + if key in reserved_iac_names(): + # Warns about every ungranted claim; the granted ones come back. + granted = [ep for ep in eps if not _claims_reserved_name(ep)] + if len(granted) > 1: + logger.warning( + "iac_provider_name_ambiguous: several plugins (%s) are granted the built-in " + "IaC platform %r; none was imported", + ", ".join(repr(_who(ep)) for ep in granted), + key, + extra={"evt": "iac_provider_name_ambiguous", "_name": key}, + ) + return None + return granted[0] if granted else None + if len(eps) > 1: + logger.warning( + "iac_provider_name_ambiguous: %d plugins (%s) ship the IaC platform %r; none was " + "imported; block the name with FLUID_PLUGINS_BLOCKLIST or uninstall the extras", + len(eps), + ", ".join(repr(_who(ep)) for ep in eps), + key, + extra={"evt": "iac_provider_name_ambiguous", "_name": key}, + ) + return None + return eps[0] + + +def _instantiate(name: str, obj: Any, logger: logging.Logger) -> Optional[IacProviderPlugin]: + try: + return obj() if isinstance(obj, type) else obj + except Exception as e: # noqa: BLE001 - isolate a bad plugin, log by type only + logger.warning("iac plugin %r failed to instantiate: %s", name, type(e).__name__) + return None + + +def _compatible(name: str, plugin: Any, logger: logging.Logger) -> bool: + """The version gate, shared with the provider registry (advisory unless strict).""" + from fluid_build.providers import _CLI_VERSION, _check_sdk_compat, _safe_log, _strict_compat + + if _check_sdk_compat(name, plugin, logger) or not _strict_compat(): + return True + _safe_log( + logger, + logging.ERROR, + "iac_plugin_compat_rejected", + name=name, + cli_version=_CLI_VERSION, + ) + return False + + +def _load_key(key: str, eps: List[Any], logger: logging.Logger) -> Optional[IacProviderPlugin]: + """Run the gates for ``key``, load the admitted entry point and register what it + yields. ``None`` if refused.""" + global _LOADING # noqa: PLW0603 + with _LOCK: + ep = _resolve(key, eps, logger) + if ep is None: + return None + _LOADING = True try: - plugin = obj() if isinstance(obj, type) else obj - except Exception as e: # noqa: BLE001 - isolate a bad plugin, log by type only - log.warning("iac plugin %r failed to instantiate: %s", name, type(e).__name__) - continue - cloud = getattr(plugin, "name", name) or name - register_iac_plugin(cloud, plugin) - if cloud in IAC_PLUGINS: - # Only record what actually registered — a name the allow/block - # policy rejected must not route to the OpenTofu engine. - IAC_ENTRYPOINT_PLUGINS.add(cloud) - log.debug("registered external iac plugin %r", name) + try: + obj = ep.load() + except Exception as e: # noqa: BLE001 - isolate a bad plugin, log by type only + logger.warning("plugin %r failed to load: %s", ep.name, type(e).__name__) + return None + plugin = _instantiate(ep.name, obj, logger) + if plugin is None or not _compatible(ep.name, plugin, logger): + return None + # Keyed by the entry-point name, the one that was vetted: the plugin's own + # ``name`` is the Terraform provider's local name and may legitimately be + # ``aws`` (an on-prem S3 emitter driving the aws provider). + register_iac_plugin(key, plugin) + finally: + _LOADING = False + if IAC_PLUGINS.get(key) is not plugin: + return None + IAC_ENTRYPOINT_PLUGINS.add(key) + logger.debug("registered external iac plugin %r", ep.name) + return plugin + + +def get_iac_plugin(name: str) -> Optional[IacProviderPlugin]: + """Return the plugin for ``name``, loading it from an entry point if need be. + + The load point. A registered (in-tree or already loaded) plugin is returned + as is. Otherwise the ``fluid_build.iac_providers`` group is consulted by + NAME (normalised: case-insensitive, ``-`` == ``_``), and only a matching, + admitted entry point is imported (``stevedore.driver.DriverManager`` + semantics). ``None`` when nothing serves ``name``; + :func:`iac_plugin_unavailable_reason` says why an installed one was refused. + """ + if not isinstance(name, str) or not name.strip(): + return IAC_PLUGINS.get(name) + key = _norm(name) + plugin = IAC_PLUGINS.get(key) + if key in _REFUSED or key in IAC_ENTRYPOINT_PLUGINS: + return plugin + if key in reserved_iac_names(): + # Served by the in-tree table alone, with no metadata scan, unless the + # operator listed a distribution-qualified grant for this name. + if not _override_listed(key): + return plugin + elif plugin is not None: + return plugin + with _LOCK: + # Re-check inside the lock: another thread may have loaded or refused it. + if key in IAC_ENTRYPOINT_PLUGINS or key in _REFUSED: + return IAC_PLUGINS.get(key) + eps = _entrypoints_by_name().get(key) + if not eps: + return IAC_PLUGINS.get(key) + if _load_key(key, eps, _log) is None: + _REFUSED.add(key) + return IAC_PLUGINS.get(key) + + +def _override_listed(key: str) -> bool: + """Whether ``FLUID_PLUGINS_ALLOWLIST`` holds any ``:`` entry (env only).""" + from fluid_build.plugin_manager import _ALLOWLIST_ENV, _QUALIFIED_SEP, _env_names + + for entry in _env_names(_ALLOWLIST_ENV): + _dist, sep, ep_name = entry.partition(_QUALIFIED_SEP) + if sep and _norm(ep_name) == key: + return True + return False + + +def is_iac_entrypoint_platform(name: str) -> bool: + """Whether an ALLOWED, non-reserved entry point is installed for ``name``. + + Names only, nothing imported: this is what lets contract detection and + ``--provider`` choices see a plugin platform without loading it. + """ + if not isinstance(name, str) or not name.strip(): + return False + key = _norm(name) + if key in reserved_iac_names(): + return False + from fluid_build.plugin_manager import is_allowed + + return bool(_entrypoints_by_name().get(key)) and is_allowed(key) + + +def known_iac_platforms() -> List[str]: + """Sorted platforms ``generate iac`` can serve: in-tree plus installed plugins. + + Plugin names are read, not loaded (blocked and reserved ones are left out). + """ + from fluid_build.plugin_manager import is_allowed + + names = set(IAC_PLUGINS) + reserved = reserved_iac_names() + for key in _entrypoints_by_name(): + if key not in reserved and is_allowed(key): + names.add(key) + return sorted(names) + + +def iac_plugin_unavailable_reason(name: str) -> Optional[str]: + """Why an installed plugin for ``name`` is not serving it, else ``None``.""" + from fluid_build.plugin_manager import is_allowed, override_granted + + key = _norm(name) + eps = _entrypoints_by_name().get(key) + if not eps: + return None + if key in reserved_iac_names(): + if not is_allowed(key) and any(override_granted(ep) for ep in eps): + return ( + f"the grant for the IaC plugin from {_who(eps[0])!r} is cancelled because " + f"{_cancelled_grant_reason(key)}" + ) + return None + if not is_allowed(key): + return ( + f"the IaC plugin from {_who(eps[0])!r} is blocked by FLUID_PLUGINS_ALLOWLIST/BLOCKLIST" + ) + if len(eps) > 1: + dists = ", ".join(repr(_who(ep)) for ep in eps) + return ( + f"the IaC platform {key!r} is shipped by several distributions ({dists}) and none " + "was loaded; block the name with FLUID_PLUGINS_BLOCKLIST or uninstall the extras" + ) + return f"the IaC plugin from {_who(eps[0])!r} was refused or failed to load (see the log)" + + +class IacProviderChoices: + """argparse ``choices=`` for ``--provider``: ``auto``, the in-tree clouds, and + the installed IaC plugin platforms. + + Membership is tested per value, and iteration (``--help``, error text) reads + entry-point NAMES; neither imports a plugin, and nothing is read when the + parser is built. + """ + + def __contains__(self, value: object) -> bool: + if not isinstance(value, str): + return False + return value == "auto" or value in IAC_PLUGINS or is_iac_entrypoint_platform(value) + + def __iter__(self): + return iter(["auto", *known_iac_platforms()]) + + def __len__(self) -> int: + return len(list(iter(self))) + + +def reset_iac_plugin_cache() -> None: + """Forget external plugins and refusals (tests; an install into a live process). + + A built-in an external plugin replaced (a granted override) is restored from + the snapshot taken when it registered. + """ + global _INDEX # noqa: PLW0603 + with _LOCK: + _INDEX = None + for key in list(IAC_ENTRYPOINT_PLUGINS): + IAC_PLUGINS.pop(key, None) + builtin = _INTREE.get(key) + if builtin is not None: + IAC_PLUGINS[key] = builtin + IAC_ENTRYPOINT_PLUGINS.clear() + _REFUSED.clear() + + +def discover_iac_entrypoints(logger: Optional[logging.Logger] = None) -> None: + """Eagerly load EVERY allowed entry-point plugin. Nothing calls this at import. + + For callers that need the whole set (the state-ownership guard before a + destructive apply, an inspection command, a test); the CLI otherwise loads by + name through :func:`get_iac_plugin`. Same gates, per-plugin fail-isolation: a + bad plugin is skipped and never drops the others. + """ + log = logger or _log + with _LOCK: + for key, eps in list(_entrypoints_by_name().items()): + if key in IAC_ENTRYPOINT_PLUGINS: + continue + _load_key(key, eps, log) diff --git a/fluid_build/iac/row_access.py b/fluid_build/iac/row_access.py index 9d758b24f..5f3e3155d 100644 --- a/fluid_build/iac/row_access.py +++ b/fluid_build/iac/row_access.py @@ -43,7 +43,7 @@ from .base import UnsupportedBindingError from .column_access import policy_where -_NAME = re.compile(r"^[a-z][a-z0-9_]{0,62}$") +_NAME = re.compile(r"^[a-z][a-z0-9_]{0,62}\Z") @dataclass(frozen=True) @@ -84,7 +84,11 @@ def _predicate_columns(where: str, at: str) -> Tuple[str, ...]: try: import sqlglot from sqlglot import expressions as exp - except ImportError: # pragma: no cover - sqlglot is a core dependency + except ImportError: # pragma: no cover - sqlglot is a base dependency + # Fails OPEN on the structural checks: without sqlglot the separator and + # comment check above still runs, but subqueries, UNION/GROUP/ORDER/LIMIT + # and undeclared columns go unchecked. This is sqlglot's real consumer in + # the base install, so keep it in `dependencies`. return () try: tree = sqlglot.parse_one(f"SELECT 1 FROM t WHERE {where}") diff --git a/fluid_build/iac/runner.py b/fluid_build/iac/runner.py index c59140aa8..879dee5da 100644 --- a/fluid_build/iac/runner.py +++ b/fluid_build/iac/runner.py @@ -26,11 +26,14 @@ from __future__ import annotations import json +import logging import os import re import shutil +import signal import subprocess from dataclasses import dataclass, field +from pathlib import Path from typing import Any, Dict, List, Mapping, Optional, Tuple # Per-command wall-clock cap. A hung ``tofu`` (e.g., an unauthenticated @@ -43,6 +46,13 @@ # can take many minutes for the eventual consistency to settle). _DEFAULT_TOFU_TIMEOUT_SECONDS = 1800 _MIN_REQUIRED_VERSION = (1, 6, 0) +# S3-native state locking (``use_lockfile``) arrived in OpenTofu 1.10. +_S3_LOCKFILE_MIN_VERSION = (1, 10, 0) +# After SIGINT on a timeout, how long tofu gets to release its lock and exit +# before it is killed. +_TIMEOUT_GRACE_SECONDS = 30 + +logger = logging.getLogger(__name__) def _resolve_timeout() -> int: @@ -162,33 +172,41 @@ def _run( ) -> TofuResult: tofu = _require_tofu() timeout = _resolve_timeout() + child = subprocess.Popen( + [tofu, *args], + cwd=workdir, + env=dict(env) if env is not None else None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) try: - proc = subprocess.run( - [tofu, *args], - cwd=workdir, - env=dict(env) if env is not None else None, - capture_output=True, - text=True, - timeout=timeout, - ) - except subprocess.TimeoutExpired as exc: + stdout, stderr = child.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + # SIGINT first: tofu's interrupt stops the operation and releases the + # state lock. A SIGKILL leaves ``.tflock`` behind and every later + # run fails until someone runs ``tofu force-unlock``. Only a tofu that + # ignores the interrupt for the grace period is killed. + try: + child.send_signal(signal.SIGINT) + stdout, _ = child.communicate(timeout=_TIMEOUT_GRACE_SECONDS) + except (subprocess.TimeoutExpired, OSError, ValueError): + child.kill() + stdout, _ = child.communicate() # Surface the partial output if any — the timeout itself is the # signal (apply engine treats a TofuResult with returncode != 0 # as a failure). Use returncode=124 to mirror coreutils ``timeout``. return TofuResult( command=command, returncode=124, - stdout=( - (exc.stdout or b"").decode("utf-8", errors="replace") - if isinstance(exc.stdout, (bytes, bytearray)) - else (exc.stdout or "") - ), + stdout=stdout or "", stderr=( f"`tofu {command}` exceeded the {timeout}s wall-clock " "limit (FLUID_TOFU_TIMEOUT_SECONDS overrides the default)." ), events=[], ) + proc = subprocess.CompletedProcess(child.args, child.returncode, stdout, stderr) events = _parse_json_events(proc.stdout) if "-json" in args else [] return TofuResult( command=command, @@ -199,6 +217,101 @@ def _run( ) +def _needs_reconfigure_for_lockfile(workdir: str, config: Mapping[str, Any]) -> bool: + """True when ``workdir`` was initialised on this same S3 state, without the lockfile. + + Adding ``use_lockfile`` to a kept workdir changes its backend config, which + ``tofu init`` refuses without ``-reconfigure``. That is safe only when the + recorded backend is this one: the same bucket, key and region (compared when + the module names one), ``use_lockfile`` not already true. A recorded backend + that differs in any of them is left to tofu's own error, never repointed. + """ + from .state_migration import recorded_backend + + recorded = recorded_backend(Path(workdir)) + if not recorded or set(recorded) != {"s3"}: + return False + have = recorded["s3"] or {} + if have.get("use_lockfile") is True: + return False + fields = ("bucket", "key") + (("region",) if config.get("region") else ()) + return all(have.get(k) == config.get(k) for k in fields) + + +def _drops_saved_lockfile(workdir: str, config: Mapping[str, Any]) -> bool: + """True when ``workdir`` saved ``use_lockfile=true`` for this same bucket and key. + + With no ``-backend-config`` override tofu reuses that saved partial config, so + an opt-out (or a tofu we will not pass the flag to) needs ``-reconfigure`` to + drop it. Same bucket and key, so no state moves. + """ + from .state_migration import recorded_backend + + recorded = recorded_backend(Path(workdir)) + if not recorded or set(recorded) != {"s3"}: + return False + have = recorded["s3"] or {} + return have.get("use_lockfile") is True and all( + have.get(k) == config.get(k) for k in ("bucket", "key") + ) + + +def _s3_lockfile_args(workdir: str, *, force_copy: bool = False) -> List[str]: + """``-backend-config=use_lockfile=true`` when the module's backend is S3. + + The flag is passed at init, not emitted into the module, so the module's + bytes (an adopter may keep them as golden files) do not change. Gated on + the installed tofu (>= 1.10) and switched off by ``FLUID_TOFU_S3_LOCKFILE=0`` + for a role that cannot write the ``.tflock`` object. ``-reconfigure`` + rides along when a kept workdir recorded this state without the lockfile, and + when it saved the lockfile but the flag is no longer passed (so the opt-out + takes effect). Never with ``force_copy``: tofu lets ``-reconfigure`` win over + ``-force-copy`` and would silently skip the state copy. + """ + try: + with open(os.path.join(workdir, "main.tf.json"), encoding="utf-8") as fh: + backend = json.load(fh).get("terraform", {}).get("backend", {}) + except (OSError, ValueError, AttributeError): + return [] + if not isinstance(backend, dict) or not isinstance(backend.get("s3"), dict): + return [] + if os.environ.get("FLUID_TOFU_S3_LOCKFILE", "").strip().lower() in ("0", "false", "no", "off"): + return _unlock_args(workdir, backend["s3"], force_copy) + version = tofu_version() + if version is None or version < _S3_LOCKFILE_MIN_VERSION: + if version is None: + reason = ( + "the OpenTofu version could not be read, so the lock flag was skipped " + "rather than risk an unrecognised build failing init" + ) + else: + reason = ( + f"OpenTofu {'.'.join(str(p) for p in version)} predates 1.10, so two applies " + "on the same product can race. Upgrade OpenTofu to 1.10 or newer, " + "or run with --ensure-opentofu (pins a version that locks)" + ) + logger.warning( + json.dumps( + { + "event": "s3_state_unlocked", + "tofu_version": ".".join(str(p) for p in version) if version else None, + "message": f"S3 state is not locked: {reason}.", + } + ) + ) + return _unlock_args(workdir, backend["s3"], force_copy) + args = ["-backend-config=use_lockfile=true"] + if not force_copy and _needs_reconfigure_for_lockfile(workdir, backend["s3"]): + args.append("-reconfigure") + return args + + +def _unlock_args(workdir: str, config: Mapping[str, Any], force_copy: bool) -> List[str]: + if not force_copy and _drops_saved_lockfile(workdir, config): + return ["-reconfigure"] + return [] + + def tofu_init( workdir: str, *, @@ -223,7 +336,9 @@ def tofu_init( args = ["init", "-input=false", "-no-color"] if not backend: args.append("-backend=false") - if reconfigure: + else: + args.extend(a for a in _s3_lockfile_args(workdir, force_copy=force_copy) if a not in args) + if reconfigure and "-reconfigure" not in args: args.append("-reconfigure") if force_copy: args.append("-force-copy") diff --git a/fluid_build/iac/state_migration.py b/fluid_build/iac/state_migration.py index 09634d084..61d953e0b 100644 --- a/fluid_build/iac/state_migration.py +++ b/fluid_build/iac/state_migration.py @@ -251,8 +251,12 @@ def other_clouds(resources: Iterable[Mapping[str, Any]], provider: str) -> Froze def _sources_by_plugin(provider: str) -> Dict[str, FrozenSet[str]]: - from .registry import IAC_PLUGINS + from .registry import IAC_PLUGINS, discover_iac_entrypoints + # Attribute a state against EVERY admitted plugin, not only the ones loaded so + # far: this is the guard that stops a destructive apply from deleting another + # cloud's resources. Laziness matters for ``--help``/``providers``, not here. + discover_iac_entrypoints() by_plugin = {name: plugin_sources(name) for name in IAC_PLUGINS} by_plugin.setdefault(provider, plugin_sources(provider)) return by_plugin diff --git a/fluid_build/llm/litellm_backend.py b/fluid_build/llm/litellm_backend.py index 9b61d6508..0967ea108 100644 --- a/fluid_build/llm/litellm_backend.py +++ b/fluid_build/llm/litellm_backend.py @@ -83,20 +83,31 @@ def _get_litellm() -> Any: """Lazy-import litellm; raise a typed error with the install hint. - ``litellm`` is a hard dependency in ``pyproject.toml`` so this - branch only fires in pathological environments (a partial - installation, an interpreter mismatch). The suggestion points at - the canonical re-install command. + ``litellm`` is an optional dependency (the ``[litellm]`` extra; it is + still installed by default today). The gate is + :func:`fluid_build._optional.require`, which probes before importing, so a + present-but-broken litellm is reported as such rather than as missing. """ + from fluid_build._optional import MissingOptionalDependency, require + try: - import litellm # type: ignore[import-untyped] + litellm = require("litellm", "litellm", "AI-assisted authoring") + except MissingOptionalDependency as exc: + raise CopilotGenerationError( + "copilot_litellm_unavailable", + f"{exc.feature} needs {exc.package}, which is not installed.", + suggestions=[f"pip install 'data-product-forge[{exc.extra}]'"], + ) from exc except ImportError as exc: + # Present but broken (e.g. litellm >= 1.98 on Python 3.10): same typed + # error as "missing", so every existing caller degrades the same way. + # ``require`` already wrote "... installed but failed to import: ". raise CopilotGenerationError( "copilot_litellm_unavailable", - "litellm is not installed (this is a hard dependency).", + str(exc), suggestions=[ - "pip install 'data-product-forge[litellm]'", - "Or install litellm directly: pip install litellm", + "pip install 'data-product-forge[litellm]' " + "(installs the litellm version this Python supports)" ], ) from exc return litellm diff --git a/fluid_build/llm/providers.py b/fluid_build/llm/providers.py index 53e3817ad..dff2b30e4 100644 --- a/fluid_build/llm/providers.py +++ b/fluid_build/llm/providers.py @@ -495,8 +495,9 @@ def get_llm_provider(name: str) -> LlmProvider: remaining name falls through to litellm's native provider routing. Plugin discovery is lazy (first call here) so ``fluid --help`` never scans it. - ``litellm`` is a hard dependency of fluid (declared in - ``pyproject.toml``); the import never fails in a working install. + ``litellm`` is an optional dependency (the ``[litellm]`` extra, still + installed by default today). Reaching for it without the package raises a + ``copilot_litellm_unavailable`` error carrying the install line. """ normalized = (name or "").strip().lower() if normalized in ("mcp-sampling", "mcp_sampling"): @@ -605,9 +606,6 @@ def invoke_blocking( # Lazy imports avoid an import cycle: ``mcp.py`` imports the # provider catalog from this module (via LiteLLM), so importing # ``mcp`` at module-load time here would deadlock. - import anyio.from_thread - from mcp.types import SamplingMessage, TextContent - from fluid_build._mcp_sampling_context import get_sampling_context ctx, anyio_token = get_sampling_context() @@ -625,6 +623,17 @@ def invoke_blocking( ], ) + # Imported only once a sampling context exists: that context is installed + # by the MCP server, so the SDK is present whenever this line is reached, + # and a terminal run without ``mcp`` gets the no-context error above, not + # an ImportError. + import anyio.from_thread + + from fluid_build._optional import require + + mcp_types = require("mcp.types", "mcp", "MCP sampling") + SamplingMessage, TextContent = mcp_types.SamplingMessage, mcp_types.TextContent + max_tokens = int(getattr(config, "max_tokens", 4096) or 4096) temperature = getattr(config, "temperature", None) sampling_kwargs: Dict[str, Any] = { diff --git a/fluid_build/llm/router.py b/fluid_build/llm/router.py index 5af295d54..617b5a3f1 100644 --- a/fluid_build/llm/router.py +++ b/fluid_build/llm/router.py @@ -251,7 +251,9 @@ def _build_router(model_list: List[Dict[str, Any]]) -> Any: * ``set_verbose=False`` — keep litellm's debug logging off; we own observability via the structured-events bus. """ - import litellm # type: ignore[import-untyped] + from fluid_build._optional import require + + litellm = require("litellm", "litellm", "LLM fallback routing") # ``fallbacks`` is a list-of-dict-of-list: each dict maps the # primary model_name to a list of fallback model_names. With all diff --git a/fluid_build/observability/reporter.py b/fluid_build/observability/reporter.py index b0eafd546..098eea7cc 100644 --- a/fluid_build/observability/reporter.py +++ b/fluid_build/observability/reporter.py @@ -23,12 +23,7 @@ import time from enum import Enum from typing import Any, Dict, List, Mapping, Optional, Tuple -from urllib.parse import urljoin, urlparse, urlsplit - -try: - import requests -except ImportError: - requests = None # type: ignore +from urllib.parse import urlparse, urlsplit # Reuse the canonical SSRF post-DNS-resolution gate (RFC1918, # link-local 169.254.0.0/16 — AWS/GCP metadata — loopback, reserved; @@ -171,8 +166,8 @@ def credential_redirect_refusal(source: str, target: str) -> Optional[str]: ``None`` when it may: ``target`` (absolute) is on the same origin (scheme, host and port), or is the same host moved from http to https on the - default ports, the one cross-origin hop for which ``requests`` keeps - ``Authorization`` (``should_strip_auth``). Every other hop is refused + default ports, the one cross-origin hop for which the HTTP client keeps + ``Authorization``. Every other hop is refused rather than followed with the credential stripped: the request body is the run's report, which is not for another origin either. An https request is never followed down to http. @@ -193,62 +188,45 @@ def credential_redirect_refusal(source: str, target: str) -> Optional[str]: def refuse_foreign_redirect(response: Any, *_args: Any, **_kwargs: Any) -> Any: - """A ``requests`` response hook: raise :class:`CredentialRedirectRefused` for - a redirect :func:`credential_redirect_refusal` refuses, before it is followed. + """An ``httpx`` ``response`` event hook: raise :class:`CredentialRedirectRefused` + for a redirect :func:`credential_redirect_refusal` refuses, before it is followed. - ``requests`` runs a request's response hooks on every hop, so passing - ``hooks={"response": refuse_foreign_redirect}`` to one call guards that - call; :func:`guard_credential_redirects` installs it on a whole session. + ``httpx`` runs a client's ``response`` hooks on every hop, so a client built + with ``response_hooks=[refuse_foreign_redirect]`` + (:func:`fluid_build.util.http_client.new_client`) guards every request it + makes, and the hook sees each redirect before the client follows it. """ if not response.is_redirect: return response - target = urljoin(response.url, response.headers.get("location", "")) - refusal = credential_redirect_refusal(response.url, target) + import httpx # a hook only runs inside an httpx client; keep it off the cold path + + source = str(response.url) + location = response.headers.get("location", "") + try: + parsed = httpx.URL(location) + except httpx.InvalidURL: + parsed = None + if parsed is None: + refusal: Optional[str] = "it redirects to a Location that is not a valid URL" + elif parsed.scheme and not parsed.host: + # ``httpx`` copies the request's host onto such a Location (``https:/x``, + # ``http:``) and drops its port; never judge or follow that form. + refusal = "it redirects to a Location with a scheme but no host" + else: + # The URL the client will follow: the Location resolved against the + # request URL (the public ``URL.join``), not ``urljoin`` on the response URL. + refusal = credential_redirect_refusal(source, str(response.request.url.join(parsed))) if refusal is None: return response - response.close() + try: + response.close() + except RuntimeError: + # An async client's stream cannot be closed synchronously; the async + # wrapper (util.http_client.new_async_client) closes it with aclose(). + pass raise CredentialRedirectRefused(refusal) -def guard_credential_redirects(session: Any) -> Any: - """Make ``session`` (a ``requests.Session``) refuse a redirect its credentials may not follow. - - ``requests`` follows redirects by default and, when the host changes, drops - only ``Authorization``: a credential in any other header (``X-API-Key``) - went on to wherever the endpoint redirected. A response hook - (:func:`refuse_foreign_redirect`) sees every hop before ``requests`` - follows it, and raises :class:`CredentialRedirectRefused` for one - :func:`credential_redirect_refusal` refuses. Returns ``session``. - """ - session.hooks["response"].append(refuse_foreign_redirect) - return session - - -def foreign_redirect_trace_config() -> Any: - """An ``aiohttp.TraceConfig`` that refuses what :func:`refuse_foreign_redirect` refuses. - - The same rule for an ``aiohttp.ClientSession``, which follows redirects by - default: built with ``trace_configs=[foreign_redirect_trace_config()]``, a - session raises :class:`CredentialRedirectRefused` for every hop - :func:`credential_redirect_refusal` refuses, on every request it makes, - before ``aiohttp`` follows it. - """ - import aiohttp - - async def _on_redirect(_session: Any, _context: Any, params: Any) -> None: - response = params.response - source = str(response.url) - location = response.headers.get("Location") or response.headers.get("URI") or "" - refusal = credential_redirect_refusal(source, urljoin(source, location)) - if refusal is not None: - response.close() - raise CredentialRedirectRefused(refusal) - - trace = aiohttp.TraceConfig() - trace.on_request_redirect.append(_on_redirect) - return trace - - class CircuitState(Enum): """Circuit breaker states.""" @@ -339,7 +317,7 @@ def _on_failure(self, error: Exception): self.last_failure_time = time.time() if self.state == CircuitState.HALF_OPEN: - # The type only: a requests exception's text repeats the URL. + # The type only: an HTTP client exception's text repeats the URL. logger.warning( "Circuit breaker transitioning HALF_OPEN → OPEN (recovery failed: %s)", type(error).__name__, @@ -409,13 +387,20 @@ def __init__(self, config: CommandCenterConfig): # Session for connection pooling. Every request carries the credential, # so a redirect is followed only within the endpoint's own origin. self.session: Optional[Any] = None - if self.enabled and requests: - self.session = guard_credential_redirects(requests.Session()) + if self.enabled: + from fluid_build.util.http_client import CaBundleError, new_client + headers = {"Content-Type": "application/json"} if config.api_key: headers["X-API-Key"] = config.api_key headers.update(config.headers or {}) - self.session.headers.update(headers) + try: + self.session = new_client(headers=headers, response_hooks=[refuse_foreign_redirect]) + except CaBundleError as exc: + # An unusable CA bundle must not take the command down, and TLS + # is not relaxed to get past it: reporting is off for this run. + logger.warning("Command Center reporter disabled: %s.", exc) + self.enabled = False def start(self): """Start background worker thread.""" @@ -423,11 +408,6 @@ def start(self): logger.debug("Command Center not configured, reporter disabled") return - if not requests: - logger.warning("requests library not installed, Command Center integration disabled") - self.enabled = False - return - # SSRF gate: ``config.url`` is user/config-supplied and every # event POST carries the X-API-Key header. Refuse to start the # reporter against a host that resolves to a private/link-local/ @@ -669,7 +649,7 @@ def _send_event(self, event: Dict[str, Any]): # Its text names the origins only. outcome = f"not followed, {e}" else: - # The type and the status only: a requests exception's text + # The type and the status only: an HTTP client exception's text # repeats the URL, and nothing here needs the response body. status = getattr(getattr(e, "response", None), "status_code", None) outcome = str(status or type(e).__name__) diff --git a/fluid_build/output_ports/mcp/__init__.py b/fluid_build/output_ports/mcp/__init__.py index 4a4ce5392..f9b5f1706 100644 --- a/fluid_build/output_ports/mcp/__init__.py +++ b/fluid_build/output_ports/mcp/__init__.py @@ -85,15 +85,19 @@ OutputPortMcpServer, run_stdio, ) -except ImportError: # pragma: no cover - exercised only when mcp SDK absent +except ImportError as _import_error: # pragma: no cover - exercised only when mcp SDK absent SERVER_NAME = "forge-cli-output-port-mcp" SERVER_VERSION = "0.1.0" + _server_import_error = _import_error def _missing_sdk(*_args, **_kwargs): # type: ignore[unused-ignore] - raise ImportError( - "The 'mcp' package is required for the Fluid MCP output-port " - "server. Install with: pip install 'mcp>=1.0,<2.0'" - ) + from fluid_build._optional import MissingOptionalDependency, is_available + + if is_available("mcp"): + # The SDK is installed but ``.server`` failed to import: say so + # rather than telling the user to install what they already have. + raise _server_import_error + raise MissingOptionalDependency("mcp", "mcp", "The MCP output-port server") OutputPortMcpServer = _missing_sdk # type: ignore[assignment] run_stdio = _missing_sdk # type: ignore[assignment] diff --git a/fluid_build/plugin_manager.py b/fluid_build/plugin_manager.py index b74157dcb..9213890b8 100644 --- a/fluid_build/plugin_manager.py +++ b/fluid_build/plugin_manager.py @@ -34,7 +34,10 @@ industry-standard property (dbt / pluggy / Backstage don't sandbox third-party plugins either; see ``SECURITY``). The operator pins exactly which plugins load via ``FLUID_PLUGINS_ALLOWLIST`` / ``FLUID_PLUGINS_BLOCKLIST`` (comma-separated -entry-point names; allowlist wins — if set, only listed names load). Every load +entry-point names, matched case-insensitively with ``-`` == ``_``). The +blocklist always wins; an allowlist holding a bare name restricts loading to the +listed names, while ``:`` entries only grant built-in +overrides and restrict nothing. Every load and invocation is fail-isolated and logged **by exception type only**, so a plugin bug or a secret-bearing exception message can neither crash the CLI nor leak into logs. @@ -44,6 +47,7 @@ import logging import os +import re from typing import Any, Dict, Iterator, List, Optional, Tuple # role tag (fluid_sdk.BasePlugin.role) -> entry-point group. Single source of truth. @@ -116,20 +120,80 @@ def _env_names(var: str) -> set: return {x.strip() for x in os.environ.get(var, "").split(",") if x.strip()} +def _policy_key(name: str) -> str: + """How allow/block entries and plugin names compare: case-insensitive, ``-`` == ``_``.""" + return str(name).strip().lower().replace("-", "_") + + def is_allowed(name: str) -> bool: """Return whether entry-point ``name`` may load under the allow/block policy. - Blocklist always wins; if an allowlist is set, only listed names load. + Names match case-insensitively, with ``-`` and ``_`` equal, so ``Kube-Rnetes`` + cannot slip past ``FLUID_PLUGINS_BLOCKLIST=kube_rnetes``. The blocklist always + wins. If the allowlist holds a bare name, only listed names load. A + ``:`` entry only grants an override (:func:`override_granted`) + and restricts nothing. """ - block = _env_names(_BLOCKLIST_ENV) - if name in block: + key = _policy_key(name) + if key in {_policy_key(n) for n in _env_names(_BLOCKLIST_ENV)}: return False - allow = _env_names(_ALLOWLIST_ENV) - if allow and name not in allow: + allow = {_policy_key(n) for n in _env_names(_ALLOWLIST_ENV) if _QUALIFIED_SEP not in n} + if allow and key not in allow: return False return True +_QUALIFIED_SEP = ":" + +#: PEP 508 project-name grammar (https://peps.python.org/pep-0508/#names), used +#: with ``fullmatch`` so a trailing newline cannot pass. +_DIST_NAME = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?") + + +def _norm_dist(name: str) -> str: + """PEP 503-normalised distribution name (``Acme_Fluid.AWS`` -> ``acme-fluid-aws``).""" + return re.sub(r"[-_.]+", "-", name.strip()).lower() + + +def granted_distribution(ep: Any) -> Optional[str]: + """The name of the distribution that ships ``ep``, when it is a valid project name. + + Read from the distribution itself (``ep.dist.name``), never from the + ``" "`` display string of :func:`entry_point_distribution`, + so a self-declared ``Name: acme-fluid-aws spoof`` cannot satisfy the grant + ``acme-fluid-aws:aws``. ``None``, so no grant is possible, for anything else. + """ + dist = getattr(ep, "dist", None) + name = getattr(dist, "name", None) if dist is not None else None + if isinstance(name, str) and _DIST_NAME.fullmatch(name): + return name + return None + + +def override_granted(ep: Any) -> bool: + """Whether the operator explicitly allowed ``ep`` to replace a built-in name. + + Built-in provider and IaC platform names are reserved: a third-party entry + point that claims one is skipped before it loads, because an installed + plugin could otherwise take over a name silently. An operator who wants + the override grants it with a distribution-qualified entry in + ``FLUID_PLUGINS_ALLOWLIST``: ``:``, for + example ``acme-fluid-aws:aws``. A bare name cannot grant it, because the + built-in has the same name. Qualified entries only grant overrides; they do + not make the allowlist restrict other plugins. + """ + dist = granted_distribution(ep) + if dist is None: + return False + want = f"{_norm_dist(dist)}{_QUALIFIED_SEP}{_policy_key(ep.name)}" + granted = set() + for entry in _env_names(_ALLOWLIST_ENV): + dist_name, sep, ep_name = entry.partition(_QUALIFIED_SEP) + if sep and dist_name.strip() and ep_name.strip(): + granted.add(f"{_norm_dist(dist_name)}{_QUALIFIED_SEP}{_policy_key(ep_name)}") + return want in granted + + def _entry_points(group: str) -> List[Any]: import importlib.metadata as md @@ -214,12 +278,14 @@ def installed_plugins(role: Optional[str] = None) -> Dict[str, List[Dict[str, An can audit or uninstall. Name-reading here is not the same as the *process* being plugin-free: CLI - bootstrap builds the full argparse tree first, and the ``iac_provider`` - and ``modeling_technique`` roles feed argparse ``choices=`` from their - registries, so those two roles are discovered — and their entry points - loaded — before any command runs, ``fluid --version`` included. Set - ``FLUID_PLUGINS_BLOCKLIST`` to inspect an untrusted plugin without - executing it; the policy is checked before every ``ep.load()``. + bootstrap builds the full argparse tree first, and the + ``modeling_technique`` role feeds argparse ``choices=`` from its registry, + so that role is discovered, and its entry points loaded, before any command + runs, ``fluid --version`` included. The ``iac_provider`` role is lazy: its + plugin is loaded by name when a contract's platform needs it + (``iac.registry.get_iac_plugin``). Set ``FLUID_PLUGINS_BLOCKLIST`` to + inspect an untrusted plugin without executing it; the policy is checked + before every ``ep.load()``. """ groups = governed_groups() keys = [role] if role else list(groups) diff --git a/fluid_build/providers/__init__.py b/fluid_build/providers/__init__.py index 30c54eab2..242444bf3 100644 --- a/fluid_build/providers/__init__.py +++ b/fluid_build/providers/__init__.py @@ -105,6 +105,68 @@ def _normalize_name(name: str) -> str: return name.strip().lower().replace("-", "_") +#: Names that are no longer providers -> what to do instead. Not +#: registry entries: a user who typed one gets this line (exit 2) rather than a +#: bare "unknown provider". Keys are :func:`_normalize_name` spellings. +RETIRED_PROVIDERS: Dict[str, str] = { + "redshift": "Redshift is part of the aws provider; use --provider aws", + "datamesh_manager": ( + "Data Mesh Manager is a catalog, not a provider; use " + "`fluid publish --target datamesh-manager` (or `fluid datamesh-manager`)" + ), +} + + +#: The provider that now serves a retired name used as a *platform* in a contract +#: (``builds[].execution.runtime.platform: redshift`` still validates). Data Mesh +#: Manager is a catalog, not a platform, so it has none. +RETIRED_SUCCESSORS: Dict[str, str] = {"redshift": "aws"} + + +def resolve_retired_platform(name: str) -> str: + """The successor provider for a retired platform name; any other name unchanged.""" + if not isinstance(name, str): + return name + return RETIRED_SUCCESSORS.get(_normalize_name(name), name) + + +def _claims_retired_name(name: str, who: str, logger: Optional[logging.Logger]) -> bool: + """True (after a WARNING) when a plugin tries to register a retired provider name. + + A retired name stays reserved: an entry point able to claim ``redshift`` would + replace the retirement hint for ``--provider redshift`` and win the platform + lookup for contracts. Only discovery calls this; an explicit + ``register_provider(...)`` call in code stays allowed. + """ + if not isinstance(name, str) or _normalize_name(name) not in RETIRED_PROVIDERS: + return False + # The plugin goes into the message itself, repr'd so control characters are + # escaped: the CLI's formatters print the message only, and this line is how + # an operator learns which package to uninstall. + (logger or _log).warning( + "provider_retired_name_reserved: %r tried to register the retired provider %r; " + "it was skipped", + who, + _normalize_name(name), + extra={ + "evt": "provider_retired_name_reserved", + "_name": _normalize_name(name), + "plugin": who, + }, + ) + return True + + +def retired_provider_hint(name: Optional[str]) -> Optional[str]: + """One-line ``--provider``/``FLUID_PROVIDER`` error for a retired name, else ``None``.""" + if not name: + return None + use = RETIRED_PROVIDERS.get(_normalize_name(name)) + if use is None: + return None + return f"Provider '{name}' was retired: {use}" + + def _add_discovery_error(source: str, modname: str, exc: BaseException) -> None: # DISCOVERY_ERRORS is surfaced to users via registry_dump() / `fluid providers # --debug`, so it must NOT carry raw exception text or a full traceback — either @@ -466,7 +528,7 @@ def discover_providers(logger: Optional[logging.Logger] = None, *, force: bool = return # What the policy now allows is no longer blocked. A name it still - # blocks stays recorded: one registered at import time (redshift, by + # blocks stays recorded: one registered at import time (aws, by # the aws package) is not offered again when its module is cached. from fluid_build.plugin_manager import is_allowed @@ -541,6 +603,12 @@ def _discover_entrypoints(logger: Optional[logging.Logger]) -> None: ) POLICY_BLOCKED.add(_normalize_name(ep.name)) continue + from fluid_build.plugin_manager import entry_point_distribution + + if _claims_retired_name( + ep.name, entry_point_distribution(ep) or str(getattr(ep, "value", ep)), logger + ): + continue try: provider_cls = ep.load() register_provider( @@ -634,6 +702,8 @@ def _auto_register_from_module(mod, logger: Optional[logging.Logger]) -> None: providers_map = getattr(mod, "PROVIDERS", None) if isinstance(providers_map, dict) and providers_map: for name, prov in list(providers_map.items()): + if _claims_retired_name(name, getattr(mod, "__name__", ""), logger): + continue try: register_provider(name, prov, override=False, logger=logger) except Exception as exc: @@ -651,6 +721,8 @@ def _auto_register_from_module(mod, logger: Optional[logging.Logger]) -> None: name = getattr(mod, "NAME", None) prov = getattr(mod, "Provider", None) if isinstance(name, str) and prov is not None: + if _claims_retired_name(name, getattr(mod, "__name__", ""), logger): + return try: register_provider(name, prov, override=False, logger=logger) _safe_log( @@ -686,6 +758,8 @@ def _auto_register_from_module(mod, logger: Optional[logging.Logger]) -> None: inferred = _normalize_name(getattr(cls, "name", cls.__name__).replace("Provider", "")) if not _is_valid_name(inferred): inferred = _normalize_name(getattr(mod, "__name__", "provider").rsplit(".", 1)[-1]) + if _claims_retired_name(inferred, getattr(mod, "__name__", ""), logger): + return register_provider(inferred, cls, override=False, logger=logger) _safe_log( logger, diff --git a/fluid_build/providers/_plan_validation.py b/fluid_build/providers/_plan_validation.py new file mode 100644 index 000000000..c2d82dd3d --- /dev/null +++ b/fluid_build/providers/_plan_validation.py @@ -0,0 +1,34 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The typed refusal a provider raises when its own plan fails validation. + +Kept apart from any one provider so ``cli/generate_iac.py`` can recognise it by +type without importing a provider package (which would register every provider +on a path that may be about another cloud). +""" + +from __future__ import annotations + +from fluid_build.providers.base import ProviderError + + +class PlanValidationError(ProviderError): + """A provider planned actions its own validator rejected. + + A planner that ran and refused, unlike one that could not run (no SDK, no + credentials): ``generate iac`` and ``apply`` treat the first as fatal and + only the second as best-effort. Typed so callers match on the class, never + on the message. + """ diff --git a/fluid_build/providers/aws/__init__.py b/fluid_build/providers/aws/__init__.py index 852679bf5..70e79879c 100644 --- a/fluid_build/providers/aws/__init__.py +++ b/fluid_build/providers/aws/__init__.py @@ -19,7 +19,8 @@ S3 / Glue / Athena / Lambda / EventBridge / Redshift. * :class:`RedshiftProvider` — owns Redshift's rollback DDL (DROP + CTAS) which is incompatible with the rest of AWS's S3 prefix-copy - semantics; registered under the ``"redshift"`` provider name. + semantics. Not a registered provider (retired as one, see + ``providers.RETIRED_PROVIDERS``); the rollback dispatch imports it directly. The legacy ``aws.py::AWSProvider`` (1601 LOC, registered as ``LegacyAWSProvider``) was deleted — its rollback methods migrated @@ -33,8 +34,5 @@ from .redshift_provider import RedshiftProvider register_provider("aws", AwsProvider) -# Redshift dispatches by its own name so the rollback writer can route -# DROP+CTAS DDL emission to a class with the right semantics. -register_provider("redshift", RedshiftProvider) __all__ = ["AwsProvider", "RedshiftProvider"] diff --git a/fluid_build/providers/aws/plan/planner.py b/fluid_build/providers/aws/plan/planner.py index 9f78013d2..e8cdda816 100644 --- a/fluid_build/providers/aws/plan/planner.py +++ b/fluid_build/providers/aws/plan/planner.py @@ -31,7 +31,7 @@ from ..util import warehouse as _warehouse from ..util.logging import format_event -from ..util.names import normalize_database_name +from ..util.names import EVENTBRIDGE_RULE_NAME, bounded_aws_name, normalize_database_name def _resolve_env_templates(value: str) -> str: @@ -740,7 +740,10 @@ def _plan_scheduling( { "op": "events.ensure_schedule", "id": "schedule_rule", - "name": f"{contract.get('id', 'contract').replace('.', '-')}-schedule", + "name": bounded_aws_name( + f"{contract.get('id', 'contract').replace('.', '-')}-schedule", + EVENTBRIDGE_RULE_NAME, + ), "schedule": eventbridge_schedule, "description": f"Schedule for {contract.get('name')}", "tags": _get_resource_tags(contract), diff --git a/fluid_build/providers/aws/plan/schedule.py b/fluid_build/providers/aws/plan/schedule.py index 6a3913fb4..f710417b1 100644 --- a/fluid_build/providers/aws/plan/schedule.py +++ b/fluid_build/providers/aws/plan/schedule.py @@ -28,8 +28,23 @@ import logging from typing import Any, Dict, List, Optional +from ..util.names import ( + EVENTBRIDGE_RULE_NAME, + LAMBDA_FUNCTION_NAME, + LAMBDA_STATEMENT_ID, + MWAA_ENVIRONMENT_NAME, + SCHEDULER_SCHEDULE_NAME, + STATE_MACHINE_NAME, + bounded_aws_name, +) + logger = logging.getLogger(__name__) +#: The IAM roles the planned Lambdas and schedules reference by literal ARN. The +#: planner emits no ``aws_iam_role``: both must already exist in the account. +WORKFLOW_EXECUTION_ROLE = "fluid-workflow-execution" +SCHEDULER_INVOKE_ROLE = "EventBridgeSchedulerRole" + class SchedulePlanner: """ @@ -52,6 +67,17 @@ def __init__(self, account_id: str, region: str, logger: Optional[logging.Logger self.region = region self.logger = logger or logging.getLogger(__name__) + @staticmethod + def workflow_function_name(contract_id: Any) -> str: + """The workflow Lambda's name: the one every trigger Lambda invokes. + + Every AWS name derived from a contract id goes through + :func:`bounded_aws_name`. A dotted id (``gold.hr.employee_360_v1``, the + schema's own example) or a long one used to reach the plan validator raw, + which refused the whole plan, so the schedule went missing from the module. + """ + return bounded_aws_name(f"fluid-workflow-{contract_id}", LAMBDA_FUNCTION_NAME) + def plan_schedule_actions(self, contract: Dict[str, Any]) -> List[Dict[str, Any]]: """ Generate AWS scheduling actions from contract. @@ -124,7 +150,7 @@ def _plan_mwaa_schedule( actions = [] contract_id = contract.get("id", "unknown") - environment_name = f"mwaa-{contract_id}" + environment_name = bounded_aws_name(f"mwaa-{contract_id}", MWAA_ENVIRONMENT_NAME) # Check if MWAA environment already specified mwaa_config = orchestration.get("mwaa", {}) @@ -174,7 +200,7 @@ def _plan_eventbridge_schedule( timezone = orchestration.get("timezone", "UTC") # 1. Create Lambda function for workflow execution - function_name = f"fluid-workflow-{contract_id}" + function_name = self.workflow_function_name(contract_id) actions.append( { @@ -183,7 +209,7 @@ def _plan_eventbridge_schedule( "function_name": function_name, "runtime": "python3.11", "handler": "index.handler", - "role": f"arn:aws:iam::{self.account_id}:role/fluid-workflow-execution", + "role": f"arn:aws:iam::{self.account_id}:role/{WORKFLOW_EXECUTION_ROLE}", "code": self._generate_workflow_lambda_code(contract, orchestration), "timeout": orchestration.get("timeout", 300), "memory_size": orchestration.get("memory", 256), @@ -197,7 +223,7 @@ def _plan_eventbridge_schedule( ) # 2. Create EventBridge Scheduler rule - schedule_name = f"fluid-schedule-{contract_id}" + schedule_name = bounded_aws_name(f"fluid-schedule-{contract_id}", SCHEDULER_SCHEDULE_NAME) actions.append( { @@ -208,7 +234,7 @@ def _plan_eventbridge_schedule( "timezone": timezone, "target": { "arn": f"arn:aws:lambda:{self.region}:{self.account_id}:function:{function_name}", - "role_arn": f"arn:aws:iam::{self.account_id}:role/EventBridgeSchedulerRole", + "role_arn": f"arn:aws:iam::{self.account_id}:role/{SCHEDULER_INVOKE_ROLE}", "input": json.dumps( {"contract_id": contract_id, "execution_type": "scheduled"} ), @@ -226,7 +252,9 @@ def _plan_eventbridge_schedule( "id": f"lambda_permission_{contract_id}", "op": "lambda.add_permission", "function_name": function_name, - "statement_id": f"AllowEventBridgeInvoke-{contract_id}", + "statement_id": bounded_aws_name( + f"AllowEventBridgeInvoke-{contract_id}", LAMBDA_STATEMENT_ID + ), "action": "lambda:InvokeFunction", "principal": "scheduler.amazonaws.com", "source_arn": f"arn:aws:scheduler:{self.region}:{self.account_id}:schedule/default/{schedule_name}", @@ -277,7 +305,9 @@ def _plan_s3_trigger( actions = [] contract_id = contract.get("id", "unknown") - function_name = f"fluid-s3-trigger-{contract_id}-{trigger_id}" + function_name = bounded_aws_name( + f"fluid-s3-trigger-{contract_id}-{trigger_id}", LAMBDA_FUNCTION_NAME + ) # 1. Create Lambda function actions.append( @@ -287,7 +317,7 @@ def _plan_s3_trigger( "function_name": function_name, "runtime": "python3.11", "handler": "index.handler", - "role": f"arn:aws:iam::{self.account_id}:role/fluid-workflow-execution", + "role": f"arn:aws:iam::{self.account_id}:role/{WORKFLOW_EXECUTION_ROLE}", "code": self._generate_s3_trigger_lambda_code(contract, trigger), "timeout": 300, "tags": self._get_schedule_tags(contract), @@ -318,7 +348,9 @@ def _plan_s3_trigger( "id": f"lambda_permission_s3_{trigger_id}", "op": "lambda.add_permission", "function_name": function_name, - "statement_id": f"AllowS3Invoke-{trigger_id}", + "statement_id": bounded_aws_name( + f"AllowS3Invoke-{trigger_id}", LAMBDA_STATEMENT_ID + ), "action": "lambda:InvokeFunction", "principal": "s3.amazonaws.com", "source_arn": f"arn:aws:s3:::{bucket}", @@ -334,7 +366,9 @@ def _plan_dynamodb_trigger( actions = [] contract_id = contract.get("id", "unknown") - function_name = f"fluid-dynamodb-trigger-{contract_id}-{trigger_id}" + function_name = bounded_aws_name( + f"fluid-dynamodb-trigger-{contract_id}-{trigger_id}", LAMBDA_FUNCTION_NAME + ) # 1. Create Lambda function actions.append( @@ -344,7 +378,7 @@ def _plan_dynamodb_trigger( "function_name": function_name, "runtime": "python3.11", "handler": "index.handler", - "role": f"arn:aws:iam::{self.account_id}:role/fluid-workflow-execution", + "role": f"arn:aws:iam::{self.account_id}:role/{WORKFLOW_EXECUTION_ROLE}", "code": self._generate_dynamodb_trigger_lambda_code(contract, trigger), "timeout": 300, "tags": self._get_schedule_tags(contract), @@ -376,8 +410,12 @@ def _plan_eventbridge_trigger( actions = [] contract_id = contract.get("id", "unknown") - rule_name = f"fluid-event-{contract_id}-{trigger_id}" - function_name = f"fluid-event-trigger-{contract_id}-{trigger_id}" + rule_name = bounded_aws_name( + f"fluid-event-{contract_id}-{trigger_id}", EVENTBRIDGE_RULE_NAME + ) + function_name = bounded_aws_name( + f"fluid-event-trigger-{contract_id}-{trigger_id}", LAMBDA_FUNCTION_NAME + ) # 1. Create Lambda function actions.append( @@ -387,7 +425,7 @@ def _plan_eventbridge_trigger( "function_name": function_name, "runtime": "python3.11", "handler": "index.handler", - "role": f"arn:aws:iam::{self.account_id}:role/fluid-workflow-execution", + "role": f"arn:aws:iam::{self.account_id}:role/{WORKFLOW_EXECUTION_ROLE}", "code": self._generate_event_trigger_lambda_code(contract, trigger), "timeout": 300, "tags": self._get_schedule_tags(contract), @@ -420,7 +458,9 @@ def _plan_eventbridge_trigger( "id": f"lambda_permission_eventbridge_{trigger_id}", "op": "lambda.add_permission", "function_name": function_name, - "statement_id": f"AllowEventBridgeInvoke-{trigger_id}", + "statement_id": bounded_aws_name( + f"AllowEventBridgeInvoke-{trigger_id}", LAMBDA_STATEMENT_ID + ), "action": "lambda:InvokeFunction", "principal": "events.amazonaws.com", "source_arn": f"arn:aws:events:{self.region}:{self.account_id}:rule/{rule_name}", @@ -436,7 +476,9 @@ def _plan_sqs_trigger( actions = [] contract_id = contract.get("id", "unknown") - function_name = f"fluid-sqs-trigger-{contract_id}-{trigger_id}" + function_name = bounded_aws_name( + f"fluid-sqs-trigger-{contract_id}-{trigger_id}", LAMBDA_FUNCTION_NAME + ) # 1. Create Lambda function actions.append( @@ -446,7 +488,7 @@ def _plan_sqs_trigger( "function_name": function_name, "runtime": "python3.11", "handler": "index.handler", - "role": f"arn:aws:iam::{self.account_id}:role/fluid-workflow-execution", + "role": f"arn:aws:iam::{self.account_id}:role/{WORKFLOW_EXECUTION_ROLE}", "code": self._generate_sqs_trigger_lambda_code(contract, trigger), "timeout": 300, "tags": self._get_schedule_tags(contract), @@ -480,7 +522,7 @@ def _plan_step_functions( actions = [] contract_id = contract.get("id", "unknown") - state_machine_name = f"fluid-workflow-{contract_id}" + state_machine_name = bounded_aws_name(f"fluid-workflow-{contract_id}", STATE_MACHINE_NAME) # Generate state machine definition from tasks state_machine_def = self._generate_state_machine_definition(contract, orchestration) @@ -518,6 +560,10 @@ def _generate_workflow_lambda_code( if t.get("type") == "provider_action" ] ) + # Contract values reach this source only as ``repr()`` literals (``!r``): + # spliced inside quotes, a task param or an id carrying a quote closed the + # literal and ran as code on every tick (or left a SyntaxError behind). + default_id = str(contract.get("id")) return { "ZipFile": f""" import json @@ -527,10 +573,10 @@ def _generate_workflow_lambda_code( logger = logging.getLogger() logger.setLevel(logging.INFO) -TASKS = json.loads('''{task_entries}''') +TASKS = json.loads({task_entries!r}) def handler(event, context): - contract_id = event.get('contract_id', '{contract.get("id")}') + contract_id = event.get('contract_id', {default_id!r}) logger.info(f"Executing FLUID workflow for contract: {{contract_id}}") completed = set() @@ -563,7 +609,7 @@ def _generate_s3_trigger_lambda_code( ) -> Dict[str, Any]: """Generate Lambda code for S3 trigger.""" contract_id = contract.get("id", "unknown") - workflow_fn = f"fluid-workflow-{contract_id}" + workflow_fn = self.workflow_function_name(contract_id) return { "ZipFile": f""" import json @@ -580,10 +626,10 @@ def handler(event, context): key = record['s3']['object']['key'] logger.info(f"S3 trigger: s3://{{bucket}}/{{key}}") lam.invoke( - FunctionName='{workflow_fn}', + FunctionName={workflow_fn!r}, InvocationType='Event', Payload=json.dumps({{ - 'contract_id': '{contract_id}', + 'contract_id': {contract_id!r}, 'trigger': 's3', 'bucket': bucket, 'key': key, @@ -598,7 +644,7 @@ def _generate_dynamodb_trigger_lambda_code( ) -> Dict[str, Any]: """Generate Lambda code for DynamoDB trigger.""" contract_id = contract.get("id", "unknown") - workflow_fn = f"fluid-workflow-{contract_id}" + workflow_fn = self.workflow_function_name(contract_id) return { "ZipFile": f""" import json @@ -614,10 +660,10 @@ def handler(event, context): event_name = record.get('eventName') logger.info(f"DynamoDB trigger: {{event_name}}") lam.invoke( - FunctionName='{workflow_fn}', + FunctionName={workflow_fn!r}, InvocationType='Event', Payload=json.dumps({{ - 'contract_id': '{contract_id}', + 'contract_id': {contract_id!r}, 'trigger': 'dynamodb', 'event_name': event_name, 'keys': record.get('dynamodb', {{}}).get('Keys', {{}}), @@ -632,7 +678,7 @@ def _generate_event_trigger_lambda_code( ) -> Dict[str, Any]: """Generate Lambda code for EventBridge trigger.""" contract_id = contract.get("id", "unknown") - workflow_fn = f"fluid-workflow-{contract_id}" + workflow_fn = self.workflow_function_name(contract_id) return { "ZipFile": f""" import json @@ -646,10 +692,10 @@ def handler(event, context): logger.info(f"EventBridge trigger: {{json.dumps(event)}}") lam = boto3.client('lambda') lam.invoke( - FunctionName='{workflow_fn}', + FunctionName={workflow_fn!r}, InvocationType='Event', Payload=json.dumps({{ - 'contract_id': '{contract_id}', + 'contract_id': {contract_id!r}, 'trigger': 'eventbridge', 'detail_type': event.get('detail-type', ''), 'detail': event.get('detail', {{}}), @@ -664,7 +710,7 @@ def _generate_sqs_trigger_lambda_code( ) -> Dict[str, Any]: """Generate Lambda code for SQS trigger.""" contract_id = contract.get("id", "unknown") - workflow_fn = f"fluid-workflow-{contract_id}" + workflow_fn = self.workflow_function_name(contract_id) return { "ZipFile": f""" import json @@ -680,10 +726,10 @@ def handler(event, context): body = json.loads(record['body']) logger.info(f"SQS trigger: {{body}}") lam.invoke( - FunctionName='{workflow_fn}', + FunctionName={workflow_fn!r}, InvocationType='Event', Payload=json.dumps({{ - 'contract_id': '{contract_id}', + 'contract_id': {contract_id!r}, 'trigger': 'sqs', 'body': body, }}), diff --git a/fluid_build/providers/aws/provider.py b/fluid_build/providers/aws/provider.py index 389cb2ae2..3cd4a2c08 100644 --- a/fluid_build/providers/aws/provider.py +++ b/fluid_build/providers/aws/provider.py @@ -25,7 +25,7 @@ import time from collections.abc import Mapping -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, NoReturn, Optional from fluid_build.providers.base import ( ApplyResult, @@ -166,6 +166,10 @@ def plan( Athena queries / workgroups, Redshift schemas, Lambda functions, EventBridge rules / schedules, IAM roles / policies. """ + # Imported here, not at module scope: this module is loaded by + # ``fluid --help``, whose module count is budgeted. + from fluid_build.providers._plan_validation import PlanValidationError + self.debug_kv( event="plan_started", contract_id=contract.get("id"), @@ -174,9 +178,13 @@ def plan( ) try: - # Validate sovereignty constraints (FLUID 0.7.1) + # Validate sovereignty constraints (FLUID 0.7.1). First, and with no + # account needed: this is the only place AWS enforces jurisdiction. self._validate_sovereignty(contract) + if not self.account_id: + self._no_plan_without_account(contract) + # ``plan_actions`` constructs an :class:`AwsPlanner` and # calls :meth:`BasePlanner.plan` which already runs the # 6-phase scaffold (infrastructure / IAM / replace-snapshots @@ -202,7 +210,7 @@ def plan( if not validation_result["valid"]: error_msg = "Plan validation failed:\n" + "\n".join(validation_result["errors"]) - raise ProviderError(error_msg) + raise PlanValidationError(error_msg) # Log warnings if any for warning in validation_result.get("warnings", []): @@ -316,6 +324,28 @@ def _validate_sovereignty(self, contract: Mapping[str, Any]) -> None: self.err_kv(event="sovereignty_violation", error=str(e)) raise ProviderError(str(e)) from e + def _no_plan_without_account(self, contract: Mapping[str, Any]) -> NoReturn: + """No account resolved, so no name can be derived: stop before one is. + + With ``None`` the planner builds ``None-fluid-staging`` and the validator + refuses the plan with a message about a bucket, hiding the cause. A + contract that declares orchestration (a schedule, triggers or tasks) + needs this plan, since its Lambda, permission and schedule come from + nowhere else, so the gap is refused by name. Any other contract's + resources come from its exposes, which tofu applies with its own + credentials: the planner reports that it could not run, and + ``native_actions`` treats that as best-effort, as before. ``generate + iac`` passes a placeholder account, so it never reaches this. + """ + from .util.config import AwsAccountUnresolvedError, account_unresolved_message + + orchestration = contract.get("orchestration") + if isinstance(orchestration, Mapping) and any( + orchestration.get(key) for key in ("schedule", "triggers", "tasks") + ): + raise AwsAccountUnresolvedError(account_unresolved_message()) + raise ProviderError(f"native AWS plan skipped: {account_unresolved_message()}") + def _plan_orchestration(self, contract: Mapping[str, Any]) -> List[Dict[str, Any]]: """ Plan orchestration tasks from contract (FLUID 0.7.1). @@ -327,6 +357,8 @@ def _plan_orchestration(self, contract: Mapping[str, Any]) -> List[Dict[str, Any if not orchestration: return [] + from fluid_build.providers._plan_validation import PlanValidationError + from .plan.orchestration import OrchestrationError, plan_orchestration_tasks try: @@ -343,7 +375,7 @@ def _plan_orchestration(self, contract: Mapping[str, Any]) -> List[Dict[str, Any except OrchestrationError as e: self.err_kv(event="orchestration_planning_failed", error=str(e)) - raise ProviderError(f"Orchestration planning failed: {e}") from e + raise PlanValidationError(f"Orchestration planning failed: {e}") from e def _plan_schedule(self, contract: Mapping[str, Any]) -> List[Dict[str, Any]]: """ @@ -360,11 +392,35 @@ def _plan_schedule(self, contract: Mapping[str, Any]) -> List[Dict[str, Any]]: if not orchestration.get("schedule") and not orchestration.get("triggers"): return [] - from .plan.schedule import plan_schedule_actions + from fluid_build.providers._plan_validation import PlanValidationError + + from .plan.schedule import ( + SCHEDULER_INVOKE_ROLE, + WORKFLOW_EXECUTION_ROLE, + plan_schedule_actions, + ) try: actions = plan_schedule_actions(contract, self.account_id, self.region, self.logger) + if any(a.get("op") == "eventbridge.ensure_schedule" for a in actions): + # The schedule fires a Lambda whose body only logs each task and + # records it ok; it runs no build. Its two roles are named by + # literal ARN and nothing here creates them. Say so on every + # plan that emits it, rather than let a green apply imply a + # working pipeline. + self.warn_kv( + event="schedule_placeholder_lambda", + message=( + "The scheduled Lambda is a placeholder: it logs each task and does " + "not run this product's builds. The IAM roles " + f"arn:aws:iam::{self.account_id}:role/{WORKFLOW_EXECUTION_ROLE} " + f"(Lambda) and arn:aws:iam::{self.account_id}:role/" + f"{SCHEDULER_INVOKE_ROLE} (EventBridge Scheduler) are referenced by " + "name and not created: they must already exist in the account." + ), + contract_id=contract.get("id"), + ) if actions: self.info_kv( event="schedule_planned", @@ -377,7 +433,7 @@ def _plan_schedule(self, contract: Mapping[str, Any]) -> List[Dict[str, Any]]: except Exception as e: self.err_kv(event="schedule_planning_failed", error=str(e)) - raise ProviderError(f"Schedule planning failed: {e}") from e + raise PlanValidationError(f"Schedule planning failed: {e}") from e def _utc_timestamp(self) -> str: """Generate UTC timestamp string.""" diff --git a/fluid_build/providers/aws/redshift_provider.py b/fluid_build/providers/aws/redshift_provider.py index 55595dc84..089de31d4 100644 --- a/fluid_build/providers/aws/redshift_provider.py +++ b/fluid_build/providers/aws/redshift_provider.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Standalone Redshift provider for rollback / DDL emission. +"""Standalone Redshift helper for rollback / DDL emission. The full AWS provider (``fluid_build.providers.aws.AwsProvider``) handles Redshift planning. Redshift's rollback semantics are SQL-DDL-based (``DROP`` + ``CREATE TABLE AS SELECT`` inside a transaction) while the rest of AWS (S3 / Glue / Athena) uses prefix-copy semantics. -Splitting Redshift out as its own provider class keeps the rollback +Splitting Redshift out as its own class keeps the rollback surface modular: ``cli/_rollback_writer.py::_delegate_to_provider`` asks each provider for its restore DDL by name, and Redshift owns its SQL while ``AwsProvider`` returns ``[]`` (S3 prefix-copy is non-DDL). @@ -32,9 +32,10 @@ two unrelated rollback semantics. One class per restore-semantics is cleaner than one class with two switches. -Registered as ``"redshift"`` in the provider registry so -``_delegate_to_provider("redshift", "restore_ddl", snapshot)`` succeeds -without falling through to the legacy lookup. +Not in the provider registry (``redshift`` is no longer a ``--provider``; +Redshift is part of ``aws``). +``_delegate_to_provider("redshift", "restore_ddl", snapshot)`` imports this +class directly, keyed by the snapshot's provider name. """ from __future__ import annotations @@ -46,7 +47,7 @@ class RedshiftProvider(BaseProvider): - """Thin Redshift provider — owns rollback semantics only. + """Thin Redshift helper — owns rollback semantics only. ``plan`` delegates to the full ``AwsProvider``; this class exists primarily so ``restore_ddl`` and ``cleanup_backups`` participate in diff --git a/fluid_build/providers/aws/util/config.py b/fluid_build/providers/aws/util/config.py index cde828f8d..0ac9fc5a9 100644 --- a/fluid_build/providers/aws/util/config.py +++ b/fluid_build/providers/aws/util/config.py @@ -17,9 +17,49 @@ AWS configuration resolution utilities. """ +import importlib.util import os from typing import Optional, Tuple +from fluid_build.providers._plan_validation import PlanValidationError + + +class AwsAccountUnresolvedError(PlanValidationError): + """No AWS account id could be resolved, so no resource name can be planned. + + A :class:`PlanValidationError` so ``native_actions`` re-raises it (matched by + type) rather than treating the planner as one that could not run: the + account feeds names (``{account}-fluid-data``) and every role ARN, and a + plan built on ``None`` is refused by the validator with a message about a + bucket name, not the cause. + """ + + +def _boto3_missing() -> bool: + """True when boto3 (the ``[aws]`` extra) is not installed.""" + return importlib.util.find_spec("boto3") is None + + +def account_unresolved_message() -> str: + """Why no account resolved, and what fixes it. + + ``resolve_account_and_region`` swallows the ``ImportError`` for a missing + boto3 along with every STS failure, so the cause is re-derived here: + blaming credentials for a package that is not installed sends the user to + the wrong place. + """ + if _boto3_missing(): + return ( + "The AWS account id could not be resolved: boto3 is not installed, so " + "sts:GetCallerIdentity cannot be called. Install it with " + "pip install 'data-product-forge[aws]', or set AWS_ACCOUNT_ID." + ) + return ( + "The AWS account id could not be resolved. Set AWS_ACCOUNT_ID, or " + "configure AWS credentials (environment, profile or role) so that " + "sts:GetCallerIdentity succeeds." + ) + def resolve_account_and_region( account_id: Optional[str] = None, region: Optional[str] = None diff --git a/fluid_build/providers/aws/util/dependencies.py b/fluid_build/providers/aws/util/dependencies.py index 3ebf351a9..89341ec5f 100644 --- a/fluid_build/providers/aws/util/dependencies.py +++ b/fluid_build/providers/aws/util/dependencies.py @@ -31,7 +31,12 @@ class DependencyGraph: def __init__(self): """Initialize empty dependency graph.""" - self.nodes: Set[str] = set() + # Insertion-ordered (a dict, not a set): ``topological_sort`` walks it, so a + # set made the action order, and with it the emitted module, differ between + # runs. The IaC emitter interpolates a Lambda's attributes only when the + # function was emitted first, so a schedule or permission planned for it + # came out as a literal ARN or a reference depending on the hash seed. + self.nodes: Dict[str, None] = {} self.edges: Dict[str, Set[str]] = defaultdict(set) # node -> dependencies self.action_map: Dict[str, Dict[str, Any]] = {} @@ -46,7 +51,7 @@ def add_action(self, action: Dict[str, Any]) -> str: Action ID """ action_id = action.get("id", self._generate_id(action)) - self.nodes.add(action_id) + self.nodes[action_id] = None self.action_map[action_id] = action return action_id diff --git a/fluid_build/providers/aws/util/names.py b/fluid_build/providers/aws/util/names.py index dbba04283..5c15c1f46 100644 --- a/fluid_build/providers/aws/util/names.py +++ b/fluid_build/providers/aws/util/names.py @@ -17,6 +17,7 @@ AWS resource name normalization utilities. """ +import hashlib import re from typing import Optional @@ -155,3 +156,59 @@ def normalize_lambda_name(name: str) -> str: # Truncate to 64 characters return normalized[:64] + + +#: Per-service naming rules for names the planner derives from a contract id. +#: Each is ``(characters NOT allowed, maximum length)``; pinned from the AWS API +#: references so the planner's output is valid before ``validate_actions`` sees +#: it: +#: +#: * Lambda function name, ``CreateFunction`` ``FunctionName``: letters, digits, +#: ``-`` and ``_``; 64 characters when only the name is given (the API's longer +#: limit applies to a full ARN). +#: https://docs.aws.amazon.com/lambda/latest/api/API_CreateFunction.html +#: * Lambda permission statement id, ``AddPermission`` ``StatementId``: letters, +#: digits, ``-`` and ``_``; 1-100 characters. +#: https://docs.aws.amazon.com/lambda/latest/api/API_AddPermission.html +#: * EventBridge Scheduler schedule name, ``CreateSchedule`` ``Name``: letters, +#: digits, ``-``, ``_`` and ``.``; 1-64 characters. +#: https://docs.aws.amazon.com/scheduler/latest/APIReference/API_CreateSchedule.html +#: * EventBridge rule name, ``PutRule`` ``Name``: letters, digits, ``-``, ``_`` +#: and ``.``; 1-64 characters. +#: https://docs.aws.amazon.com/eventbridge/latest/APIReference/API_PutRule.html +#: * MWAA environment name, ``CreateEnvironment`` ``Name``: starts with a letter, +#: then letters, digits, ``-`` and ``_``; 1-80 characters (every name built +#: here starts with the literal ``mwaa-``). +#: https://docs.aws.amazon.com/mwaa/latest/API/API_CreateEnvironment.html +#: * Step Functions state machine name, ``CreateStateMachine`` ``name``: 1-80 +#: characters, no whitespace, brackets, wildcards or ``" # % \ ^ | ~ ` $ & , ; : /``. +#: https://docs.aws.amazon.com/step-functions/latest/apireference/API_CreateStateMachine.html +LAMBDA_FUNCTION_NAME = (re.compile(r"[^a-zA-Z0-9\-_]"), 64) +LAMBDA_STATEMENT_ID = (re.compile(r"[^a-zA-Z0-9\-_]"), 100) +SCHEDULER_SCHEDULE_NAME = (re.compile(r"[^a-zA-Z0-9\-_.]"), 64) +EVENTBRIDGE_RULE_NAME = (re.compile(r"[^a-zA-Z0-9\-_.]"), 64) +MWAA_ENVIRONMENT_NAME = (re.compile(r"[^a-zA-Z0-9\-_]"), 80) +STATE_MACHINE_NAME = (re.compile(r"[\s<>{}\[\]?*\"#%\\^|~`$&,;:/\x00-\x1f\x7f-\x9f]"), 80) + +#: Hex digits of the digest appended to a name that had to be shortened. +_NAME_HASH_LEN = 8 + + +def bounded_aws_name(raw: str, rule: "tuple[re.Pattern[str], int]") -> str: + """``raw`` as a name ``rule`` accepts, deterministic and collision-safe. + + Characters outside the rule's allowed set become ``-``. A name that had to + change at all (a replacement, or a cut to the service limit) ends in + ``-``, the hash taken from the *whole raw* name: ``a.b`` and ``a-b`` + would otherwise both be ``fluid-workflow-a-b``, two products in one account + fighting over one Lambda, and two long ids sharing a prefix would collide + after the cut. A name the rule already accepts is returned as written, so + nothing already deployed under it is renamed. + """ + pattern, limit = rule + name = pattern.sub("-", raw) + if name == raw and len(name) <= limit: + return name + digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:_NAME_HASH_LEN] + keep = limit - _NAME_HASH_LEN - 1 + return f"{name[:keep]}-{digest}" diff --git a/fluid_build/providers/aws/util/validation.py b/fluid_build/providers/aws/util/validation.py index 2eb820e40..03ab32ee7 100644 --- a/fluid_build/providers/aws/util/validation.py +++ b/fluid_build/providers/aws/util/validation.py @@ -37,10 +37,10 @@ class ResourceValidator: """Validates AWS resource configurations before execution.""" # AWS naming rules - S3_BUCKET_PATTERN = re.compile(r"^[a-z0-9][a-z0-9\-]{1,61}[a-z0-9]$") - GLUE_DATABASE_PATTERN = re.compile(r"^[a-z0-9_]+$") - GLUE_TABLE_PATTERN = re.compile(r"^[a-z0-9_]+$") - LAMBDA_FUNCTION_PATTERN = re.compile(r"^[a-zA-Z0-9\-_]+$") + S3_BUCKET_PATTERN = re.compile(r"^[a-z0-9][a-z0-9.\-]{1,61}[a-z0-9]\Z") + GLUE_DATABASE_PATTERN = re.compile(r"^[a-z0-9_]+\Z") + GLUE_TABLE_PATTERN = re.compile(r"^[a-z0-9_]+\Z") + LAMBDA_FUNCTION_PATTERN = re.compile(r"^[a-zA-Z0-9\-_]+\Z") # AWS service quotas (defaults - can be increased) DEFAULT_QUOTAS = { @@ -69,9 +69,10 @@ def validate_s3_bucket_name(self, bucket: str) -> Tuple[bool, Optional[str]]: Rules: - 3-63 characters - - Lowercase letters, numbers, hyphens + - Lowercase letters, numbers, dots, hyphens (a dot-separated name is a + legal bucket name) - Must start and end with letter or number - - No consecutive hyphens + - No consecutive dots, no dot next to a hyphen, no consecutive hyphens - No IP address format Args: @@ -91,14 +92,17 @@ def validate_s3_bucket_name(self, bucket: str) -> Tuple[bool, Optional[str]]: if not self.S3_BUCKET_PATTERN.match(bucket): return False, ( - f"Bucket name '{bucket}' is invalid. " - "Must contain only lowercase letters, numbers, and hyphens. " + f"Bucket name {bucket!r} is invalid. " + "Must contain only lowercase letters, numbers, dots, and hyphens. " "Must start and end with letter or number." ) if "--" in bucket: return False, "Bucket name cannot contain consecutive hyphens" + if ".." in bucket or ".-" in bucket or "-." in bucket: + return False, "Bucket name cannot contain '..', '.-' or '-.'" + # Check for IP address format (e.g., 192.168.0.1) if re.match(r"^\d+\.\d+\.\d+\.\d+$", bucket): return False, "Bucket name cannot be formatted as an IP address" @@ -135,7 +139,7 @@ def validate_glue_database_name(self, database: str) -> Tuple[bool, Optional[str if not self.GLUE_DATABASE_PATTERN.match(database): return False, ( - f"Database name '{database}' is invalid. " + f"Database name {database!r} is invalid. " "Must contain only lowercase letters, numbers, and underscores." ) @@ -164,7 +168,7 @@ def validate_glue_table_name(self, table: str) -> Tuple[bool, Optional[str]]: if not self.GLUE_TABLE_PATTERN.match(table): return False, ( - f"Table name '{table}' is invalid. " + f"Table name {table!r} is invalid. " "Must contain only lowercase letters, numbers, and underscores." ) @@ -193,7 +197,7 @@ def validate_lambda_function_name(self, function: str) -> Tuple[bool, Optional[s if not self.LAMBDA_FUNCTION_PATTERN.match(function): return False, ( - f"Function name '{function}' is invalid. " + f"Function name {function!r} is invalid. " "Must contain only letters, numbers, hyphens, and underscores." ) diff --git a/fluid_build/providers/bigquery_validation.py b/fluid_build/providers/bigquery_validation.py index 4d6864e90..6482cb21c 100644 --- a/fluid_build/providers/bigquery_validation.py +++ b/fluid_build/providers/bigquery_validation.py @@ -43,7 +43,7 @@ # need a dedicated allowlist: 6–30 chars, lowercase-or-mixed letter start, # letters/digits/hyphens in the middle, and a letter/digit terminator. # See https://cloud.google.com/resource-manager/docs/creating-managing-projects. -_BQ_PROJECT_ID = re.compile(r"^[A-Za-z][A-Za-z0-9-]{4,28}[A-Za-z0-9]$") +_BQ_PROJECT_ID = re.compile(r"^[A-Za-z][A-Za-z0-9-]{4,28}[A-Za-z0-9]\Z") def _quote_bq_project(project: str) -> str: diff --git a/fluid_build/providers/datamesh_manager/__init__.py b/fluid_build/providers/datamesh_manager/__init__.py index a420a93f3..5ead50cf8 100644 --- a/fluid_build/providers/datamesh_manager/__init__.py +++ b/fluid_build/providers/datamesh_manager/__init__.py @@ -20,17 +20,11 @@ from fluid_build.providers.datamesh_manager.datamesh_manager import DataMeshManagerProvider -# Self-register with the provider registry so ``fluid providers`` lists us -# and ``discover_providers()`` picks us up automatically. -try: - from fluid_build.providers import register_provider +# Data Mesh Manager is a catalog, not a deployment provider: it is reached by +# ``fluid publish --target datamesh-manager`` and ``fluid datamesh-manager``, +# which import the class directly. Opt out of every passive registration +# (including the single-BaseProvider-subclass fallback) so it stays out of +# ``fluid providers``. +__fluid_no_autoregister__ = True - register_provider("datamesh-manager", DataMeshManagerProvider) -except Exception: - pass # Registry not yet available (e.g. during isolated import) - -# Expose NAME + Provider for auto-discovery strategy 2 as a fallback -NAME = "datamesh-manager" -Provider = DataMeshManagerProvider - -__all__ = ["DataMeshManagerProvider", "NAME", "Provider"] +__all__ = ["DataMeshManagerProvider"] diff --git a/fluid_build/providers/datamesh_manager/datamesh_manager.py b/fluid_build/providers/datamesh_manager/datamesh_manager.py index 10ff76643..522218fbe 100644 --- a/fluid_build/providers/datamesh_manager/datamesh_manager.py +++ b/fluid_build/providers/datamesh_manager/datamesh_manager.py @@ -54,25 +54,13 @@ ) if TYPE_CHECKING: - import requests as requests_typing + import httpx as httpx_typing - RequestsSession = requests_typing.Session - RequestsResponse = requests_typing.Response + HttpClient = httpx_typing.Client + HttpResponse = httpx_typing.Response else: - RequestsSession = Any - RequestsResponse = Any - -try: - import requests - from requests.adapters import HTTPAdapter - from urllib3.util.retry import Retry - - REQUESTS_AVAILABLE = True -except ImportError: # pragma: no cover - requests = None # type: ignore[assignment] - HTTPAdapter = None # type: ignore[assignment,misc] - Retry = None # type: ignore[assignment,misc] - REQUESTS_AVAILABLE = False + HttpClient = Any + HttpResponse = Any LOG = logging.getLogger(__name__) @@ -184,13 +172,7 @@ def __init__( default=False, ) - if not REQUESTS_AVAILABLE: - raise ProviderError( - "The 'requests' library is required for the Data Mesh Manager provider.\n" - "Install it with: pip install requests" - ) - - self._session_instance: Optional[RequestsSession] = None + self._session_instance: Optional[HttpClient] = None # ---- BaseProvider abstract methods ------------------------------------ @@ -1246,19 +1228,26 @@ def _derive_team_id(fluid: Mapping[str, Any]) -> str: # ---- HTTP helpers ----------------------------------------------------- - def _session(self) -> RequestsSession: + def _session(self) -> HttpClient: + # Follows redirects, honours the proxy environment and retries 429/5xx on + # GET/PUT/DELETE (not POST) with exponential backoff -- what the + # ``requests`` session with a urllib3 ``Retry`` did. See util/http_client. if self._session_instance is None: - s = requests.Session() - retry = Retry( - total=3, - backoff_factor=0.5, - status_forcelist=[429, 500, 502, 503, 504], - allowed_methods=["GET", "PUT", "DELETE"], + from fluid_build.observability.reporter import refuse_foreign_redirect + from fluid_build.util.http_client import new_client + + # Every request carries ``x-api-key``, which httpx (unlike + # ``Authorization``) keeps across origins: a redirect is followed + # only within the API's own origin. + self._session_instance = new_client( + response_hooks=[refuse_foreign_redirect], + retry={ + "total": 3, + "backoff_factor": 0.5, + "status_forcelist": [429, 500, 502, 503, 504], + "allowed_methods": ["GET", "PUT", "DELETE"], + }, ) - adapter = HTTPAdapter(max_retries=retry) - s.mount("https://", adapter) - s.mount("http://", adapter) - self._session_instance = s return self._session_instance def _headers(self) -> Dict[str, str]: @@ -1274,7 +1263,11 @@ def _request( path: str, *, json_body: Any = None, - ) -> RequestsResponse: + ) -> HttpResponse: + import httpx + + from fluid_build.observability.reporter import CredentialRedirectRefused + url = f"{self.api_url}{path}" self._log.debug("%s %s", method, url) try: @@ -1285,11 +1278,24 @@ def _request( json=json_body, timeout=_TIMEOUT, ) - except requests.ConnectionError as exc: + except CredentialRedirectRefused as exc: + # Not retried (the hook runs above the retry transport) and not + # followed: the key and body are for the configured origin only. + raise ProviderError( + f"Refused to follow a redirect from the Entropy Data API: {exc}. " + "The API key and request body were not sent there; if the API moved, " + "update DMM_API_URL." + ) from exc + except ( + httpx.NetworkError, + httpx.RemoteProtocolError, + httpx.ProxyError, + httpx.ConnectTimeout, # requests' ConnectTimeout was a ConnectionError first + ) as exc: raise ProviderError(f"Connection failed: {url} — {exc}") from exc - except requests.Timeout as exc: + except httpx.TimeoutException as exc: raise ProviderError(f"Request timed out: {url}") from exc - except requests.RequestException as exc: + except (httpx.HTTPError, httpx.InvalidURL) as exc: raise ProviderError(f"HTTP request failed: {url} — {exc}") from exc if resp.status_code >= 400: diff --git a/fluid_build/providers/gcp/plan/planner.py b/fluid_build/providers/gcp/plan/planner.py index a4d4ce62b..91883c8aa 100644 --- a/fluid_build/providers/gcp/plan/planner.py +++ b/fluid_build/providers/gcp/plan/planner.py @@ -37,7 +37,7 @@ # contain hyphens, so ``_sql_safety.validate_ident`` (which rejects ``-``) # cannot vet them — this dedicated check guards the project component of a # BigQuery fully-qualified name before it is interpolated into DDL/DML. -_GCP_PROJECT_RE = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$") +_GCP_PROJECT_RE = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]\Z") def _validate_bq_project(project: str) -> str: diff --git a/fluid_build/providers/gcp/util/names.py b/fluid_build/providers/gcp/util/names.py index 6a3f80fe5..c25db891b 100644 --- a/fluid_build/providers/gcp/util/names.py +++ b/fluid_build/providers/gcp/util/names.py @@ -324,7 +324,7 @@ def validate_name(name: str, resource_type: str) -> bool: try: if resource_type in ["dataset", "table"]: # BigQuery naming rules - if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", name): + if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*\Z", name): return False return len(name) <= 1024 @@ -332,7 +332,7 @@ def validate_name(name: str, resource_type: str) -> bool: # Cloud Storage naming rules if len(name) < 3 or len(name) > 63: return False - if not re.match(r"^[a-z0-9][a-z0-9\-]*[a-z0-9]$", name): + if not re.match(r"^[a-z0-9][a-z0-9\-]*[a-z0-9]\Z", name): return False if re.match(r"^\d+\.\d+\.\d+\.\d+$", name): return False @@ -344,7 +344,7 @@ def validate_name(name: str, resource_type: str) -> bool: return False if not name[0].isalpha(): return False - return re.match(r"^[a-zA-Z][a-zA-Z0-9\-\._~%+]*$", name) is not None + return re.match(r"^[a-zA-Z][a-zA-Z0-9\-\._~%+]*\Z", name) is not None elif resource_type == "job": # Job naming rules @@ -354,7 +354,7 @@ def validate_name(name: str, resource_type: str) -> bool: return False if name[-1] == "-": return False - return re.match(r"^[a-z][a-z0-9\-]*[a-z0-9]$", name) is not None + return re.match(r"^[a-z][a-z0-9\-]*[a-z0-9]\Z", name) is not None else: # Unknown resource type diff --git a/fluid_build/providers/local/mocks.py b/fluid_build/providers/local/mocks.py index c2957d81c..74cf134b2 100644 --- a/fluid_build/providers/local/mocks.py +++ b/fluid_build/providers/local/mocks.py @@ -163,13 +163,15 @@ def stop(self) -> Dict[str, Any]: def create_bucket(self, bucket_name: str) -> Dict[str, Any]: """Create GCS bucket in fake server.""" - import requests + import httpx url = f"{self.endpoint_url}/storage/v1/b" data = {"name": bucket_name} try: - response = requests.post(url, json=data) + # requests.post followed redirects and set no timeout; keep the + # redirects and bound the wait. + response = httpx.post(url, json=data, follow_redirects=True, timeout=30) return {"success": response.status_code in [200, 201], "bucket": bucket_name} except Exception as e: self.logger.error(f"Failed to create bucket: {e}") diff --git a/fluid_build/providers/odcs/io.py b/fluid_build/providers/odcs/io.py index 8a9b68ea3..2938a5e83 100644 --- a/fluid_build/providers/odcs/io.py +++ b/fluid_build/providers/odcs/io.py @@ -40,6 +40,38 @@ LOG = logging.getLogger(__name__) +#: Fallback kinds already reported in this process (one WARNING per kind). +_FALLBACK_WARNED: set = set() + + +def _warn_fallback_once(kind: str, path: Any, exc: BaseException) -> None: + """One WARNING per process when the typed ODCS library path is abandoned. + + ``open-data-contract-standard`` is a base dependency, not an extra, so an + ``ImportError`` here means a broken install and the hint is a reinstall. + Any other failure is the library rejecting this file: the raw YAML is used + and the typed validation it would have done is skipped. + """ + if kind in _FALLBACK_WARNED: + return + _FALLBACK_WARNED.add(kind) + if isinstance(exc, ImportError): + LOG.warning( + "open-data-contract-standard could not be imported, so ODCS files are %s as plain " + "YAML with no typed validation. Reinstall it with: " + "pip install --force-reinstall 'open-data-contract-standard>=3.1.2'", + "read" if kind == "read" else "written", + ) + return + LOG.warning( + "open-data-contract-standard rejected %s (%s: %s); falling back to plain YAML %s " + "without typed validation. Shown once per process.", + path, + type(exc).__name__, + str(exc).splitlines()[0][:200] if str(exc) else "", + "parsing" if kind == "read" else "output", + ) + def read_input(path: Union[str, Path]) -> Dict[str, Any]: """Load an ODCS YAML/JSON file. @@ -69,11 +101,7 @@ def read_input(path: Union[str, Path]) -> Dict[str, Any]: # author never set. return dc.model_dump(by_alias=True, exclude_none=True) except Exception as exc: # noqa: BLE001 — soft-fail to raw parse - LOG.debug( - "OpenDataContractStandard.from_file failed on %s; falling back to raw YAML: %s", - input_path, - exc, - ) + _warn_fallback_once("read", input_path, exc) return raw @@ -99,11 +127,7 @@ def write_output(data: Dict[str, Any], path: Union[Path, str], fmt: str) -> None output_path.write_text(dc.to_yaml(), encoding="utf-8") return except Exception as exc: # noqa: BLE001 — soft-fail to raw dump - LOG.debug( - "OpenDataContractStandard.to_yaml failed for %s; falling back to raw YAML dump: %s", - output_path, - exc, - ) + _warn_fallback_once("write", output_path, exc) with open(output_path, "w", encoding="utf-8") as f: if fmt == "yaml": diff --git a/fluid_build/providers/odcs/mappers/normalize.py b/fluid_build/providers/odcs/mappers/normalize.py index 94ff451dc..b7e028deb 100644 --- a/fluid_build/providers/odcs/mappers/normalize.py +++ b/fluid_build/providers/odcs/mappers/normalize.py @@ -462,7 +462,7 @@ def _root_identifier(raw: Any, version: str) -> tuple[str, Optional[str]]: return _IMPORTED_ID_FALLBACK, None text = str(raw) pattern = _identifier_pattern(version) - if pattern is None or pattern.match(text): + if pattern is None or pattern.fullmatch(text): return text, None slug = _slugged_root_id(text) # %r: the id is document text and may carry control characters. diff --git a/fluid_build/providers/odps_standard/provider.py b/fluid_build/providers/odps_standard/provider.py index e03e17edd..c03a07c50 100644 --- a/fluid_build/providers/odps_standard/provider.py +++ b/fluid_build/providers/odps_standard/provider.py @@ -1141,7 +1141,7 @@ def _schema_defs(version: str) -> Mapping[str, Any]: def _is_semver(value: Any, version: str) -> bool: pattern = _semver_pattern(version) - return pattern is None or bool(pattern.match(str(value))) + return pattern is None or bool(pattern.fullmatch(str(value))) def _rehydrate_document(fluid: Mapping[str, Any]) -> Mapping[str, Any]: diff --git a/fluid_build/providers/quality_engine.py b/fluid_build/providers/quality_engine.py index d4ba6191c..a3f8741d7 100644 --- a/fluid_build/providers/quality_engine.py +++ b/fluid_build/providers/quality_engine.py @@ -41,7 +41,7 @@ LOG = logging.getLogger("fluid.providers.quality_engine") # Safe SQL identifier regex (letters, digits, underscores only) -_SAFE_IDENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_SAFE_IDENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*\Z") def _validate_ident(name: str) -> str: diff --git a/fluid_build/providers/snowflake/util/config.py b/fluid_build/providers/snowflake/util/config.py index 6a44ade67..e2fc67f9f 100644 --- a/fluid_build/providers/snowflake/util/config.py +++ b/fluid_build/providers/snowflake/util/config.py @@ -36,8 +36,8 @@ # share the canonical pattern without redefining or importing a private name. ENV_TEMPLATE_RE = _ENV_TEMPLATE_RE _ACCOUNT_HOST_RE = re.compile(r"\.snowflakecomputing\.com$", re.IGNORECASE) -_ACCOUNT_SEGMENT_RE = re.compile(r"^[A-Za-z0-9_-]+$") -_ACCOUNT_REGION_SEGMENT_RE = re.compile(r"^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)+$") +_ACCOUNT_SEGMENT_RE = re.compile(r"^[A-Za-z0-9_-]+\Z") +_ACCOUNT_REGION_SEGMENT_RE = re.compile(r"^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)+\Z") _ACCOUNT_CLOUD_SUFFIXES = frozenset({"aws", "azure", "gcp"}) # Keys that carry secret material. These MUST never appear in the `_sources` diff --git a/fluid_build/schedulers/airflow/fluid_apply.py b/fluid_build/schedulers/airflow/fluid_apply.py index 7a38496fe..5c38c18bd 100644 --- a/fluid_build/schedulers/airflow/fluid_apply.py +++ b/fluid_build/schedulers/airflow/fluid_apply.py @@ -487,11 +487,11 @@ def validate_contract_path(raw: Any) -> str: def validate_id(raw: Any, *, kind: str) -> str: - """:func:`validate_identifier`, refusing a trailing newline as well. + """:func:`validate_identifier` as a :class:`ScheduleRenderError`, with its own newline check. - The shared grammar ends in ``$``, which also matches before a final - ``\\n``; an id or env that carries one names nothing fluid can find - (``--env 'aws\\n'`` silently applies no overlay). + An id or env that ends in a newline names nothing fluid can find (``--env + 'aws\\n'`` silently applies no overlay). The shared grammar ends in + ``\\Z`` and refuses one already; the check below still holds if it changes. """ try: value = validate_identifier(raw, kind=kind) diff --git a/fluid_build/schema_manager.py b/fluid_build/schema_manager.py index 4a8144ea4..a157506fb 100644 --- a/fluid_build/schema_manager.py +++ b/fluid_build/schema_manager.py @@ -21,13 +21,14 @@ from __future__ import annotations +import functools import json import logging import re from dataclasses import dataclass, field from datetime import datetime, timedelta from pathlib import Path -from typing import Any, Callable, Dict, FrozenSet, List, Optional, Union +from typing import Any, Callable, Dict, FrozenSet, Iterator, List, Optional, Union import yaml @@ -110,6 +111,7 @@ def _locate_error(error: Any, locate: Callable[..., Optional[str]]) -> Optional[ known = schema.get("properties") or {} patterns = list((schema.get("patternProperties") or {}).keys()) for key in error.instance: + # re.search, as jsonschema's additionalProperties reads these keys. if key not in known and not any(re.search(p, str(key)) for p in patterns): return locate([*path, key]) if error.validator == "required": @@ -132,6 +134,72 @@ def _error_path(error: Any, locate: Optional[Callable[..., Optional[str]]] = Non return f"{where} ({path})" if where else path +def _end_anchored(pattern: str) -> str: + r"""``pattern`` with each ``$`` anchor written ``\Z``, the end of the value. + + JSON Schema writes ``pattern`` in ECMA-262, where ``$`` matches only at the + end of the value. Python's ``$`` also matches before a final ``\n``, so + jsonschema's ``re.search`` let ``"orders\n"`` through ``^[a-z_]+$``. + fastjsonschema and js-regex make the same rewrite with a ``(? re.Pattern[str]: + return re.compile(_end_anchored(pattern)) + + +def _pattern_keyword(validator: Any, pattern: str, instance: Any, schema: Any) -> Iterator[Any]: + """jsonschema's ``pattern`` keyword, with ``$`` read as ECMA-262 reads it.""" + if not validator.is_type(instance, "string") or _compiled_pattern(pattern).search(instance): + return + message = f"{instance!r} does not match {pattern!r}" + if re.search(pattern, instance): # only the final line break fails + message += " (it ends in a line break, which a YAML | or > block keeps and |- or >- drops)" + yield jsonschema.ValidationError(message) + + +@functools.lru_cache(maxsize=None) +def _end_anchored_validator(validator_class: Any) -> Any: + return jsonschema.validators.extend(validator_class, {"pattern": _pattern_keyword}) + + +def schema_validator(schema: Dict[str, Any]) -> Any: + """A jsonschema validator for ``schema``, in the dialect it declares, whose + ``pattern`` ends at the end of the value (:func:`_end_anchored`). Validate + against a FLUID schema through this, never ``validator_for`` directly. + + The keyword is replaced the way check-jsonschema replaces it, with + ``jsonschema.validators.extend``. Two places keep jsonschema's own + ``re.search``: ``patternProperties`` keys, as its ``additionalProperties`` + and :func:`_locate_error` read them, and a subschema that declares its own + ``$schema`` (or a ``$ref`` to the root, which does), for which jsonschema + picks that dialect's own class. No FLUID schema has either; a test fails if + one does. + """ + return _end_anchored_validator(jsonschema.validators.validator_for(schema))(schema) + + def _schema_version_sort_key(version: str) -> tuple[int, int, int]: match = VERSION_PATTERN.match(version) if not match: @@ -764,7 +832,7 @@ def _validate_with_jsonschema( # 2020-12; validating them with Draft 7 worked only because none # has yet used a 2020-12-only keyword. The first one to do so # would have been accepted without error. - validator = jsonschema.validators.validator_for(schema)(schema) + validator = schema_validator(schema) # Validate and collect errors errors = sorted(validator.iter_errors(contract), key=lambda e: e.path) diff --git a/fluid_build/util/contract.py b/fluid_build/util/contract.py index 31bceda67..0db30e1f5 100644 --- a/fluid_build/util/contract.py +++ b/fluid_build/util/contract.py @@ -202,6 +202,21 @@ def is_databricks_job_build(build: Mapping[str, Any]) -> bool: return isinstance(platform, str) and platform.strip().lower() == "databricks" +def is_duckdb_acquisition_build(build: Mapping[str, Any]) -> bool: + """True when forge-cli's DuckDB acquisition runner runs the build: ``pattern: + acquisition`` and ``engine: duckdb``. That runner applies ``policy.privacy.masking`` to + the rows it lands, before they leave the machine (``build_runners/_masking.py``) and + lands a Databricks expose in its table (``build_runners/_databricks_load.py``).""" + pattern = build.get("pattern") + engine = get_build_engine(build) + return ( + isinstance(pattern, str) + and pattern.strip().lower() == "acquisition" + and isinstance(engine, str) + and engine.strip().lower() == "duckdb" + ) + + def is_unscheduled_build(build: Mapping[str, Any]) -> bool: """True when no scheduler starts the build's runs: its job runs outside forge-cli (:func:`is_externally_run_build`), continuously @@ -1062,3 +1077,37 @@ def slugify_identifier(value: str, *, fallback: str = "project") -> str: if slug[0].isdigit(): slug = f"x-{slug}" return slug + + +# ── Column keys ──────────────────────────────────────────────────────────── +# Which columns are keys, read the same way wherever it matters: the dbt engine +# gives these a ``primary_key`` constraint (``engines/dbt/schema_yml.py``), and +# ``fluid diff`` reports one that becomes nullable as breaking. + +_PK_KEYS = ("primary", "primaryKey", "primary_key", "pk", "isPrimary") + + +def is_truthy(value: Any) -> bool: + """Loose truthiness for YAML/JSON booleans-as-strings (``"true"``/``True``).""" + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in ("true", "1", "yes") + return bool(value) + + +def column_is_key(col: Mapping[str, Any]) -> bool: + """True when the column is declared a uniqueness key / primary key.""" + if is_truthy(col.get("unique")): + return True + if any(is_truthy(col.get(k)) for k in _PK_KEYS): + return True + if str(col.get("semanticType") or "").strip().lower() in ("identifier", "primary_key"): + return True + labels = col.get("labels") + if isinstance(labels, Mapping): + if is_truthy(labels.get("unique")): + return True + if str(labels.get("constraint") or "").strip().lower() in ("primary_key", "unique"): + return True + return False diff --git a/fluid_build/util/http_client.py b/fluid_build/util/http_client.py new file mode 100644 index 000000000..6964cfd95 --- /dev/null +++ b/fluid_build/util/http_client.py @@ -0,0 +1,347 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The one plain ``httpx`` client factory for callers that used ``requests``. + +This is NOT the SSRF-guarded client (:func:`fluid_build.util.safe_http.safe_httpx_client` +-- DNS pinning, private-address refusal). The sites that use this module +carry their own host gates (the Command Center host gate, the operator-named +registry rule, a fixed vendor API URL) and must keep reaching loopback and +internal hosts; they only needed the transport swapped from ``requests``. + +What it keeps from ``requests`` so behaviour does not change for users: + +* **Redirects are followed** (``requests`` follows by default; ``httpx`` does + not) up to 30 hops (``requests.Session.max_redirects``; ``httpx`` defaults to + 20). Callers that must stop a redirect do so in a ``response`` event hook, + which ``httpx`` runs on every hop before following it. +* **Environment proxies** (``HTTP(S)_PROXY``, ``NO_PROXY``) -- ``trust_env``. +* **A custom CA bundle** from ``REQUESTS_CA_BUNDLE`` / ``CURL_CA_BUNDLE``, + which ``requests`` honoured and ``httpx`` does not (it reads + ``SSL_CERT_FILE`` / ``SSL_CERT_DIR``). Verification stays ON in every case. +* **Retries** with ``urllib3.util.retry.Retry`` semantics, as + :class:`RetryTransport`. + +Borrow-before-build receipts: ``httpx.HTTPTransport(retries=N)`` retries +connection failures only (never a status code, never a read error), so it +cannot stand in for ``Retry(status_forcelist=...)``. The ``httpx-retries`` +package (MIT, will-ockmore/httpx-retries) is the maintained transport-wrapper +answer, but it is another base dependency in a change whose point is fewer; +this module adapts its shape (a transport wrapper with a urllib3-style +``Retry``) in ~80 lines. ``tenacity`` was weighed and not taken for the same +reason, and ``build_runners/catalog_registrars/_http_retry.py`` is an +operation-level loop living above this layer (``util`` must not import +``build_runners``), with different semantics (4 attempts, jitter). + +``httpx`` is imported at module scope: import this module lazily, from the +function that makes the request, so the cold ``fluid --help`` path stays free +of it. +""" + +from __future__ import annotations + +import email.utils +import os +import ssl +import time +from datetime import datetime, timezone +from typing import Any, Callable, Iterable, Mapping, Optional, Union + +import httpx + +#: ``requests.Session.max_redirects``. +MAX_REDIRECTS = 30 + +#: urllib3 ``Retry.DEFAULT_BACKOFF_MAX``. +_BACKOFF_MAX = 120.0 + +#: urllib3 ``Retry.RETRY_AFTER_STATUS_CODES``: the only statuses whose +#: ``Retry-After`` header is honoured. +_RETRY_AFTER_STATUSES = frozenset({413, 429, 503}) + +_IDEMPOTENT_METHODS = frozenset({"HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"}) + + +class RetryError(httpx.TransportError): + """Retries ran out on a retryable status (``requests.exceptions.RetryError``). + + ``requests`` did not hand back the last 5xx/429 response once + ``Retry(status_forcelist=...)`` was exhausted; it raised. This keeps that. + """ + + +class CaBundleError(OSError): + """``REQUESTS_CA_BUNDLE`` / ``CURL_CA_BUNDLE`` names a path that cannot be loaded. + + An ``OSError``, as ``requests`` raised it (per request); the text names the + variable. Verification is never switched off or swapped for certifi to get + past it. + """ + + +def tls_verify() -> Union[bool, ssl.SSLContext]: + """The ``verify=`` value ``requests`` would have used, for ``httpx``. + + ``True`` (certifi) unless ``REQUESTS_CA_BUNDLE`` or ``CURL_CA_BUNDLE`` names + a file or directory, which ``requests`` used in place of its default. Never + ``False``: nothing here switches verification off. + """ + name = "REQUESTS_CA_BUNDLE" if os.environ.get("REQUESTS_CA_BUNDLE") else "CURL_CA_BUNDLE" + bundle = os.environ.get(name) + if not bundle: + return True + try: + if os.path.isdir(bundle): + return ssl.create_default_context(capath=bundle) + return ssl.create_default_context(cafile=bundle) + except (OSError, ssl.SSLError) as exc: + raise CaBundleError( + f"{name} is set to {bundle!r}, which cannot be loaded as a CA bundle ({exc})" + ) from exc + + +def merge_query(url: str, params: Optional[Mapping[str, Any]]) -> str: + """``url`` with ``params`` merged into its own query string, as ``requests`` did. + + ``httpx`` replaces a URL's query with ``params=``; ``requests`` kept the + URL's own pairs and appended the new ones. Empty ``params`` leave ``url`` + untouched (``httpx`` would erase its query). + """ + if not params: + return str(url) + return str(httpx.URL(str(url)).copy_merge_params(params)) + + +def parse_retry_after(raw: Optional[str]) -> Optional[float]: + """Seconds from a ``Retry-After`` value: delta-seconds or an HTTP-date (RFC 9110). + + ``None`` when absent, negative or unparseable. Shared with + ``build_runners.catalog_registrars._http_retry``. + """ + if not raw: + return None + raw = raw.strip() + try: + secs = float(int(raw)) + return secs if secs >= 0 else None + except OverflowError: + # An integer too large for a float (hundreds of digits): unusable. + return None + except ValueError: + pass # not a number of seconds; try the HTTP-date form below + try: + parsed = email.utils.parsedate_to_datetime(raw) + except (ValueError, TypeError, OverflowError): + return None + if parsed is None: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + try: + return max(0.0, (parsed - datetime.now(timezone.utc)).total_seconds()) + except OverflowError: + return None + + +class RetryTransport(httpx.BaseTransport): + """``urllib3.util.retry.Retry`` semantics around an ``httpx`` transport. + + * connect failures (``ConnectError`` / ``ConnectTimeout``) are retried for + every method -- nothing was sent; + * read-side failures (``ReadError``, ``ReadTimeout``, ``RemoteProtocolError``, + ``WriteError``) are retried only for ``allowed_methods`` -- the request + may have been processed; + * a response whose status is in ``status_forcelist`` is retried for + ``allowed_methods``; when retries run out :class:`RetryError` is raised; + * the sleep between attempts is ``backoff_factor * 2 ** (n - 1)`` (none + before the first retry, capped at 120s), or the ``Retry-After`` header on + 413/429/503, whichever the response carries. + + ``total`` is the number of retries, so ``total + 1`` attempts at most. + """ + + def __init__( + self, + *, + total: int = 3, + backoff_factor: float = 0.0, + status_forcelist: Iterable[int] = (), + allowed_methods: Iterable[str] = _IDEMPOTENT_METHODS, + transport: Optional[httpx.BaseTransport] = None, + sleep: Callable[[float], None] = time.sleep, + ) -> None: + self.total = total + self.backoff_factor = backoff_factor + self.status_forcelist = frozenset(status_forcelist) + self.allowed_methods = frozenset(m.upper() for m in allowed_methods) + self._sleep = sleep + self._transport = transport or httpx.HTTPTransport() + + def _backoff(self, retry_number: int) -> float: + if retry_number <= 1: + return 0.0 + return min(_BACKOFF_MAX, self.backoff_factor * (2 ** (retry_number - 1))) + + def _delay_after_error( + self, exc: httpx.TransportError, replayable: bool, retries: int + ) -> float: + """The sleep before retrying after ``exc``; re-raises ``exc`` when it is final.""" + connect_failure = isinstance(exc, (httpx.ConnectError, httpx.ConnectTimeout)) + read_failure = isinstance( + exc, + (httpx.ReadError, httpx.ReadTimeout, httpx.WriteError, httpx.RemoteProtocolError), + ) + retryable = connect_failure or (read_failure and replayable) + if not retryable or retries >= self.total: + raise exc + return self._backoff(retries + 1) + + def _delay_after_response( + self, request: httpx.Request, response: httpx.Response, replayable: bool, retries: int + ) -> Optional[float]: + """The sleep before retrying after ``response``; ``None`` to hand it back.""" + if not replayable or response.status_code not in self.status_forcelist: + return None + if retries >= self.total: + response.close() + raise RetryError(f"too many {response.status_code} error responses", request=request) + delay = self._backoff(retries + 1) + if response.status_code in _RETRY_AFTER_STATUSES: + advertised = parse_retry_after(response.headers.get("Retry-After")) + if advertised is not None: + # Capped like the backoff (httpx-retries' ``max_backoff_wait``): a + # server does not get to park the CLI for days. + delay = min(_BACKOFF_MAX, advertised) + response.close() + return delay + + def handle_request(self, request: httpx.Request) -> httpx.Response: + replayable = request.method.upper() in self.allowed_methods + retries = 0 + while True: + try: + response = self._transport.handle_request(request) + except httpx.TransportError as exc: + delay = self._delay_after_error(exc, replayable, retries) + else: + maybe_delay = self._delay_after_response(request, response, replayable, retries) + if maybe_delay is None: + return response + delay = maybe_delay + retries += 1 + if delay > 0: + self._sleep(delay) + + def close(self) -> None: + self._transport.close() + + +class _Client(httpx.Client): + """``httpx.Client`` whose every transport -- direct and per-proxy -- retries. + + A ``transport=`` argument would make ``httpx`` ignore the proxy environment + (``allow_env_proxies = trust_env and transport is None``), so the retry + wrapper is applied at the two factory methods ``httpx`` builds transports + through instead. + """ + + def __init__(self, *, retry: Optional[Mapping[str, Any]] = None, **kwargs: Any) -> None: + self._retry_options = dict(retry) if retry is not None else None + super().__init__(**kwargs) + + def _wrap(self, transport: httpx.BaseTransport) -> httpx.BaseTransport: + if self._retry_options is None: + return transport + return RetryTransport(transport=transport, **self._retry_options) + + def _init_transport(self, *args: Any, **kwargs: Any) -> httpx.BaseTransport: + return self._wrap(super()._init_transport(*args, **kwargs)) + + def _init_proxy_transport(self, *args: Any, **kwargs: Any) -> httpx.BaseTransport: + return self._wrap(super()._init_proxy_transport(*args, **kwargs)) + + +def new_client( + *, + timeout: Union[float, httpx.Timeout, None] = None, + headers: Optional[Mapping[str, str]] = None, + response_hooks: Iterable[Callable[[httpx.Response], Any]] = (), + retry: Optional[Mapping[str, Any]] = None, + follow_redirects: bool = True, + **kwargs: Any, +) -> httpx.Client: + """An ``httpx.Client`` that behaves like a ``requests`` session. + + Follows redirects (``follow_redirects=False`` where the site did not), + trusts the proxy environment, verifies TLS (:func:`tls_verify`), and runs + each ``response_hooks`` entry on every hop before it is followed. ``retry`` + is the keyword arguments of :class:`RetryTransport` (``total``, + ``backoff_factor``, ``status_forcelist``, ``allowed_methods``, ``sleep``). + """ + return _Client( + retry=retry, + timeout=timeout, + headers=dict(headers) if headers else None, + follow_redirects=follow_redirects, + max_redirects=MAX_REDIRECTS, + verify=tls_verify(), + event_hooks={"response": list(response_hooks)}, + trust_env=True, + **kwargs, + ) + + +def new_async_client( + *, + timeout: Union[float, httpx.Timeout, None] = None, + response_hooks: Iterable[Callable[[httpx.Response], Any]] = (), + follow_redirects: bool = True, + max_redirects: int = MAX_REDIRECTS, + trust_env: bool = True, + system_trust: bool = False, + **kwargs: Any, +) -> httpx.AsyncClient: + """The async twin of :func:`new_client` (``httpx.AsyncClient``). + + ``response_hooks`` stay plain functions: they are wrapped in the coroutine + ``httpx`` requires of an async client's event hooks, so one hook (for + example ``refuse_foreign_redirect``) serves the sync and async clients. + + ``system_trust=True`` verifies against the system CA store + (``ssl.create_default_context()``, what ``aiohttp`` used) instead of + certifi / ``REQUESTS_CA_BUNDLE``, for a caller that must keep trusting a + corporate CA installed only system-wide. Verification stays on. + """ + + def _as_async(hook: Callable[[httpx.Response], Any]) -> Callable[[httpx.Response], Any]: + async def _hook(response: httpx.Response) -> None: + try: + hook(response) + except Exception: + await response.aclose() + raise + + return _hook + + return httpx.AsyncClient( + timeout=timeout, + follow_redirects=follow_redirects, + max_redirects=max_redirects, + verify=( + ssl.create_default_context() if system_trust else tls_verify() if trust_env else True + ), + event_hooks={"response": [_as_async(h) for h in response_hooks]}, + trust_env=trust_env, + **kwargs, + ) diff --git a/fluid_build/util/network.py b/fluid_build/util/network.py index 35fd8ad2f..96e63f5f1 100644 --- a/fluid_build/util/network.py +++ b/fluid_build/util/network.py @@ -35,18 +35,6 @@ if TYPE_CHECKING: pass -try: - import requests as requests_module - from requests.adapters import HTTPAdapter - from urllib3.util.retry import Retry - - HAS_REQUESTS = True -except ImportError: - HAS_REQUESTS = False - requests_module = None # type: ignore - HTTPAdapter = None # type: ignore - Retry = None # type: ignore - from ..errors import NetworkError, wrap_error logger = logging.getLogger(__name__) @@ -194,35 +182,41 @@ def create_session_with_retries( max_retries: int = DEFAULT_MAX_RETRIES, backoff_factor: float = DEFAULT_BACKOFF_FACTOR, status_forcelist: tuple = (500, 502, 503, 504, 429), -) -> Any: # returns requests.Session +) -> Any: # returns httpx.Client """ - Create requests session with automatic retry configuration. + Create an httpx client with automatic retry configuration. + + Redirects are followed and the proxy environment is honoured, as the + ``requests`` session this replaces did. Retries follow + ``urllib3.util.retry.Retry`` semantics (see ``util.http_client``). Args: max_retries: Maximum number of retries - backoff_factor: Backoff multiplier (delay = backoff_factor * (2 ** retry_count)) + backoff_factor: Backoff multiplier (delay = backoff_factor * (2 ** (retry_count - 1)), + no delay before the first retry) status_forcelist: HTTP status codes that trigger retry Returns: - Configured requests.Session + Configured httpx.Client """ - if not HAS_REQUESTS: - raise ImportError("requests library not installed") - - session = requests_module.Session() - - retry_strategy = Retry( - total=max_retries, - backoff_factor=backoff_factor, - status_forcelist=status_forcelist, - allowed_methods=["HEAD", "GET", "PUT", "POST", "DELETE", "OPTIONS", "TRACE"], + from .http_client import new_client + + return new_client( + retry={ + "total": max_retries, + "backoff_factor": backoff_factor, + "status_forcelist": status_forcelist, + "allowed_methods": ["HEAD", "GET", "PUT", "POST", "DELETE", "OPTIONS", "TRACE"], + } ) - adapter = HTTPAdapter(max_retries=retry_strategy) - session.mount("http://", adapter) - session.mount("https://", adapter) - return session +_REQUESTS_ONLY_KWARGS = { + "verify": "pass the CA bundle through the client (REQUESTS_CA_BUNDLE / CURL_CA_BUNDLE " + "or util.http_client.new_client) -- verify= is a client option in httpx", + "cert": "client certificates are a client option in httpx (new_client(cert=...))", + "stream": "use session.stream(...) for a streamed response in httpx", +} def safe_request( @@ -234,7 +228,7 @@ def safe_request( session: Optional[Any] = None, circuit_breaker: Optional[CircuitBreaker] = None, **kwargs, -) -> Any: # returns requests.Response +) -> Any: # returns httpx.Response """ Make HTTP request with safety features. @@ -243,9 +237,9 @@ def safe_request( url: Request URL timeout: Read timeout in seconds connect_timeout: Connection timeout in seconds - session: Optional requests session (creates one with retries if not provided) + session: Optional httpx client (creates one with retries if not provided) circuit_breaker: Optional circuit breaker - **kwargs: Additional arguments passed to requests + **kwargs: Additional arguments passed to httpx Returns: Response object @@ -253,8 +247,7 @@ def safe_request( Raises: NetworkError: If request fails after retries """ - if not HAS_REQUESTS: - raise ImportError("requests library not installed") + import httpx # Set timeouts if timeout is None: @@ -262,15 +255,23 @@ def safe_request( if connect_timeout is None: connect_timeout = DEFAULT_CONNECT_TIMEOUT - # Use tuple (connect_timeout, read_timeout) - timeout_tuple = (connect_timeout, timeout) + # connect_timeout bounds the connection, timeout the read (and write/pool) + request_timeout = httpx.Timeout(timeout, connect=connect_timeout) # Create session if not provided if session is None: session = create_session_with_retries() + # ``requests`` keyword names that ``httpx`` spells differently: translate the + # one that maps one-to-one, refuse the rest rather than ignore them silently. + if "allow_redirects" in kwargs: + kwargs["follow_redirects"] = kwargs.pop("allow_redirects") + for name, instead in _REQUESTS_ONLY_KWARGS.items(): + if name in kwargs: + raise TypeError(f"safe_request() no longer takes {name!r}: {instead}") + # Add timeout to kwargs - kwargs["timeout"] = timeout_tuple + kwargs["timeout"] = request_timeout # Execute request try: @@ -279,12 +280,15 @@ def safe_request( else: response = session.request(method, url, **kwargs) - # Raise for HTTP errors - response.raise_for_status() + # Raise for 4xx/5xx only, as ``requests`` did: ``httpx``'s + # ``raise_for_status()`` also raises on a 1xx/3xx (a 304, a 301 with no + # ``Location``). + if response.is_error: + response.raise_for_status() return response - except requests_module.exceptions.Timeout as e: + except httpx.TimeoutException as e: raise NetworkError( f"Request timed out after {timeout}s: {url}", context={"url": url, "method": method, "timeout": timeout}, @@ -296,7 +300,7 @@ def safe_request( original_error=e, ) - except requests_module.exceptions.ConnectionError as e: + except (httpx.NetworkError, httpx.RemoteProtocolError, httpx.ProxyError) as e: raise NetworkError( f"Connection failed: {url}", context={"url": url, "method": method}, @@ -308,8 +312,8 @@ def safe_request( original_error=e, ) - except requests_module.exceptions.HTTPError as e: - status_code = e.response.status_code if e.response else None + except httpx.HTTPStatusError as e: + status_code = e.response.status_code raise NetworkError( f"HTTP {status_code} error: {url}", context={"url": url, "method": method, "status_code": status_code}, @@ -331,38 +335,38 @@ def safe_request( # Convenience functions -def safe_get(url: str, **kwargs) -> Any: # returns requests.Response +def safe_get(url: str, **kwargs) -> Any: # returns httpx.Response """Safe GET request with timeouts and retries""" return safe_request("GET", url, **kwargs) -def safe_post(url: str, **kwargs) -> Any: # returns requests.Response +def safe_post(url: str, **kwargs) -> Any: # returns httpx.Response """Safe POST request with timeouts and retries""" return safe_request("POST", url, **kwargs) -def safe_put(url: str, **kwargs) -> Any: # returns requests.Response +def safe_put(url: str, **kwargs) -> Any: # returns httpx.Response """Safe PUT request with timeouts and retries""" return safe_request("PUT", url, **kwargs) -def safe_patch(url: str, **kwargs) -> Any: # returns requests.Response +def safe_patch(url: str, **kwargs) -> Any: # returns httpx.Response """Safe PATCH request with timeouts and retries""" return safe_request("PATCH", url, **kwargs) -def safe_delete(url: str, **kwargs) -> Any: # returns requests.Response +def safe_delete(url: str, **kwargs) -> Any: # returns httpx.Response """Safe DELETE request with timeouts and retries""" return safe_request("DELETE", url, **kwargs) # Global instances for common use -_default_session: Optional[Any] = None # requests.Session +_default_session: Optional[Any] = None # httpx.Client _default_rate_limiter: Optional[RateLimiter] = None _default_circuit_breaker: Optional[CircuitBreaker] = None -def get_default_session() -> Any: # returns requests.Session +def get_default_session() -> Any: # returns httpx.Client """Get or create default session""" global _default_session if _default_session is None: diff --git a/pyproject.toml b/pyproject.toml index 811b06331..16b8c2c74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,9 +44,8 @@ dependencies = [ "pydantic>=2.7", "typing-extensions>=4.8", "jinja2>=3.1.6", # security: >=3.1.6 closes CVE-2025-27516 (sandbox escape via |attr) - "httpx>=0.27", # For FluidCommandCenter catalog provider + "httpx>=0.27", # The one HTTP client: Command Center, Data Mesh Manager, marketplace, reporter "h11>=0.16", # security: pin transitive h11 >=0.16 to close CVE-2025-43859 (chunked request smuggling) - "requests>=2.32.5", # For Data Mesh Manager catalog publishing and shared HTTP helpers # LiteLLM is the canonical LLM backend (was an opt-in extra). It # replaces ~1,300 lines of per-provider wire-format code with one # unified API across openai / anthropic / gemini / bedrock / azure / @@ -160,11 +159,12 @@ dependencies = [ # thread per instance, which leaked unboundedly with a per-session # SessionState. Reintroduce a backend here only if a multi-replica # deployment needs shared limits.) - # Borrowed SQL parser (MIT) used by fluid_build.providers._sql_safety to - # AST-classify statement bodies for the Snowflake action surfaces, so a - # contract cannot smuggle account-level / multi-statement SQL through the - # custom-SQL / task-body / view-body surfaces. Borrowed-not-built per - # /borrow-before-build: https://github.com/tobymao/sqlglot + # Borrowed SQL parser (MIT). Real consumers: fluid_build/iac/row_access.py + # (parses and validates a row-filter predicate before it is written into a + # platform policy; it FAILS OPEN if sqlglot is missing, so keep this in the + # base install), iac/providers/databricks_access.py, forge/core/validators.py + # and the from_ddl parser. (providers/_sql_safety.py does not use it.) + # Borrowed-not-built per /borrow-before-build: https://github.com/tobymao/sqlglot "sqlglot>=25.0", # Bitol-vendor-listed Python library for ODCS v3.1.0. Provides typed # Pydantic v2 models + ``from_file()`` / ``to_yaml()`` for lossless @@ -347,16 +347,47 @@ postgres = [ "psycopg[binary]>=3.1", ] -# LiteLLM is now a core dependency (above), not an extra. The -# ``[litellm]`` alias is kept empty for back-compat with anyone who -# scripted ``pip install data-product-forge[litellm]`` during the opt-in -# preview window. -litellm = [] +# AI authoring (litellm). litellm is still in the base install today, so this +# extra is redundant for now; it exists so that pip/uv installs of +# ``data-product-forge[litellm]`` keep working after litellm leaves the base +# install in 0.28.0. The two lines are copied verbatim from ``dependencies``: +# the Python-version marker split (PR #555) must never be collapsed, because +# litellm >= 1.98 cannot be imported on Python 3.10. +# tests/ci_guards/extras_alongside_base.py pins both lines. +litellm = [ + "litellm>=1.83.14,<2; python_version >= '3.11'", + "litellm>=1.83.14,<1.98; python_version < '3.11'", +] + +# MCP server and output-port gateway (``fluid mcp serve``, +# ``fluid mcp output-port serve``). mcp is still in the base install today; +# this extra is redundant for now and carries the install after 0.28.0. +# pyjwt is declared explicitly because fluid_build/output_ports/mcp/auth.py +# imports ``jwt`` directly. Floor 2.15.0: every release up to 2.14.0 has +# advisories on the PyJWKClient/decode path auth.py uses (e.g. +# GHSA-ffc3-869f-jxw9); [crypto] for the RS256/ES256/EdDSA verifiers. +mcp = [ + "mcp>=1.22,<3.0", + "pyjwt[crypto]>=2.15.0", +] -# mcp is in the base install today. This empty alias lets -# ``pip install 'data-product-forge[mcp]'`` resolve now; it gets filled -# when mcp leaves the base install. -mcp = [] +# OpenLineage run-event emission. openlineage-python is still in the base +# install today; this extra is redundant for now and carries the install +# after it leaves base. +lineage = [ + "openlineage-python>=1.52", +] + +# AI + MCP together: restores today's base behaviour once litellm and mcp +# leave the base install in 0.28.0 (the ``fastapi[standard]`` precedent). +standard = [ + "data-product-forge[litellm,mcp]", +] + +# Alias of ``[litellm]`` for readers who look for an ``ai`` extra. +ai = [ + "data-product-forge[litellm]", +] # V1.5 metadata-source catalog adapters. # @@ -443,6 +474,9 @@ dev = [ # the unit lanes install ``.[dev,local]``, which has no pyarrow, so without # it those tests would skip in CI instead of running. "pyarrow>=14", + # litellm, mcp and openlineage leave the base install in 0.28.0; the tests + # import all three at module level, so dev names them explicitly. + "data-product-forge[litellm,mcp,lineage]", ] # Light (in-process) integration-test emulators — Stage 1. moto mocks the @@ -484,7 +518,7 @@ test-emulators-heavy = [ # One-shot install for contributors all = [ - "data-product-forge[local,aws,gcp,snowflake,viz,odps,tui,copilot,vector,postgres,docs,dev]" + "data-product-forge[local,aws,gcp,snowflake,viz,odps,tui,copilot,vector,postgres,docs,litellm,mcp,lineage,dev]" ] [project.scripts] @@ -529,6 +563,21 @@ snowflake = "fluid_build.providers.snowflake:SnowflakeProvider" # Built-ins are seeded in code, so this group is declared by THIRD-PARTY # packages only (listing the built-ins here would self-shadow). +# OpenTofu emitters for an additional platform are a discovered plugin group +# (``iac/registry.py``). A third-party package ships one so that installing it +# makes ``binding.platform: `` compile through ``fluid generate iac`` and +# apply through the OpenTofu engine, with no core edit: +# +# [project.entry-points."fluid_build.iac_providers"] +# onprem = "my_pkg.iac:OnPremIacPlugin" # an IacProviderPlugin class/instance +# +# Loaded lazily, by name, when a contract's platform needs it (never at import, +# so ``fluid --help`` / ``fluid providers`` run no plugin code). Governed by +# FLUID_PLUGINS_ALLOWLIST / FLUID_PLUGINS_BLOCKLIST and the ``requires_cli`` gate. +# Built-in platform names (aws / gcp / snowflake / confluent / databricks, their +# aliases, local) are reserved: an entry point claiming one is skipped before it +# is imported. Declared by THIRD-PARTY packages only. + [project.urls] Homepage = "https://agenticstiger.github.io/forge_docs/" Repository = "https://github.com/Agenticstiger/forge-cli" diff --git a/scripts/mcp_output_port_certify.py b/scripts/mcp_output_port_certify.py index 9fb93dd79..705db9ae1 100755 --- a/scripts/mcp_output_port_certify.py +++ b/scripts/mcp_output_port_certify.py @@ -178,10 +178,20 @@ def _check_direct_jsonrpc( """ import asyncio - from mcp import ClientSession, StdioServerParameters - from mcp.client.stdio import stdio_client + try: + from mcp import ClientSession, StdioServerParameters + from mcp.client.stdio import stdio_client - from fluid_build._mcp_compat import attr, self_attesting_client_kwargs + from fluid_build._mcp_compat import attr, self_attesting_client_kwargs + except ImportError as exc: + return CertificationCheck( + name="direct_jsonrpc", + status="fail", + detail=( + f"MCP SDK not importable ({exc}). Install it with: " + "pip install 'data-product-forge[mcp]'" + ), + ) cmd = _server_command(contract_path, expose_id=expose_id) server_params = StdioServerParameters( diff --git a/tests/build_runners/test_consumes_resolution.py b/tests/build_runners/test_consumes_resolution.py index 7b6157a4b..5d9de688f 100644 --- a/tests/build_runners/test_consumes_resolution.py +++ b/tests/build_runners/test_consumes_resolution.py @@ -931,7 +931,7 @@ def s3_endpoint(monkeypatch) -> Iterator[str]: import sys import time - import requests + import httpx port = _free_port() endpoint = f"http://127.0.0.1:{port}" @@ -944,9 +944,9 @@ def s3_endpoint(monkeypatch) -> Iterator[str]: deadline = time.monotonic() + 30 while True: try: - requests.get(f"{endpoint}/moto-api/", timeout=1) + httpx.get(f"{endpoint}/moto-api/", timeout=1) break - except requests.RequestException: + except httpx.RequestError: if proc.poll() is not None or time.monotonic() > deadline: pytest.skip("moto server did not start") time.sleep(0.2) diff --git a/tests/build_runners/test_duckdb_databricks_load.py b/tests/build_runners/test_duckdb_databricks_load.py new file mode 100644 index 000000000..daed68e6b --- /dev/null +++ b/tests/build_runners/test_duckdb_databricks_load.py @@ -0,0 +1,717 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Databricks binding is built locally, uploaded to a volume and loaded into the managed table. + +What these pin, each of which was wrong before: for ``platform: databricks`` the acquisition +runner wrote a LOCAL file (``location.path``, else ``out/``) and reported success with nothing +in Databricks. Now the run's Parquet stages under ``.fluid/staging``, goes to +``$DATABRICKS_STAGING_VOLUME`` through the Files API, one ``INSERT OVERWRITE ... read_files`` +on ``$DATABRICKS_WAREHOUSE_ID`` replaces the table's rows, and the staged files are deleted +whether the load worked or not. + +The workspace is ``tests/iac/_fake_databricks.py``, an in-process stand-in: no Databricks +account, credential or network is involved. These tests need no ``databricks-sdk``: the client is +the stand-in's own ``api_client.do`` over HTTP. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import time +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional + +import pytest + +from fluid_build.build_runners import _databricks_load as load + +# The iac-tests job collects every file with no duckdb installed. +duckdb = pytest.importorskip("duckdb") +from fluid_build.build_runners import _masking as masking # noqa: E402 +from fluid_build.build_runners.duckdb.runner import execute_duckdb_build # noqa: E402 + +from ..iac._fake_databricks import FakeDatabricks # noqa: E402 + +pytestmark = pytest.mark.unit + +VOLUME = "/Volumes/main/staging/landing" +TABLE = "main.bronze.subs" +SALT = "landing-test-salt-0123456789" +COLUMNS = [("id", "string", True), ("msisdn", "string", False), ("zip", "string", False)] +ROWS = [("1", "+46701234567", "11122"), ("2", "+46709876543", "22233"), ("3", "", "33344")] + + +@pytest.fixture +def fake() -> Iterator[FakeDatabricks]: + server = FakeDatabricks().start() + server.add_volume("main.staging.landing") + server.create_table(TABLE, COLUMNS) + try: + yield server + finally: + server.stop() + + +@pytest.fixture(autouse=True) +def workspace(monkeypatch, fake: FakeDatabricks) -> None: + """The stand-in as the only workspace the loader can reach, and its secrets as env.""" + for name in ("DATABRICKS_HOST", "DATABRICKS_TOKEN", "DATABRICKS_CONFIG_PROFILE"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("DATABRICKS_WAREHOUSE_ID", fake.warehouse_id) + monkeypatch.setenv(load.STAGING_VOLUME_ENV, VOLUME) + monkeypatch.setenv(masking.DEFAULT_SALT_ENV, SALT) + monkeypatch.setattr(load, "default_client", fake.client) + + +# ── the staging volume: one environment variable, three identifiers ────── + + +@pytest.mark.parametrize("value", [VOLUME, VOLUME + "/", "/Volumes/Main_1/_s/v2"]) +def test_a_staging_volume_is_three_identifiers_under_volumes(value): + volume = load.staging_volume({load.STAGING_VOLUME_ENV: value}) + assert (volume.catalog, volume.schema, volume.volume) == tuple(value.strip("/").split("/")[1:]) + assert volume.path == value.rstrip("/") + + +@pytest.mark.parametrize( + "value", + [ + "", + " ", + "/Volumes/main/staging", + "/Volumes/main/staging/landing/extra", + "main.staging.landing", + "/volumes/main/staging/landing", + "/Volumes/main/../landing", + "/Volumes/ma-in/staging/landing", + "/Volumes/main/stag ing/landing", + "/Volumes/main/staging/`landing`", + "/Volumes/main/staging/landing;DROP", + "/Volumes//staging/landing", + "dbfs:/Volumes/main/staging/landing", + ], +) +def test_a_staging_volume_that_is_not_one_is_refused_by_name(value): + with pytest.raises(load.DatabricksLoadError) as exc: + load.staging_volume({load.STAGING_VOLUME_ENV: value}) + assert load.STAGING_VOLUME_ENV in str(exc.value) + assert "/Volumes///" in str(exc.value) + + +def test_an_unset_staging_volume_is_refused_by_name(): + with pytest.raises(load.DatabricksLoadError) as exc: + load.staging_volume({}) + assert load.STAGING_VOLUME_ENV in str(exc.value) and "not set" in str(exc.value) + + +def test_a_run_stages_under_its_product_build_and_run(): + volume = load.staging_volume({load.STAGING_VOLUME_ENV: VOLUME}) + prefix = load.run_prefix(volume, "bronze.subs", "ingest", "run-20261103-abc") + assert prefix == f"{VOLUME}/bronze_subs/ingest/run-20261103-abc" + + +@pytest.mark.parametrize("hostile", ["../../x", "..", "a/b", "a\\b", "a'b", "a b", "", "é"]) +def test_a_contract_supplied_id_cannot_climb_out_of_the_volume(hostile): + volume = load.staging_volume({load.STAGING_VOLUME_ENV: VOLUME}) + for prefix in ( + load.run_prefix(volume, hostile, "b", "r"), + load.run_prefix(volume, "p", hostile, "r"), + load.run_prefix(volume, "p", "b", hostile), + ): + assert prefix.startswith(VOLUME + "/") + tail = prefix[len(VOLUME) + 1 :].split("/") + assert len(tail) == 3 and all(tail) and ".." not in tail + assert all( + set(part) <= set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-") + for part in tail + ) + + +# ── the target and the statement ───────────────────────────────────────── + + +def _expose(**location: Any) -> Dict[str, Any]: + loc = {"database": "Main", "schema": "bronze", "table": "subs", **location} + return { + "exposeId": "subs", + "kind": "table", + "binding": {"platform": "databricks", "format": "delta_table", "location": loc}, + "contract": { + # varchar, not string: the run-time schema gate compares the declared type with + # DuckDB's own (VARCHAR) and reads "string" as a type change; the emitter maps both + # to string + "schema": [ + {"name": "id", "type": "varchar"}, + {"name": "msisdn", "type": "varchar"}, + {"name": "zip", "type": "varchar"}, + ], + }, + } + + +def test_the_target_is_the_table_apply_made_with_its_columns_in_order(): + expose = _expose() + target = load.databricks_load_target(expose["binding"], expose) + assert target.full_name == "main.bronze.subs" # lower-cased, as the emitter creates it + assert target.columns == ("id", "msisdn", "zip") + + +@pytest.mark.parametrize("platform", ["local", "gcp", "aws", "snowflake", ""]) +def test_another_platform_has_no_databricks_target(platform): + expose = _expose() + expose["binding"]["platform"] = platform + assert load.databricks_load_target(expose["binding"], expose) is None + + +def test_the_insert_overwrite_statement_is_exact(): + expose = _expose() + target = load.databricks_load_target(expose["binding"], expose) + assert load.insert_overwrite_statement(target, f"{VOLUME}/bronze_subs/ingest/run-1") == ( + "INSERT OVERWRITE `main`.`bronze`.`subs` (`id`, `msisdn`, `zip`) SELECT `id`, `msisdn`, `zip` " + f"FROM read_files('{VOLUME}/bronze_subs/ingest/run-1', format => 'parquet')" + ) + + +def test_the_path_is_a_spark_string_literal_with_backslash_escapes(): + expose = _expose() + target = load.databricks_load_target(expose["binding"], expose) + sql = load.insert_overwrite_statement(target, "/Volumes/a/b/c/it's\\x") + assert sql.endswith("read_files('/Volumes/a/b/c/it\\'s\\\\x', format => 'parquet')") + + +@pytest.mark.parametrize( + "mutate, fragment", + [ + (lambda e: e["binding"]["location"].update(table="subs`; DROP TABLE x"), "table"), + (lambda e: e["binding"]["location"].update(database="a.b"), "database"), + (lambda e: e["binding"]["location"].pop("schema"), "schema"), + (lambda e: e["contract"].update(schema=[]), "contract.schema"), + (lambda e: e["contract"]["schema"].append({"name": "ID", "type": "string"}), "twice"), + ( + lambda e: e["contract"]["schema"].append({"name": "x`) FROM t --", "type": "string"}), + "name", + ), + ], +) +def test_an_unsafe_or_incomplete_binding_is_refused_before_any_statement(mutate, fragment): + expose = _expose() + mutate(expose) + with pytest.raises(load.DatabricksLoadError) as exc: + load.databricks_load_target(expose["binding"], expose) + assert fragment in str(exc.value) + + +@pytest.mark.parametrize( + "mode, sink, streams, fragment", + [ + ("full_refresh", "parquet", 2, "exactly one stream"), + ("full_refresh", "csv", 1, "sink format"), + ("incremental_append", "parquet", 1, "full_refresh"), + ("cdc", "parquet", 1, "full_refresh"), + ], +) +def test_unsupported_shapes_are_named(mode, sink, streams, fragment): + assert fragment in load.unsupported_reason(mode, sink, streams) + + +def test_a_full_refresh_of_one_parquet_stream_is_supported(): + assert load.unsupported_reason("full_refresh", "parquet", 1) is None + + +# ── upload, load, delete: the landing itself ───────────────────────────── + + +def _parquet(tmp_path: Path, rows=ROWS, name: str = "subs.parquet") -> Path: + path = tmp_path / name + con = duckdb.connect() + try: + values = ", ".join("(" + ", ".join(repr(v) for v in row) + ")" for row in rows) + con.execute( + f"COPY (SELECT * FROM (VALUES {values}) AS t(id, msisdn, zip)) TO '{path}' (FORMAT parquet)" + ) + finally: + con.close() + return path + + +def _landing(fake: FakeDatabricks) -> "load.Landing": + expose = _expose() + target = load.databricks_load_target(expose["binding"], expose) + return load.open_landing(target) + + +def _load(landing, path: Path, *, expected: int = 3, run_id: str = "run-1") -> Dict[str, Any]: + return landing.load( + str(path), + product_id="bronze.subs", + build_id="ingest", + run_id=run_id, + expected_rows=expected, + logger=logging.getLogger("test"), + ) + + +def test_the_file_is_uploaded_loaded_and_the_staged_copy_deleted(fake, tmp_path): + facts = _load(_landing(fake), _parquet(tmp_path)) + + assert [r["id"] for r in fake.rows(TABLE)] == ["1", "2", "3"] + assert facts["table"] == TABLE and facts["rows"] == 3 + assert facts["staged_files_deleted"] is True + run_dir = f"{VOLUME}/bronze_subs/ingest/run-1" + assert facts["staged_to"] == run_dir + assert fake.volume_files() == [], "never leave landed data in the volume" + puts = [p for m, p in fake.file_ops if m == "PUT"] + assert puts == [f"{run_dir}/subs.parquet"] + # in order: earlier runs looked for, the directory, the file, then (after the load) the file + # and the directory go + assert [m for m, _ in fake.file_ops] == ["LIST", "PUT-DIR", "PUT", "DELETE", "DELETE-DIR"] + assert fake.sql_log[-1].startswith("INSERT OVERWRITE `main`.`bronze`.`subs`") + assert fake.unknown == [] + + +def test_the_uploaded_bytes_are_the_files_bytes(fake, tmp_path): + path = _parquet(tmp_path) + fake.fail_on("INSERT OVERWRITE") # the load fails, so only the upload is looked at + landing = _landing(fake) + seen: List[bytes] = [] + real = fake._fs_put + + def spy(m, q, b): + seen.append(b.get("_raw")) + return real(m, q, b) + + fake.routes = [(mth, rx, spy if fn == real else fn) for mth, rx, fn in fake.routes] + with pytest.raises(load.DatabricksLoadError): + _load(landing, path) + assert seen == [path.read_bytes()] + + +def test_a_second_load_replaces_the_rows_it_does_not_append(fake, tmp_path): + landing = _landing(fake) + _load(landing, _parquet(tmp_path), run_id="run-1") + newer = _parquet(tmp_path, rows=[("9", "+1", "99999")], name="later.parquet") + _load(landing, newer, expected=1, run_id="run-2") + assert [r["id"] for r in fake.rows(TABLE)] == ["9"] + assert fake.volume_files() == [] + + +def test_the_load_keeps_the_table_s_definition_row_filter_and_masks(fake, tmp_path): + fake.tables[TABLE]["row_filter"] = { + "function_name": "main.bronze.fluid_rf_x", + "input_column_names": ["id"], + } + fake.tables[TABLE]["columns"][1]["mask"] = {"function_name": "main.bronze.fluid_mask_y"} + before = json.dumps(fake.tables[TABLE], sort_keys=True) + _load(_landing(fake), _parquet(tmp_path)) + assert json.dumps(fake.tables[TABLE], sort_keys=True) == before + assert not [ + s for s in fake.sql_log if s.lstrip().upper().startswith(("CREATE", "ALTER", "DROP")) + ] + assert not [s for s in fake.sql_log if "REPLACE TABLE" in s.upper()], "INSERT OVERWRITE only" + + +def test_a_missing_table_names_fluid_apply_and_uploads_nothing(fake, tmp_path): + del fake.tables[TABLE] + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + assert "fluid apply" in str(exc.value) and TABLE in str(exc.value) + assert fake.file_ops == [], "nothing was uploaded for a table that is not there" + assert not [s for s in fake.sql_log if "INSERT" in s] + assert TABLE not in fake.tables, "the loader never creates the table" + + +def test_a_failed_load_still_deletes_the_staged_files(fake, tmp_path): + fake.fail_on("INSERT OVERWRITE", "[INSUFFICIENT_PERMISSIONS] no MODIFY on the table") + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + assert "INSUFFICIENT_PERMISSIONS" in str(exc.value) + assert fake.volume_files() == [] + assert "deleted" in str(exc.value) + assert fake.rows(TABLE) == [] + + +def test_a_short_load_fails_the_build_and_deletes_the_staged_files(fake, tmp_path): + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path), expected=4) + assert f"loaded 3 rows into {TABLE}, but the landed file holds 4" in str(exc.value) + assert fake.volume_files() == [] + + +def test_a_load_that_reports_no_count_is_counted_in_the_table(fake, tmp_path): + fake.insert_reports_count = False + facts = _load(_landing(fake), _parquet(tmp_path)) + assert facts["rows"] == 3 and facts["rows_from"] == "count_after_load" + assert any("COUNT" in s.upper() for s in fake.sql_log) + + +def test_a_count_that_fails_after_the_load_says_the_load_ran(fake, tmp_path): + fake.insert_reports_count = False + fake.fail_on("COUNT(", "[INTERNAL_ERROR] no count") + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + text = str(exc.value) + assert f"loaded {TABLE}, but counting its rows afterwards failed" in text + assert "loading" not in text.split("but")[0] + assert fake.volume_files() == [] + + +def test_a_staged_file_that_cannot_be_deleted_fails_the_run_naming_it(fake, tmp_path): + fake.fail_route("DELETE", r"/fs/files/", 403, "no permission to delete") + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + text = str(exc.value) + assert "could not delete" in text and "subs.parquet" in text + assert [r["id"] for r in fake.rows(TABLE)] == ["1", "2", "3"], "the load itself worked" + assert fake.volume_files() != [], "and the file is still there: the error says so" + + +def test_a_failed_upload_is_cleaned_up_and_nothing_loads(fake, tmp_path): + fake.fail_route("PUT", r"/fs/files/", 403, "PERMISSION_DENIED: no WRITE VOLUME") + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + assert "PERMISSION_DENIED" in str(exc.value) + assert not [s for s in fake.sql_log if "INSERT" in s] + assert fake.volume_files() == [] + + +def test_the_load_puts_each_column_in_the_column_of_its_name(fake, tmp_path): + # fluid apply appended `id` to a live table that keeps the other two first: by position, + # the msisdn values would land in `id` + cols = fake.tables[TABLE]["columns"] + fake.tables[TABLE]["columns"] = cols[1:] + cols[:1] + _load(_landing(fake), _parquet(tmp_path)) + assert [r["id"] for r in fake.rows(TABLE)] == ["1", "2", "3"] + assert fake.rows(TABLE)[0]["msisdn"] == ROWS[0][1] + + +def _volume_file( + fake: FakeDatabricks, volume_path: str, data: bytes, *, age_seconds: float = 0 +) -> None: + real = fake._fs_path(volume_path) + assert real is not None, volume_path + Path(real).parent.mkdir(parents=True, exist_ok=True) + Path(real).write_bytes(data) + if age_seconds: + when = time.time() - age_seconds + os.utime(real, (when, when)) + + +_OLD = load.STALE_STAGING_SECONDS + 60 +#: A run id as the acquisition runner makes one (``generate_run_id``). +EARLIER_RUN = "01" + "0" * 15 + "Z" + + +def test_the_load_reads_the_uploaded_file_not_the_run_directory(fake, tmp_path): + run_dir = f"{VOLUME}/bronze_subs/ingest/run-1" + # someone with WRITE VOLUME drops a file into the run's directory before the INSERT + planted = _parquet(tmp_path, rows=[("666", "+0", "00000")], name="planted.parquet") + _volume_file(fake, f"{run_dir}/planted.parquet", planted.read_bytes()) + + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + + insert = [s for s in fake.sql_log if s.startswith("INSERT OVERWRITE")] + assert insert and insert[0].endswith( + f"read_files('{run_dir}/subs.parquet', format => 'parquet')" + ) + assert [r["id"] for r in fake.rows(TABLE)] == ["1", "2", "3"], "the planted rows are not loaded" + # the directory cannot be removed while the planted file is in it: the run says so + assert "could not delete" in str(exc.value) and run_dir in str(exc.value) + + +def test_an_interrupt_mid_load_still_deletes_the_staged_files(fake, tmp_path, monkeypatch): + landing = _landing(fake) + + def interrupted(statement): + raise KeyboardInterrupt + + monkeypatch.setattr(landing._sql, "execute", interrupted) + with pytest.raises(KeyboardInterrupt): + _load(landing, _parquet(tmp_path)) + assert fake.volume_files() == [], "Ctrl-C must not leave landed data in the volume" + assert [m for m, _ in fake.file_ops][-2:] == ["DELETE", "DELETE-DIR"] + + +def test_an_interrupted_statement_is_cancelled_on_the_warehouse(): + from fluid_build.iac.providers.databricks_enforce import SqlRunner + + calls: List[tuple] = [] + + class _Api: + def do(self, method, path, **kwargs): + calls.append((method, path)) + if method == "POST" and path.endswith("/statements/"): + return {"statement_id": "s-1", "status": {"state": "RUNNING"}} + if method == "GET": + raise KeyboardInterrupt + return {} + + class _Client: + api_client = _Api() + + runner = SqlRunner(_Client(), "wh", sleep=lambda _s: None) + with pytest.raises(KeyboardInterrupt): + runner.execute("INSERT OVERWRITE `a`.`b`.`c` SELECT 1") + assert ("POST", "/api/2.0/sql/statements/s-1/cancel") in calls + + +def test_staged_files_an_earlier_killed_run_left_are_deleted_first(fake, tmp_path): + stale = f"{VOLUME}/bronze_subs/ingest/{EARLIER_RUN}" + other_build = f"{VOLUME}/bronze_subs/other/{EARLIER_RUN}" + _volume_file(fake, f"{stale}/subs.parquet", b"left by a run that was killed", age_seconds=_OLD) + _volume_file(fake, f"{other_build}/subs.parquet", b"another build's run", age_seconds=_OLD) + + facts = _load(_landing(fake), _parquet(tmp_path), run_id="run-1") + + assert facts["earlier_runs_staging_deleted"] == [stale] + assert fake.volume_files() == [f"{other_build}/subs.parquet"], "only this build's directory" + assert [r["id"] for r in fake.rows(TABLE)] == ["1", "2", "3"] + + +def test_a_recent_run_of_the_same_build_keeps_its_staged_files(fake, tmp_path): + # another run of this build, uploaded a minute ago, may still be loading + running = f"{VOLUME}/bronze_subs/ingest/{EARLIER_RUN}" + _volume_file(fake, f"{running}/subs.parquet", b"in flight", age_seconds=60) + + facts = _load(_landing(fake), _parquet(tmp_path), run_id="run-1") + + assert facts["earlier_runs_staging_deleted"] == [] + assert fake.volume_files() == [f"{running}/subs.parquet"] + + +@pytest.mark.parametrize( + "path", + [ + "{volume}/bronze_subs/ingest/2024/part-0.parquet", # not a run directory's name + "{volume}/bronze_subs/ingest/{run}/notes.txt", # not what a load stages + "{volume}/bronze_subs/ingest/{run}/nested/part-0.parquet", # a subdirectory + ], +) +def test_the_sweep_leaves_what_a_load_did_not_stage(fake, tmp_path, path): + planted = path.format(volume=VOLUME, run=EARLIER_RUN) + _volume_file(fake, planted, b"someone else's data", age_seconds=_OLD) + + facts = _load(_landing(fake), _parquet(tmp_path), run_id="run-1") + + assert facts["earlier_runs_staging_deleted"] == [] + assert fake.volume_files() == [planted] + + +def test_a_stale_run_that_cannot_be_deleted_does_not_stop_the_load(fake, tmp_path): + stale = f"{VOLUME}/bronze_subs/ingest/{EARLIER_RUN}" + _volume_file(fake, f"{stale}/subs.parquet", b"left behind", age_seconds=_OLD) + fake.fail_route( + "DELETE", rf"/fs/files/Volumes/.*/ingest/{EARLIER_RUN}/", 403, "PERMISSION_DENIED" + ) + facts = _load(_landing(fake), _parquet(tmp_path), run_id="run-1") + assert facts["earlier_runs_staging_deleted"] == [] + assert fake.volume_files() == [f"{stale}/subs.parquet"] + assert [r["id"] for r in fake.rows(TABLE)] == ["1", "2", "3"] + + +def test_a_volume_that_does_not_exist_names_it(fake, tmp_path, monkeypatch): + monkeypatch.setenv(load.STAGING_VOLUME_ENV, "/Volumes/main/staging/nowhere") + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + assert "/Volumes/main/staging/nowhere" in str(exc.value) + + +def test_no_credential_reaches_an_error_message(fake, tmp_path, monkeypatch): + secret = "dapi-0123456789abcdef-not-real" # pragma: allowlist secret + monkeypatch.setenv("DATABRICKS_TOKEN", secret) + fake.fail_on("INSERT OVERWRITE", f"[FAILED] rejected the token {secret}") + with pytest.raises(load.DatabricksLoadError) as exc: + _load(_landing(fake), _parquet(tmp_path)) + assert secret not in str(exc.value) + assert "rejected the token" in str(exc.value) + + +def test_nothing_calls_a_second_client_or_endpoint(fake, tmp_path): + _load(_landing(fake), _parquet(tmp_path)) + assert fake.unknown == [] + allowed = ("/api/2.0/fs/", "/api/2.0/sql/statements", "/api/2.1/unity-catalog/tables/") + assert all(path.startswith(allowed) for _, path in fake.requests), fake.requests + + +# ── the runner: a databricks binding lands in the table, never in a local file ──── + + +def _source(tmp_path: Path, rows=ROWS) -> Path: + src = tmp_path / "subs.csv" + lines = ["id,msisdn,zip"] + [",".join(row) for row in rows] + src.write_text("\n".join(lines) + "\n", encoding="utf-8") + return src + + +def _contract( + tmp_path: Path, + *, + mode: str = "full_refresh", + rows=ROWS, + masking_rules: Optional[List[Dict[str, Any]]] = None, + location: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + expose = _expose(**(location or {})) + expose["binding"]["location"]["database"] = "main" + if masking_rules is not None: + expose["policy"] = {"privacy": {"masking": masking_rules}} + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "bronze.subs", + "name": "Subs", + "builds": [ + { + "id": "ingest", + "pattern": "acquisition", + "engine": "duckdb", + "properties": { + "source": { + "kind": "filesystem", + "mode": mode, + "connection": {"uri": str(_source(tmp_path, rows))}, + # all_varchar: the CSV reader would otherwise type msisdn and zip as numbers + "reader": {"format": "csv", "options": {"all_varchar": True}}, + "streams": ["subs"], + }, + "sink": {"format": "parquet"}, + }, + "outputs": ["subs"], + } + ], + "exposes": [expose], + } + + +def _run(contract: Dict[str, Any], tmp_path: Path) -> int: + return execute_duckdb_build(contract["builds"][0], contract, tmp_path, dry_run=False) + + +def _sha(value: str) -> str: + return hashlib.sha256((SALT + value).encode("utf-8")).hexdigest() + + +HASH = [{"column": "msisdn", "strategy": "hash"}] + + +def test_the_runner_lands_the_masked_rows_in_the_table(fake, tmp_path): + assert _run(_contract(tmp_path, masking_rules=HASH), tmp_path) == 0 + + rows = sorted(fake.rows(TABLE), key=lambda r: r["id"]) + assert [r["id"] for r in rows] == ["1", "2", "3"] + assert [r["msisdn"] for r in rows] == [_sha("+46701234567"), _sha("+46709876543"), None] + assert [r["zip"] for r in rows] == ["11122", "22233", "33344"], "the rest is unchanged" + assert "+4670" not in json.dumps(rows), "no cleartext left the machine" + assert fake.volume_files() == [], "and nothing is left in the volume" + assert fake.unknown == [] + + +def test_the_run_record_says_what_was_loaded(fake, tmp_path): + assert _run(_contract(tmp_path, masking_rules=HASH), tmp_path) == 0 + (record,) = (tmp_path / ".fluid" / "runs" / "bronze.subs" / "ingest" / "runs").glob("*.json") + facets = json.loads(record.read_text(encoding="utf-8"))["facets"] + facts = facets["databricks_load"] + assert facts["table"] == TABLE and facts["rows"] == 3 + assert facts["staged_files_deleted"] is True + assert facts["staged_to"].startswith(f"{VOLUME}/bronze_subs/ingest/") + assert facets["masking"]["columns"]["msisdn"]["strategy"] == "hash" + assert VOLUME not in json.dumps(facets["landed"]), "the landed facet names no volume path" + + +@pytest.mark.parametrize( + "location", + [None, {"path": "./bronze/subs.parquet"}, {"path": "dbfs:/mnt/bronze/subs/"}], +) +def test_a_databricks_binding_never_writes_the_local_fallback(fake, tmp_path, location): + """It used to write ``location.path``, or ``out/.parquet``, and report success.""" + assert _run(_contract(tmp_path, location=location), tmp_path) == 0 + assert [r["id"] for r in sorted(fake.rows(TABLE), key=lambda r: r["id"])] == ["1", "2", "3"] + assert list((tmp_path / "out").glob("*")) == [] + assert not (tmp_path / "bronze").exists() + assert not (tmp_path / "dbfs:").exists() + written = {p.name for p in tmp_path.rglob("*.parquet")} + assert written <= {"subs.parquet"} + assert all(".fluid/staging" in str(p) for p in tmp_path.rglob("*.parquet")) + + +def test_a_second_run_overwrites_the_table(fake, tmp_path): + assert _run(_contract(tmp_path), tmp_path) == 0 + again = _contract(tmp_path, rows=[("7", "+1", "00007")]) + assert _run(again, tmp_path) == 0 + assert [r["id"] for r in fake.rows(TABLE)] == ["7"] + + +def test_an_unset_staging_volume_is_refused_before_any_work(fake, tmp_path, monkeypatch, caplog): + monkeypatch.delenv(load.STAGING_VOLUME_ENV) + assert _run(_contract(tmp_path), tmp_path) != 0 + assert load.STAGING_VOLUME_ENV in caplog.text + assert not (tmp_path / ".fluid" / "staging").exists(), "refused before anything landed" + assert fake.requests == [], "and before the workspace was touched" + + +def test_an_unset_warehouse_is_refused_before_any_work(fake, tmp_path, monkeypatch, caplog): + monkeypatch.delenv("DATABRICKS_WAREHOUSE_ID") + assert _run(_contract(tmp_path), tmp_path) != 0 + assert "DATABRICKS_WAREHOUSE_ID" in caplog.text + assert fake.requests == [] + + +def test_an_unsupported_mode_is_refused_before_any_work(fake, tmp_path, caplog): + assert _run(_contract(tmp_path, mode="incremental_append"), tmp_path) != 0 + assert "full_refresh" in caplog.text + assert not (tmp_path / ".fluid" / "staging").exists() + assert fake.requests == [] + + +def test_a_missing_table_fails_the_build_naming_fluid_apply(fake, tmp_path, caplog): + del fake.tables[TABLE] + assert _run(_contract(tmp_path), tmp_path) != 0 + assert "fluid apply" in caplog.text + assert not [p for m, p in fake.file_ops if m == "PUT"] + + +def test_a_failed_load_fails_the_build_and_leaves_the_volume_empty(fake, tmp_path, caplog): + fake.fail_on("INSERT OVERWRITE", "[PERMISSION_DENIED] no MODIFY") + assert _run(_contract(tmp_path, masking_rules=HASH), tmp_path) != 0 + assert "databricks load" in caplog.text and "PERMISSION_DENIED" in caplog.text + assert fake.volume_files() == [] + + +def test_a_local_binding_never_touches_databricks(fake, tmp_path): + contract = _contract(tmp_path) + out = tmp_path / "out" / "subs.parquet" + contract["exposes"][0]["binding"] = { + "platform": "local", + "format": "parquet", + "location": {"path": str(out)}, + } + assert _run(contract, tmp_path) == 0 + assert out.exists() + assert fake.requests == [] + + +def test_each_build_loads_its_own_table(fake, tmp_path): + """The target is the expose the build's ``outputs`` names, not ``exposes[0]``.""" + fake.create_table("main.bronze.other", COLUMNS) + contract = _contract(tmp_path) + decoy = _expose(table="other") + decoy["exposeId"] = "other" + contract["exposes"].insert(0, decoy) + assert _run(contract, tmp_path) == 0 + assert fake.rows("main.bronze.other") == [] + assert len(fake.rows(TABLE)) == 3 diff --git a/tests/ci_guards/extras_alongside_base.py b/tests/ci_guards/extras_alongside_base.py new file mode 100644 index 000000000..e1eefeb79 --- /dev/null +++ b/tests/ci_guards/extras_alongside_base.py @@ -0,0 +1,86 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read pyproject's extras the way pip does, for the extras-alongside-base guard. + +litellm, mcp and openlineage-python sit in ``[project] dependencies`` today +and leave the base install in 0.28.0. Until then the extras that will carry +them are declared next to base (a duplicate requirement is harmless to pip +and uv), so every consumer can switch to explicit extras first. + +This module only parses; ``tests/test_ci_extras_alongside_base.py`` asserts. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Dict, List, Set + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - Python < 3.11 + import tomli as tomllib # type: ignore[no-redef] + +PYPROJECT = Path(__file__).resolve().parents[2] / "pyproject.toml" + +#: The project's own name, as it appears in self-referencing extras. +SELF = "data-product-forge" + +_SELF_REF = re.compile(r"^data-product-forge\[([^\]]+)\]$", re.IGNORECASE) + + +def _norm(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def load() -> dict: + return tomllib.loads(PYPROJECT.read_text(encoding="utf-8"))["project"] + + +def base_requirements(project: dict) -> List[str]: + return list(project["dependencies"]) + + +def extras(project: dict) -> Dict[str, List[str]]: + return {_norm(k): list(v) for k, v in project["optional-dependencies"].items()} + + +def requirement_name(requirement: str) -> str: + """The distribution a requirement string names (marker and extras dropped).""" + head = requirement.split(";", 1)[0].strip() + return _norm(re.split(r"[\[<>=!~ ]", head, maxsplit=1)[0]) + + +def closure(name: str, table: Dict[str, List[str]]) -> Set[str]: + """Every requirement ``name`` installs, following self-referencing extras. + + Self references (``data-product-forge[a,b]``) are expanded and not + returned; every other requirement string is returned verbatim. + """ + seen: Set[str] = set() + out: Set[str] = set() + stack = [_norm(name)] + while stack: + current = stack.pop() + if current in seen: + continue + seen.add(current) + for requirement in table[current]: + match = _SELF_REF.match(requirement.strip()) + if match: + stack.extend(_norm(part) for part in match.group(1).split(",")) + else: + out.add(requirement) + return out diff --git a/tests/cli/test_apply_build_ids_before_tofu.py b/tests/cli/test_apply_build_ids_before_tofu.py new file mode 100644 index 000000000..af4086f08 --- /dev/null +++ b/tests/cli/test_apply_build_ids_before_tofu.py @@ -0,0 +1,530 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fluid apply`` refuses an id the build phase cannot use before it applies anything. + +The build phase turns ``contract.id`` and ``builds[].id`` into directory names and +scheduler code, so it takes a narrower grammar (``build_runners._ids``) than the +schema's ``$defs.identifier``: a leading digit (``1orders``) or an id longer than +128 characters passes ``fluid validate`` and ``fluid plan``. Only the build runner +checked these ids, and on the OpenTofu engine it runs after ``tofu apply``. So +``--mode amend-and-build`` provisioned the product and then failed with +"Unexpected error", because ``IdentifierViolation`` had no typed mapping. + +A ``--build-id`` that names no build failed late the same way: the runner keeps the +builds whose id equals it and, with none left, logs ``Build not found`` and returns 1, +after ``tofu apply`` on the OpenTofu engine. + +These tests drive ``run()`` through the real argparse surface. ``tofu`` and the +apply hooks are recorders; the build runner is the real one, behind a spy, so +"refused first" is asserted as "``tofu``, the hooks and the runner never ran". +""" + +from __future__ import annotations + +import copy +import json +import logging +from pathlib import Path +from typing import Any, Dict, Iterator, Optional +from unittest import mock + +import pytest +import yaml + +import fluid_build.cli as cli_mod +from fluid_build.build_runners import _ids +from fluid_build.build_runners import base as runner_base +from fluid_build.build_runners._ids import IdentifierViolation +from fluid_build.cli._common import CLIError +from fluid_build.cli.apply import _gate_contract_for_apply +from fluid_build.cli.apply import run as apply_run +from fluid_build.forge.core.plan_digest import inject_digests + +LOG = logging.getLogger("test.apply_build_ids_before_tofu") + +_CONTRACT: Dict[str, Any] = { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "orders", + "name": "Orders", + "description": "Inline SQL build.", + "domain": "Demo", + "metadata": {"layer": "Bronze", "owner": {"team": "dp", "email": "dp@example.com"}}, + "builds": [ + { + "id": "make_rows", + "description": "Inline rows.", + "pattern": "embedded-logic", + "engine": "sql", + "properties": {"sql": "SELECT 1 AS id"}, + "outputs": ["rows"], + } + ], + "exposes": [ + { + "exposeId": "rows", + "kind": "table", + "binding": { + "platform": "local", + "format": "parquet", + "location": {"path": "./out/rows.parquet"}, + }, + "contract": {"schema": [{"name": "id", "type": "INTEGER", "required": True}]}, + } + ], +} + +# Ids the schema admits and the build phase refuses: (field, value). +_UNUSABLE = [ + pytest.param("contract.id", "1orders", id="contract-id-leading-digit"), + pytest.param("contract.id", "o" * 129, id="contract-id-129-chars"), + pytest.param("build.id", "1make_rows", id="build-id-leading-digit"), + pytest.param("build.id", "b" * 129, id="build-id-129-chars"), +] + +_BUILD_MODES = [ + pytest.param("amend-and-build", (), id="amend-and-build"), + pytest.param("replace-and-build", ("--allow-data-loss",), id="replace-and-build"), +] + + +def _with(field: str, value: str) -> Dict[str, Any]: + """``_CONTRACT`` with ``value`` as its ``contract.id`` or its build's id.""" + contract = copy.deepcopy(_CONTRACT) + if field == "contract.id": + contract["id"] = value + else: + contract["builds"][0]["id"] = value + return contract + + +@pytest.fixture +def workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("FLUID_ENV", raising=False) + (tmp_path / "runtime").mkdir() + return tmp_path + + +def _source(form: str, contract: Dict[str, Any], *, mode: str) -> str: + """Write ``contract`` as the YAML file an operator applies, or as a + digest-bound ``plan.json`` that embeds it.""" + if form == "yaml": + path, text = Path("contract.fluid.yaml"), yaml.safe_dump(contract, sort_keys=False) + else: + plan = {"format_version": "0.7.5", "mode": mode, "actions": [], "contract": contract} + path, text = Path("runtime/plan.json"), json.dumps(inject_digests(plan, bundle_path=None)) + path.write_text(text, encoding="utf-8") + return str(path) + + +def _apply(src: str, *flags: str) -> int: + """``fluid apply --yes`` through the real top-level parser.""" + return apply_run(cli_mod.build_parser().parse_args(["apply", src, *flags, "--yes"]), LOG) + + +@pytest.fixture +def tofu() -> Iterator[mock.Mock]: + """Route ``run()`` onto the OpenTofu engine and record the engine call.""" + with ( + mock.patch( + "fluid_build.cli._apply_opentofu_engine.resolve_apply_engine", + return_value="opentofu", + ), + mock.patch( + "fluid_build.cli._apply_opentofu_engine.apply_via_opentofu", return_value=0 + ) as engine, + ): + yield engine + + +@pytest.fixture +def native() -> Iterator[None]: + with mock.patch( + "fluid_build.cli._apply_opentofu_engine.resolve_apply_engine", return_value="native" + ): + yield + + +@pytest.fixture +def hooks() -> Iterator[mock.Mock]: + with mock.patch("fluid_build.cli.apply._run_apply_hooks", return_value=0) as recorder: + yield recorder + + +@pytest.fixture +def runner() -> Iterator[mock.Mock]: + """The real build runner, behind a spy.""" + with mock.patch( + "fluid_build.build_runners.run_builds_from_args", + wraps=runner_base.run_builds_from_args, + ) as spy: + yield spy + + +@pytest.mark.parametrize("field,value", _UNUSABLE) +def test_the_schema_admits_these_ids(field: str, value: str) -> None: + """Why the refusal came late: these ids pass the gate ``fluid validate`` and + ``fluid plan`` run. If the schema starts refusing them, the schema gate + refuses them first and the tests below need another example.""" + _gate_contract_for_apply(_with(field, value), LOG) + + +@pytest.mark.usefixtures("workspace") +class TestRefusedBeforeAnythingIsApplied: + @pytest.mark.parametrize("form", ["yaml", "plan"]) + @pytest.mark.parametrize("field,value", _UNUSABLE) + @pytest.mark.parametrize("mode,extra", _BUILD_MODES) + def test_refused_before_tofu( + self, + tofu: mock.Mock, + hooks: mock.Mock, + runner: mock.Mock, + form: str, + field: str, + value: str, + mode: str, + extra: Any, + ) -> None: + src = _source(form, _with(field, value), mode=mode) + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", mode, *extra) + tofu.assert_not_called() + hooks.assert_not_called() + runner.assert_not_called() + assert exc.value.event == "apply_build_identifier_invalid" + assert exc.value.context["field"] == field + assert exc.value.context["value"] == repr(value) + + @pytest.mark.usefixtures("native") + @pytest.mark.parametrize("field,value", _UNUSABLE) + def test_native_engine_refuses_with_the_same_typed_error( + self, runner: mock.Mock, field: str, value: str + ) -> None: + """The native engine applies nothing before its builds, but its refusal was + the same untyped ``IdentifierViolation``.""" + src = _source("yaml", _with(field, value), mode="amend-and-build") + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", "amend-and-build") + runner.assert_not_called() + assert exc.value.event == "apply_build_identifier_invalid" + + def test_a_build_the_filter_skips_is_still_checked( + self, tofu: mock.Mock, runner: mock.Mock + ) -> None: + """The runner checks every build's id before ``--build-id`` picks one, so a + check that skipped the other builds would let ``tofu`` run first.""" + contract = copy.deepcopy(_CONTRACT) + other = dict(contract["builds"][0], id="1other") + contract["builds"].append(other) + src = _source("yaml", contract, mode="amend-and-build") + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", "amend-and-build", "--build-id", "make_rows") + tofu.assert_not_called() + assert exc.value.context["value"] == repr("1other") + + def test_the_refusal_says_what_to_change(self, tofu: mock.Mock) -> None: + src = _source("yaml", _with("contract.id", "1orders"), mode="amend-and-build") + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", "amend-and-build") + hint = exc.value.context["hint"] + assert "Nothing was applied" in hint + assert "start with a letter or underscore" in hint + assert "128" in hint + assert exc.value.suggestions, "the catalog gives the event its suggestions" + + def test_the_cli_renders_it_as_a_typed_error( + self, tofu: mock.Mock, capsys: pytest.CaptureFixture[str] + ) -> None: + """Through ``main()``: the slug and the id, not "Unexpected error".""" + src = _source("yaml", _with("contract.id", "1orders"), mode="amend-and-build") + assert cli_mod.main(["apply", src, "--mode", "amend-and-build", "--yes"]) == 1 + out = capsys.readouterr().out + assert "ERR_APPLY_BUILD_IDENTIFIER_INVALID" in out + assert "'1orders'" in out + assert "Unexpected error" not in out + tofu.assert_not_called() + + @pytest.mark.parametrize( + "field,value", + [ + pytest.param("contract.id", "orders\n", id="contract-id"), + pytest.param("build.id", "make_rows\n", id="build-id"), + ], + ) + def test_a_trailing_newline_is_refused_before_tofu( + self, + tofu: mock.Mock, + hooks: mock.Mock, + runner: mock.Mock, + field: str, + value: str, + ) -> None: + """``_IDENT_RE`` ends in ``\\Z``, so the build phase refuses ``orders\\n``. The + schema's ``^...$`` pattern still lets ``re.search`` match before the newline. + This check refuses it until the schema pattern is fixed; after that the schema + gate does. Either way the refusal is typed and comes before ``tofu``.""" + src = _source("yaml", _with(field, value), mode="amend-and-build") + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", "amend-and-build") + tofu.assert_not_called() + hooks.assert_not_called() + runner.assert_not_called() + assert exc.value.event in {"apply_build_identifier_invalid", "apply_contract_invalid"} + + +@pytest.mark.usefixtures("workspace") +class TestNothingNewIsRefused: + """The check refuses exactly what the build runner would refuse.""" + + @pytest.mark.parametrize("form", ["yaml", "plan"]) + @pytest.mark.parametrize( + "field,value", + [ + pytest.param("contract.id", "orders", id="plain"), + pytest.param("contract.id", "o" * 128, id="contract-id-128-chars"), + pytest.param("build.id", "_make_rows", id="build-id-leading-underscore"), + ], + ) + def test_usable_ids_reach_tofu_then_the_runner( + self, tofu: mock.Mock, form: str, field: str, value: str + ) -> None: + with mock.patch("fluid_build.build_runners.run_builds_from_args", return_value=0) as run: + mode = "amend-and-build" + assert _apply(_source(form, _with(field, value), mode=mode), "--mode", mode) == 0 + tofu.assert_called_once() + run.assert_called_once() + + def test_a_mode_without_a_build_phase_is_not_checked( + self, tofu: mock.Mock, runner: mock.Mock + ) -> None: + src = _source("yaml", _with("contract.id", "1orders"), mode="amend") + assert _apply(src, "--mode", "amend") == 0 + tofu.assert_called_once() + runner.assert_not_called() + + def test_a_contract_without_builds_is_not_checked( + self, tofu: mock.Mock, runner: mock.Mock + ) -> None: + """With no builds the runner returns before its own check.""" + contract = _with("contract.id", "1orders") + del contract["builds"] + src = _source("yaml", contract, mode="amend-and-build") + assert _apply(src, "--mode", "amend-and-build") == 0 + tofu.assert_called_once() + runner.assert_called_once() + + +class TestOneRuleForBothCallers: + """``validate_build_ids`` is the rule the runner applies and ``fluid apply`` + runs ahead of it.""" + + @pytest.mark.parametrize("field,value", _UNUSABLE) + def test_it_names_the_id_that_failed(self, field: str, value: str) -> None: + with pytest.raises(IdentifierViolation) as exc: + _ids.validate_build_ids(_with(field, value)) + assert (exc.value.kind, exc.value.value) == (field, value) + + def test_the_runner_still_refuses_on_its_own(self, tmp_path: Path) -> None: + """Kept as the backstop for a caller that skips ``fluid apply``.""" + import argparse + + path = tmp_path / "contract.fluid.yaml" + path.write_text("{}", encoding="utf-8") + args = argparse.Namespace(contract=str(path), env=None, build_id=None) + with pytest.raises(IdentifierViolation, match="contract.id"): + runner_base.run_builds_from_args(args, LOG, contract=_with("contract.id", "1orders")) + + @pytest.mark.parametrize( + "contract", + [ + pytest.param({"id": "1orders"}, id="no-builds"), + pytest.param({"id": "1orders", "builds": []}, id="empty-builds"), + pytest.param({"id": "", "builds": [{"id": ""}]}, id="empty-ids"), + pytest.param({"builds": [{"engine": "sql"}]}, id="absent-ids"), + pytest.param({"id": "orders", "builds": ["make_rows", None]}, id="non-mapping-builds"), + ], + ) + def test_what_it_does_not_check(self, contract: Dict[str, Any]) -> None: + _ids.validate_build_ids(contract) + + +# ``--build-id`` values that name no build of ``_CONTRACT``. The runner compares ids +# exactly, so a near miss is as unknown as a typo. +_UNKNOWN_BUILD_IDS = [ + pytest.param("make_row", id="typo"), + pytest.param("Make_Rows", id="case"), + pytest.param("make_rows ", id="trailing-space"), + pytest.param("make", id="prefix"), +] + + +def _with_build_ids(*build_ids: Optional[str]) -> Dict[str, Any]: + """``_CONTRACT`` with one build per id; ``None`` is a build that declares no id.""" + contract = copy.deepcopy(_CONTRACT) + template = contract["builds"][0] + contract["builds"] = [] + for build_id in build_ids: + build = copy.deepcopy(template) + if build_id is None: + del build["id"] + else: + build["id"] = build_id + contract["builds"].append(build) + return contract + + +@pytest.mark.usefixtures("workspace") +class TestAnUnknownBuildIdIsRefusedFirst: + """``--build-id`` names no build of the contract. + + The runner keeps the builds whose id equals ``--build-id`` and, with none left, + logs ``Build not found`` and returns 1. On the OpenTofu engine that came after + ``tofu apply``: a mistyped id provisioned the product, then the run failed. + """ + + @pytest.mark.parametrize("form", ["yaml", "plan"]) + @pytest.mark.parametrize("build_id", _UNKNOWN_BUILD_IDS) + @pytest.mark.parametrize("mode,extra", _BUILD_MODES) + def test_refused_before_tofu( + self, + tofu: mock.Mock, + hooks: mock.Mock, + runner: mock.Mock, + form: str, + build_id: str, + mode: str, + extra: Any, + ) -> None: + src = _source(form, _CONTRACT, mode=mode) + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", mode, *extra, "--build-id", build_id) + tofu.assert_not_called() + hooks.assert_not_called() + runner.assert_not_called() + assert exc.value.event == "apply_build_id_not_found" + assert exc.value.context["build_id"] == repr(build_id) + assert exc.value.context["available"] == ["make_rows"] + + @pytest.mark.usefixtures("native") + def test_native_engine_refuses_with_the_same_typed_error(self, runner: mock.Mock) -> None: + """The native engine applies nothing before its builds, but its refusal was a + log line and exit 1.""" + src = _source("yaml", _CONTRACT, mode="amend-and-build") + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", "amend-and-build", "--build-id", "make_row") + runner.assert_not_called() + assert exc.value.event == "apply_build_id_not_found" + + def test_it_lists_the_build_ids_in_contract_order(self, tofu: mock.Mock) -> None: + src = _source("yaml", _with_build_ids("make_rows", "load_orders"), mode="amend-and-build") + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", "amend-and-build", "--build-id", "load_order") + assert exc.value.context["available"] == ["make_rows", "load_orders"] + + def test_builds_that_declare_no_id_match_no_build_id( + self, tofu: mock.Mock, runner: mock.Mock + ) -> None: + """``builds[].id`` is optional, and the runner matches no ``--build-id`` to a + build without one.""" + src = _source("yaml", _with_build_ids(None), mode="amend-and-build") + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", "amend-and-build", "--build-id", "make_rows") + tofu.assert_not_called() + runner.assert_not_called() + assert exc.value.context["available"] == [] + assert "none of the contract's builds declares an id" in exc.value.context["hint"] + + def test_an_unusable_id_is_refused_first(self, tofu: mock.Mock) -> None: + """The runner checks the ids before it filters, and apply keeps that order.""" + src = _source("yaml", _with("contract.id", "1orders"), mode="amend-and-build") + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", "amend-and-build", "--build-id", "make_row") + assert exc.value.event == "apply_build_identifier_invalid" + + def test_the_refusal_says_what_to_change(self, tofu: mock.Mock) -> None: + src = _source("yaml", _CONTRACT, mode="amend-and-build") + with pytest.raises(CLIError) as exc: + _apply(src, "--mode", "amend-and-build", "--build-id", "make_row") + hint = exc.value.context["hint"] + assert "Nothing was applied" in hint + assert "drop --build-id" in hint + assert exc.value.suggestions, "the catalog gives the event its suggestions" + + def test_the_cli_renders_it_as_a_typed_error( + self, tofu: mock.Mock, capsys: pytest.CaptureFixture[str] + ) -> None: + """Through ``main()``: the slug, the id asked for and the ids there are.""" + src = _source("yaml", _CONTRACT, mode="amend-and-build") + argv = ["apply", src, "--mode", "amend-and-build", "--build-id", "make_row", "--yes"] + assert cli_mod.main(argv) == 1 + out = capsys.readouterr().out + assert "ERR_APPLY_BUILD_ID_NOT_FOUND" in out + assert "'make_row'" in out + assert "['make_rows']" in out + tofu.assert_not_called() + + @pytest.mark.parametrize("build_id", _UNKNOWN_BUILD_IDS) + def test_the_runner_still_refuses_on_its_own( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture, build_id: str + ) -> None: + """Kept as the backstop for a caller that skips ``fluid apply``. It refuses the + same near misses: both compare ids exactly.""" + import argparse + + path = tmp_path / "contract.fluid.yaml" + path.write_text("{}", encoding="utf-8") + args = argparse.Namespace(contract=str(path), env=None, build_id=build_id) + with caplog.at_level(logging.ERROR, logger=LOG.name): + rc = runner_base.run_builds_from_args(args, LOG, contract=copy.deepcopy(_CONTRACT)) + assert rc == 1 + assert f"Build not found: {build_id}" in caplog.text + + +@pytest.mark.usefixtures("workspace") +class TestAKnownBuildIdStillRuns: + """The check refuses exactly the ``--build-id`` values the runner would refuse.""" + + @pytest.mark.parametrize("form", ["yaml", "plan"]) + @pytest.mark.parametrize("build_id", ["make_rows", "load_orders"]) + def test_it_reaches_tofu_then_the_runner( + self, tofu: mock.Mock, form: str, build_id: str + ) -> None: + src = _source(form, _with_build_ids("make_rows", "load_orders"), mode="amend-and-build") + with mock.patch("fluid_build.build_runners.run_builds_from_args", return_value=0) as run: + assert _apply(src, "--mode", "amend-and-build", "--build-id", build_id) == 0 + tofu.assert_called_once() + run.assert_called_once() + assert run.call_args.args[0].build_id == build_id + + def test_an_empty_build_id_filters_nothing(self, tofu: mock.Mock) -> None: + """The runner runs every build for ``--build-id ''``, so it is not refused.""" + src = _source("yaml", _CONTRACT, mode="amend-and-build") + with mock.patch("fluid_build.build_runners.run_builds_from_args", return_value=0) as run: + assert _apply(src, "--mode", "amend-and-build", "--build-id", "") == 0 + tofu.assert_called_once() + run.assert_called_once() + + def test_a_contract_without_builds_is_not_checked( + self, tofu: mock.Mock, runner: mock.Mock + ) -> None: + """With no builds the runner returns 0 before it filters, so a ``--build-id`` is + not refused there either.""" + contract = copy.deepcopy(_CONTRACT) + del contract["builds"] + src = _source("yaml", contract, mode="amend-and-build") + assert _apply(src, "--mode", "amend-and-build", "--build-id", "make_rows") == 0 + tofu.assert_called_once() + runner.assert_called_once() diff --git a/tests/cli/test_apply_reports_to_command_center.py b/tests/cli/test_apply_reports_to_command_center.py index c60e76a41..80782468d 100644 --- a/tests/cli/test_apply_reports_to_command_center.py +++ b/tests/cli/test_apply_reports_to_command_center.py @@ -592,15 +592,21 @@ def test_a_failed_build_is_reported_with_its_build_id(product, tofu, build, cc, def test_a_build_id_that_is_not_in_the_contract_is_the_reason( product, tofu, build, cc, monkeypatch ): + """Refused before ``tofu`` runs: the run reports the refusal and no infrastructure.""" url, recorder = cc _configure(monkeypatch, url) _with_build(product) - assert _apply(product, "--mode", "amend-and-build", "--build-id", "no_such_build") == 1 + with pytest.raises(CLIError) as exc: + _apply(product, "--mode", "amend-and-build", "--build-id", "no_such_build") + assert exc.value.event == "apply_build_id_not_found" + (post,) = _by(recorder, "POST") (patch,) = _by(recorder, "PATCH") - assert patch["body"]["result"]["error_event"] == "build_not_found:no_such_build" - assert patch["body"]["current_phase"] == "build" + assert post["body"]["metadata"]["product_id"] == "bronze.customer_subscriptions" + result = patch["body"]["result"] + assert result["error_event"] == "apply_build_id_not_found" + assert "planned_changes" not in result assert build["calls"] == 0 diff --git a/tests/cli/test_apply_simple_provider.py b/tests/cli/test_apply_simple_provider.py index e0bbbf5fb..df678d7c8 100644 --- a/tests/cli/test_apply_simple_provider.py +++ b/tests/cli/test_apply_simple_provider.py @@ -125,6 +125,29 @@ def _refusal(contract: Dict[str, Any]) -> CLIError: return exc.value +class TestRetiredRedshiftPlatform: + """``redshift`` stopped being a provider but is still a valid runtime platform; + it resolves to ``aws``, which the retired registry entry forwarded to.""" + + def test_a_redshift_runtime_build_beside_a_local_product_selects_aws(self) -> None: + contract = _contract( + [_expose("orders", "local")], [_sql("make_orders", ["orders"], platform="redshift")] + ) + assert _simple_apply_provider(contract, LOG) == "aws" + + def test_a_redshift_expose_selects_aws(self) -> None: + assert _simple_apply_provider(_contract([_expose("orders", "redshift")]), LOG) == "aws" + + def test_plan_resolves_the_same_way(self) -> None: + from fluid_build.cli._common import resolve_provider_from_contract + + contract = _contract( + [_expose("orders", "local")], [_sql("make_orders", ["orders"], platform="redshift")] + ) + contract["exposes"][0]["binding"].pop("platform") + assert resolve_provider_from_contract(contract)[0] == "aws" + + class TestOutsideForgeCli: """A job that runs outside forge-cli, and what it writes, choose no provider.""" diff --git a/tests/cli/test_verify_databricks.py b/tests/cli/test_verify_databricks.py index efd876f4e..af46522aa 100644 --- a/tests/cli/test_verify_databricks.py +++ b/tests/cli/test_verify_databricks.py @@ -137,10 +137,10 @@ def __init__(self) -> None: self.api_client = StubApiClient(self) self.calls: List[Any] = [] - def add_jobs(self, name: str, count: int) -> None: + def add_jobs(self, name: str, count: int, **settings: Any) -> None: for _ in range(count): job_id = len(self.jobs) + 1 - self.jobs.append({"job_id": job_id, "settings": {"name": name}}) + self.jobs.append({"job_id": job_id, "settings": {"name": name, **settings}}) def answer(self, method: str, path: str, query: Dict[str, Any]) -> Dict[str, Any]: assert method == "GET" @@ -1058,3 +1058,50 @@ def test_the_real_sdk_reads_the_stand_in_workspace(tmp_path, monkeypatch): assert result["status"] == "mismatch", result assert [f["field"] for f in result["dimensions"]["structure"]["missing_fields"]] == ["region"] assert [f["principal"] for f in result["dimensions"]["catalog_access"]["findings"]] == ["bi"] + + +# ── A job this module paused because its table is not protected ───────── + + +def restricted_contract_with_job() -> Dict[str, Any]: + document = restricted_contract() + document["builds"] = contract_with_job()["builds"] + return document + + +def test_a_job_paused_while_its_table_is_not_protected_is_a_finding(workspace): + contract = restricted_contract_with_job() + workspace.add_jobs( + job_name(), 1, schedule={"quartz_cron_expression": "0 0 2 * * ?", "pause_status": "PAUSED"} + ) + + result = verify(workspace, contract) + + dimension = result["dimensions"]["jobs"] + assert dimension["status"] == "fail" + [finding] = dimension["findings"] + assert finding["build"] == "dbt_daily" + assert "is paused" in finding["message"] and "not protected" in finding["message"] + assert "run fluid apply" in finding["message"] + assert result["severity"]["level"] == "CRITICAL" + + +def test_a_paused_job_on_a_protected_table_is_not_a_finding(workspace): + contract = restricted_contract_with_job() + bind_all(workspace, contract) + workspace.add_jobs( + job_name(), 1, schedule={"quartz_cron_expression": "0 0 2 * * ?", "pause_status": "PAUSED"} + ) + + assert verify(workspace, contract)["dimensions"]["jobs"]["status"] == "pass" + + +def test_an_unpaused_job_on_an_unprotected_table_has_no_paused_finding(workspace): + contract = restricted_contract_with_job() + workspace.add_jobs( + job_name(), + 1, + schedule={"quartz_cron_expression": "0 0 2 * * ?", "pause_status": "UNPAUSED"}, + ) + + assert verify(workspace, contract)["dimensions"]["jobs"]["status"] == "pass" diff --git a/tests/cli/test_verify_databricks_retention.py b/tests/cli/test_verify_databricks_retention.py new file mode 100644 index 000000000..dae62f2a5 --- /dev/null +++ b/tests/cli/test_verify_databricks_retention.py @@ -0,0 +1,263 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fluid verify``'s retention dimension for a Databricks expose with ``lifecycle.expire``. + +The contract's retention must cover the table's live Delta history (the larger of +``delta.logRetentionDuration`` and ``delta.deletedFileRetentionDuration``, 30 and 7 days when +unset); predictive optimization off is a warning. Driven through ``verify_databricks_expose`` +with the stub of ``test_verify_databricks.py``, and once over HTTP against the stand-in workspace. +""" + +from __future__ import annotations + +import copy +from typing import Any, Dict, Optional + +import pytest + +from fluid_build.cli._verify_databricks import verify_databricks_expose +from tests.cli.test_verify_databricks import ( + CONTRACT, + LIVE_COLUMNS, + TABLE, + StubWorkspace, +) + +pytestmark = pytest.mark.unit + +DAY = 86400 + + +class RetentionWorkspace(StubWorkspace): + """The stub, with Delta properties and a predictive optimization flag on GetTable.""" + + def __init__(self) -> None: + super().__init__() + self.extras: Dict[str, Any] = {} + #: GetTable serves a managed Delta table's properties (``delta.*``) with it. + self.serves_properties = True + + def answer(self, method: str, path: str, query: Dict[str, Any]) -> Dict[str, Any]: + out = super().answer(method, path, query) + if path == f"/api/2.1/unity-catalog/tables/{TABLE}": + served = {"properties": {"delta.minReaderVersion": "3"}} + out = {**(served if self.serves_properties else {}), **out} + out = {**out, **copy.deepcopy(self.extras)} + return out + + +def _contract(retention: Optional[str] = "P365D", expire: bool = True) -> Dict[str, Any]: + contract = copy.deepcopy(CONTRACT) + if retention is not None: + contract["exposes"][0]["lifecycle"] = {"retention": retention, "expire": expire} + contract["builds"] = [ + { + "id": "dbt_daily", + "engine": "dbt", + "repository": "./dbt", + "properties": {"model": "orders"}, + "outputs": ["orders"], + "execution": {"runtime": {"platform": "databricks"}}, + } + ] + return contract + + +def _verify(workspace: Any, contract: Dict[str, Any]) -> Dict[str, Any]: + return verify_databricks_expose( + "orders", + contract["exposes"][0], + contract=contract, + client_factory=lambda: workspace, + ) + + +@pytest.fixture +def workspace() -> RetentionWorkspace: + ws = RetentionWorkspace() + ws.add_jobs("fluid dbx.fake.orders dbt_daily", 1) + return ws + + +def _dimension(result: Dict[str, Any]) -> Dict[str, Any]: + assert "retention" in result["dimensions"], result + return result["dimensions"]["retention"] + + +def test_no_expire_skips(workspace): + result = _verify(workspace, _contract(None)) + assert _dimension(result)["status"] == "skip" + assert result["status"] == "match" + + +def test_expire_false_skips(workspace): + assert _dimension(_verify(workspace, _contract("P365D", expire=False)))["status"] == "skip" + + +def test_delta_defaults_inside_the_retention_pass(workspace): + workspace.extras = {"effective_predictive_optimization_flag": {"value": "ENABLE"}} + result = _verify(workspace, _contract()) + dim = _dimension(result) + assert dim["status"] == "pass", dim + assert dim["history_seconds"] == 30 * DAY and dim["retention"] == "P365D" + assert dim["predictive_optimization"] == "ENABLE" and dim["warnings"] == [] + assert result["status"] == "match" and result["severity"]["level"] == "SUCCESS" + + +def test_a_history_exactly_the_retention_passes(workspace): + assert _dimension(_verify(workspace, _contract("P30D")))["status"] == "pass" + + +def test_a_history_longer_than_the_retention_is_critical_with_the_fix(workspace): + workspace.extras = {"properties": {"delta.logRetentionDuration": "interval 400 days"}} + result = _verify(workspace, _contract("P365D")) + dim = _dimension(result) + assert dim["status"] == "fail" and "400 days" in dim["message"] + assert "ALTER TABLE `main`.`sales`.`orders` SET TBLPROPERTIES" in dim["fix"] + assert "'delta.logRetentionDuration' = 'interval 365 days'" in dim["fix"] + assert result["status"] == "mismatch" and result["severity"]["level"] == "CRITICAL" + assert any("ALTER TABLE" in action for action in result["severity"]["actions"]) + + +def test_the_deleted_file_retention_counts(workspace): + workspace.extras = {"properties": {"delta.deletedFileRetentionDuration": "interval 500 days"}} + assert _dimension(_verify(workspace, _contract("P365D")))["status"] == "fail" + + +def test_a_history_property_of_the_runtime_wins(workspace): + workspace.extras = { + "properties": {"delta.logRetentionDuration": "interval 10 days"}, + "delta_runtime_properties_kvpairs": { + "delta_runtime_properties": {"delta.logRetentionDuration": "interval 400 days"} + }, + } + assert _dimension(_verify(workspace, _contract("P365D")))["status"] == "fail" + workspace.extras = { + "delta_runtime_properties_kvpairs": { + "delta_runtime_properties": {"delta.logRetentionDuration": "interval 60 days"} + }, + } + assert _dimension(_verify(workspace, _contract("P365D")))["status"] == "pass" + + +def test_an_unreadable_property_fails(workspace): + workspace.extras = {"properties": {"delta.logRetentionDuration": "forever"}} + dim = _dimension(_verify(workspace, _contract())) + assert dim["status"] == "fail" and "forever" in dim["message"] + + +def test_predictive_optimization_off_is_a_warning_not_a_failure(workspace): + workspace.extras = {"effective_predictive_optimization_flag": {"value": "DISABLE"}} + result = _verify(workspace, _contract()) + dim = _dimension(result) + assert dim["status"] == "warn" and dim["predictive_optimization"] == "DISABLE" + (warning,) = dim["warnings"] + assert "ALTER SCHEMA `main`.`sales` ENABLE PREDICTIVE OPTIMIZATION" in warning + assert "VACUUM `main`.`sales`.`orders`" in warning + assert result["status"] == "match" and result["severity"]["level"] == "SUCCESS" + + +def test_the_tables_own_flag_is_read_when_no_effective_one_is_served(workspace): + workspace.extras = {"enable_predictive_optimization": "DISABLE"} + assert _dimension(_verify(workspace, _contract()))["status"] == "warn" + + +@pytest.mark.parametrize( + "extras", + [ + {}, + {"enable_predictive_optimization": "INHERIT"}, + {"effective_predictive_optimization_flag": {}}, + ], +) +def test_a_flag_that_is_not_reported_is_not_a_warning(workspace, extras): + workspace.extras = extras + dim = _dimension(_verify(workspace, _contract())) + assert dim["status"] == "pass" and dim["predictive_optimization"] is None + + +def test_a_failure_keeps_the_warning(workspace): + workspace.extras = { + "properties": {"delta.logRetentionDuration": "interval 400 days"}, + "effective_predictive_optimization_flag": {"value": "DISABLE"}, + } + dim = _dimension(_verify(workspace, _contract("P60D"))) + assert dim["status"] == "fail" and len(dim["warnings"]) == 1 + + +def test_the_other_dimensions_are_untouched(workspace): + workspace.tables[TABLE] = copy.deepcopy(LIVE_COLUMNS) + result = _verify(workspace, _contract()) + assert result["dimensions"]["structure"]["status"] == "pass" + assert result["dimensions"]["jobs"]["status"] == "pass" + + +def test_a_table_the_module_refuses_to_expire_is_not_verified(workspace): + contract = _contract() + contract["builds"][0]["repository"] = "https://github.com/o/r?tag=v1" + result = _verify(workspace, contract) + assert result["status"] == "unsupported" + + +# ── over HTTP, against the stand-in workspace ─────────────────────────────── +def test_the_stand_in_workspace_serves_the_properties_verify_reads(tmp_path): + from tests.iac._fake_databricks import FakeDatabricks + + class Workspace(FakeDatabricks): + def _table_doc(self, full: str) -> Dict[str, Any]: + return { + **super()._table_doc(full), + "effective_predictive_optimization_flag": {"value": "ENABLE"}, + } + + fake = Workspace().start() + try: + fake.create_table( + TABLE, + [("id", "bigint", False), ("amount", "double", True), ("region", "string", True)], + properties={"delta.logRetentionDuration": "interval 400 days"}, + ) + fake.grant_catalog("main", "analysts", "USE_CATALOG") + fake.grant_catalog("main", "bi", "USE_CATALOG") + failing = _verify(fake.client(), _contract("P365D")) + fake.tables[TABLE]["properties"] = {"delta.logRetentionDuration": "interval 90 days"} + passing = _verify(fake.client(), _contract("P365D")) + finally: + fake.stop() + assert _dimension(failing)["status"] == "fail" + assert _dimension(passing)["status"] == "pass" + assert _dimension(passing)["predictive_optimization"] == "ENABLE" + + +def test_a_table_served_without_properties_is_a_warning_that_says_the_defaults_were_assumed( + workspace, +): + workspace.serves_properties = False + workspace.extras = {"effective_predictive_optimization_flag": {"value": "ENABLE"}} + dim = _dimension(_verify(workspace, _contract())) + assert dim["status"] == "warn", dim + assert "no table properties" in dim["warnings"][0] + assert "SHOW TBLPROPERTIES" in dim["fix"] + + +def test_verify_reads_the_contract_as_apply_does_with_its_placeholders_substituted( + workspace, monkeypatch +): + monkeypatch.setenv("FLUID_TEST_RETENTION", "P365D") + contract = _contract("{{ env.FLUID_TEST_RETENTION }}") + workspace.extras = {"effective_predictive_optimization_flag": {"value": "ENABLE"}} + dim = _dimension(_verify(workspace, contract)) + assert dim["status"] == "pass", dim + assert dim["retention"] == "P365D" diff --git a/tests/conftest.py b/tests/conftest.py index cfa6b6c4d..0eaf60cbb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -208,22 +208,20 @@ def _disable_copilot_self_eval(monkeypatch): def deterministic_welcome_scan(monkeypatch): """Make ``run_welcome_scan`` return every probe's result, however slow the runner. - The scan runs its probes in threads and discards any result that misses a - 0.4s per-probe / 1.0s total budget: right for a CLI prompt, wrong for a test - that asserts on what a probe reports (workspace products, sample data, ...). - On a loaded runner (coverage, ``-n auto``) the probe overruns, its field - keeps the "nothing detected" default, and the mode picker falls back to - ``ai`` where the test expects ``from_product`` or ``refine``. - - The budget only decides which results are kept, never how long the scan runs - (it still joins every probe thread), so wall-clock assertions are unaffected. - The dbt probe sizes its ``dbt --version`` subprocess from the same budget, so - it is pointed at "no dbt installed" instead of waiting out a slow dbt-core; - tests of the dbt probe patch ``_resolve_dbt_executable`` themselves. + The scan runs its probes in threads and discards any result that misses its + 1.0s wall-clock budget: right for a CLI prompt, wrong for a test that asserts + on what a probe reports (workspace products, sample data, ...). On a loaded + runner (coverage, ``-n auto``) a probe can overrun, its field keeps the + "nothing detected" default, and the mode picker falls back to ``ai`` where + the test expects ``from_product`` or ``refine``. + + The budget is lifted so nothing is dropped. The scan still returns as soon + as every probe has finished, so wall-clock assertions are unaffected. The dbt + probe is pointed at "no dbt installed" so the result does not depend on the + host's dbt; tests of the dbt probe patch ``_resolve_dbt_executable`` themselves. Opt in with ``pytestmark = pytest.mark.usefixtures("deterministic_welcome_scan")``. """ - monkeypatch.setattr("fluid_build.cli._welcome_scan._PROBE_TIMEOUT_S", 30.0) monkeypatch.setattr("fluid_build.cli._welcome_scan._TOTAL_TIMEOUT_S", 60.0) monkeypatch.setattr( "fluid_build.build_runners.dbt.runner._resolve_dbt_executable", lambda: None diff --git a/tests/iac/_fake_databricks.py b/tests/iac/_fake_databricks.py index 661c36440..3de8eb4a5 100644 --- a/tests/iac/_fake_databricks.py +++ b/tests/iac/_fake_databricks.py @@ -49,8 +49,17 @@ so a test asserts on exactly what the provider submitted; ``add_job`` / ``add_workspace_file`` seed brownfield objects. +It also serves the Files API on Unity Catalog volumes (``add_volume``): PUT / DELETE of a file, +PUT / GET / DELETE of a directory, backed by a temporary directory so the bytes a client uploads +are really on disk (``volume_files`` lists them, ``file_ops`` orders the calls). Two data +statements run over those files: ``INSERT OVERWRITE
SELECT FROM +read_files('', format => 'parquet')`` reads the Parquet with DuckDB and REPLACES the table's +rows (``rows``; the table's definition, row filter and masks are left as they are, as the real +statement leaves them), reporting ``num_affected_rows`` / ``num_inserted_rows`` as Databricks does +unless ``insert_reports_count`` is False, and ``SELECT COUNT(*) FROM
``. + Not proven here: what real Databricks accepts (SQL dialect, warehouse state, the privilege -model, Unity Catalog storage), whether a job's settings are valid (the fake stores any), job +model, Unity Catalog storage, ``read_files`` itself), whether a job's settings are valid (the fake stores any), job runs, and any workspace-level setting. The provider is pointed at it with ``host`` and a dummy ``token``. Standalone: @@ -62,10 +71,14 @@ import base64 import copy import json +import os import re +import shutil +import tempfile import threading import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple from urllib.parse import parse_qs, unquote, urlparse @@ -300,6 +313,7 @@ def _execute(fake: "FakeDatabricks", sql: str) -> Tuple[bool, str]: if col["name"] in kept: col["mask"] = kept[col["name"]] fake.tables[full] = new + fake.table_rows.pop(full, None) # a new table holds no rows return True, "" m = re.match( @@ -353,6 +367,17 @@ def _execute(fake: "FakeDatabricks", sql: str) -> Tuple[bool, str]: held[who] = [p for p in held.get(who, []) if p != gone] return True, "" + m = re.match(rf"DROP\s+FUNCTION\s+(IF\s+EXISTS\s+)?{_FULL}\s*$", s, re.I) + if m: + ie, cat, sch, name = m.groups() + full = f"{cat}.{sch}.{name}" + if full not in fake.functions: + return (True, "") if ie else (False, f"[ROUTINE_NOT_FOUND] {full}") + if fake.function_in_use(full): + return False, f"[FUNCTION_IN_USE] {full} is used by a row filter or a column mask" + del fake.functions[full] + return True, "" + m = re.match(rf"DROP\s+TABLE\s+(IF\s+EXISTS\s+)?{_FULL}\s*$", s, re.I) if m: ie, cat, sch, name = m.groups() @@ -360,6 +385,7 @@ def _execute(fake: "FakeDatabricks", sql: str) -> Tuple[bool, str]: if full not in fake.tables: return (True, "") if ie else (False, f"[TABLE_OR_VIEW_NOT_FOUND] {full}") del fake.tables[full] + fake.table_rows.pop(full, None) fake.grants.pop(("table", full), None) return True, "" @@ -543,6 +569,127 @@ def find(col: str) -> int: return False, f"fake_databricks: unsupported statement: {s[:200]}" +def _count_doc(n: int) -> Dict[str, Any]: + """The result of a one-column query: a row of strings, as the Statement Execution API sends.""" + return { + "manifest": { + "format": "JSON_ARRAY", + "schema": { + "column_count": 1, + "columns": [{"name": "count(1)", "type_name": "LONG", "position": 0}], + }, + "total_row_count": 1, + }, + "result": {"row_count": 1, "data_array": [[str(n)]]}, + } + + +def _insert_doc(n: int, report: bool) -> Dict[str, Any]: + """What ``INSERT`` answers: ``num_affected_rows`` and ``num_inserted_rows`` (Databricks SQL).""" + if not report: + return {} + names = ("num_affected_rows", "num_inserted_rows") + return { + "manifest": { + "format": "JSON_ARRAY", + "schema": { + "column_count": 2, + "columns": [ + {"name": name, "type_name": "LONG", "position": i} + for i, name in enumerate(names) + ], + }, + "total_row_count": 1, + }, + "result": {"row_count": 1, "data_array": [[str(n), str(n)]]}, + } + + +def _data_statement(fake: "FakeDatabricks", sql: str) -> Optional[Tuple[bool, str, Dict[str, Any]]]: + """``(ok, error, result)`` for a statement that reads or writes rows, ``None`` for any other. + + ``INSERT OVERWRITE [()] SELECT FROM read_files('', format => + 'parquet')`` replaces the rows of ```` with the Parquet file at ````, or every one + under it when it is a directory (a path on a volume the Files API holds). With a column list + each selected column goes to the target column it names (a table column it leaves out is + NULL); without one, by position in table order, as Databricks does. + """ + s = sql.strip().rstrip(";").strip() + m = re.match( + rf"INSERT\s+OVERWRITE\s+(?:TABLE\s+)?{_FULL}\s+(?:\(([^)]*)\)\s+)?SELECT\s+(.+?)\s+" + rf"FROM\s+read_files\s*\(\s*{_QUOTED}\s*,\s*format\s*=>\s*'parquet'\s*\)\s*$", + s, + re.S | re.I, + ) + if m: + cat, sch, name, targets, select, literal = m.groups() + full = f"{cat}.{sch}.{name}" + table = fake.tables.get(full) + if table is None: + return ( + False, + f"[TABLE_OR_VIEW_NOT_FOUND] The table or view `{cat}`.`{sch}`.`{name}` cannot " + "be found.", + {}, + ) + wanted = _names(select) + names = [c["name"] for c in table["columns"]] + into = _names(targets) if targets is not None else names + unknown = [c for c in into if c.lower() not in {n.lower() for n in names}] + if unknown: + return ( + False, + f"[UNRESOLVED_COLUMN] Cannot write to {full}: no column {unknown[0]!r}.", + {}, + ) + if len(wanted) != len(into): + return ( + False, + f"[INSERT_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS] Cannot write to {full}: " + f"the insert names {len(into)} columns, the query {len(wanted)}.", + {}, + ) + source = fake._fs_path(re.sub(r"\\(.)", r"\1", literal)) + if source is not None and os.path.isfile(source): + files = [source] + elif source is not None and os.path.isdir(source): + files = sorted(str(p) for p in Path(source).rglob("*.parquet")) + else: + files = [] + if not files: + return ( + False, + f"[UNABLE_TO_INFER_SCHEMA] read_files found no Parquet file at {literal!r}.", + {}, + ) + import duckdb + + con = duckdb.connect(":memory:") + try: + quoted = ", ".join('"' + c.replace('"', '""') + '"' for c in wanted) + fetched = con.execute(f"SELECT {quoted} FROM read_parquet(?)", [files]).fetchall() + except Exception as exc: # noqa: BLE001 — what the file lacks is the statement's error + return False, f"[UNRESOLVED_COLUMN] {type(exc).__name__}: {exc}", {} + finally: + con.close() + by_lower = {n.lower(): n for n in names} + fake.table_rows[full] = [ + { + **{n: None for n in names}, + **{by_lower[t.lower()]: v for t, v in zip(into, row, strict=True)}, + } + for row in fetched + ] + return True, "", _insert_doc(len(fetched), fake.insert_reports_count) + m = re.match(rf"SELECT\s+COUNT\(\s*(?:\*|1)\s*\)\s+FROM\s+{_FULL}\s*$", s, re.I) + if m: + full = ".".join(m.groups()) + if full not in fake.tables: + return False, f"[TABLE_OR_VIEW_NOT_FOUND] The table or view {full} cannot be found.", {} + return True, "", _count_doc(len(fake.table_rows.get(full, []))) + return None + + # ---------------------------------------------------------------------- a client class _HttpError(Exception): """What the SDK raises for an error status; ``status_code`` is what forge-cli reads.""" @@ -559,15 +706,29 @@ class _ApiClient: def __init__(self, endpoint: str): self._endpoint = endpoint - def do(self, method: str, path: str, query: Any = None, body: Any = None, **_: Any) -> Any: + def do( + self, + method: str, + path: str, + query: Any = None, + body: Any = None, + headers: Any = None, + data: Any = None, + **_: Any, + ) -> Any: import urllib.error import urllib.parse import urllib.request url = self._endpoint + path + ("?" + urllib.parse.urlencode(query) if query else "") - data = json.dumps(body).encode() if body is not None else None - request = urllib.request.Request(url, data=data, method=method) + if data is not None: # a stream or bytes, sent as the SDK sends a file upload + payload = data if isinstance(data, (bytes, bytearray)) else data.read() + else: + payload = json.dumps(body).encode() if body is not None else None + request = urllib.request.Request(url, data=payload, method=method) request.add_header("Content-Type", "application/json") + for name, value in (headers or {}).items(): + request.add_header(name, value) try: with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 raw = response.read() @@ -625,6 +786,16 @@ def __init__( # (method, path substring, status, message): matching requests are answered with an error self.failing_routes: List[Tuple[str, str, int, str]] = [] self.replaced: List[str] = [] # tables a CREATE OR REPLACE hit while they existed + # Unity Catalog volumes the Files API serves ("cat.schema.volume"), their bytes in a + # temporary directory, and every call against them in order: (method, path) + self.volumes: set = set() + self.files_root = tempfile.mkdtemp(prefix="fake-databricks-files-") + self.file_ops: List[Tuple[str, str]] = [] + # full table name -> rows (dicts by column name, table order) the last INSERT OVERWRITE + # wrote; a table nothing was loaded into has none + self.table_rows: Dict[str, List[Dict[str, Any]]] = {} + # False: an INSERT answers with no row count, as a statement that returns none would + self.insert_reports_count = True self._marker = 0 self._job_seq = 0 self._object_ids: Dict[str, int] = {} # workspace path -> object_id @@ -648,14 +819,16 @@ def _handle(self, method: str) -> None: n = int(self.headers.get("Content-Length") or 0) raw = self.rfile.read(n) if n else b"" ctype = self.headers.get("Content-Type") or "" + path = unquote(parsed.path) try: - if ctype.startswith("multipart/form-data"): + if path.startswith("/api/2.0/fs/files"): + body = {"_raw": raw} # an octet stream, not JSON + elif ctype.startswith("multipart/form-data"): body = _multipart(ctype, raw) else: body = json.loads(raw) if raw else {} except ValueError: body = {} - path = unquote(parsed.path) with fake._lock: fake.requests.append((method, path)) fake.queries.append((method, path, parsed.query)) @@ -694,6 +867,7 @@ def stop(self) -> None: if self._server is not None: self._server.shutdown() self._server.server_close() + shutil.rmtree(self.files_root, ignore_errors=True) @property def endpoint(self) -> str: @@ -849,6 +1023,8 @@ def _register(self) -> None: a("PATCH", r"/api/2\.1/unity-catalog/schemas/([^/]+)", self._patch_schema) a("DELETE", r"/api/2\.1/unity-catalog/schemas/([^/]+)", self._delete_schema) a("GET", r"/api/2\.1/unity-catalog/functions/([^/]+)", self._get_function) + a("GET", r"/api/2\.1/unity-catalog/functions", self._list_functions) + a("GET", r"/api/2\.1/unity-catalog/tables", self._list_tables) a("GET", r"/api/2\.1/unity-catalog/tables/([^/]+)", self._get_table) a("DELETE", r"/api/2\.1/unity-catalog/tables/([^/]+)", self._delete_table) a("GET", r"/api/2\.1/unity-catalog/permissions/([a-z_]+)/([^/]+)", self._get_perms) @@ -886,6 +1062,12 @@ def _register(self) -> None: a("POST", r"/api/2\.0/workspace/import", self._ws_import) a("GET", r"/api/2\.0/workspace/get-status", self._ws_status) a("POST", r"/api/2\.0/workspace/delete", self._ws_delete) + # ---- files (Unity Catalog volumes) + a("PUT", r"/api/2\.0/fs/directories(/.*)", self._fs_mkdir) + a("GET", r"/api/2\.0/fs/directories(/.*)", self._fs_list) + a("DELETE", r"/api/2\.0/fs/directories(/.*)", self._fs_rmdir) + a("PUT", r"/api/2\.0/fs/files(/.*)", self._fs_put) + a("DELETE", r"/api/2\.0/fs/files(/.*)", self._fs_delete) # ---- sql a("POST", r"/api/2\.0/sql/statements/?", self._exec_sql) a("GET", r"/api/2\.0/sql/statements/([^/]+)", self._get_stmt) @@ -1041,6 +1223,41 @@ def _get_function(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: }, } + def _list_functions(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + """The functions of one schema, a page at a time (``list_page_size``).""" + cat = (q.get("catalog_name") or [""])[0] + sch = (q.get("schema_name") or [""])[0] + names = sorted(n for n in self.functions if n.rsplit(".", 1)[0] == f"{cat}.{sch}") + return 200, self._page(names, q, "functions", self._function_summary) + + def _function_summary(self, full: str) -> Dict[str, Any]: + cat, sch, name = full.split(".") + return { + "name": name, + "catalog_name": cat, + "schema_name": sch, + "full_name": full, + "owner": self.functions[full].get("owner"), + } + + def _list_tables(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + cat = (q.get("catalog_name") or [""])[0] + sch = (q.get("schema_name") or [""])[0] + names = sorted(n for n in self.tables if n.rsplit(".", 1)[0] == f"{cat}.{sch}") + return 200, self._page( + names, q, "tables", lambda n: {"name": n.rsplit(".", 1)[1], "full_name": n} + ) + + def _page( + self, names: List[str], q: Any, key: str, row: Callable[[str], Any] + ) -> Dict[str, Any]: + size = int(getattr(self, "list_page_size", 100)) + start = int((q.get("page_token") or ["0"])[0]) + out: Dict[str, Any] = {key: [row(n) for n in names[start : start + size]]} + if start + size < len(names): + out["next_page_token"] = str(start + size) + return out + # -- tables def _get_table(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: full = m.group(1) @@ -1087,7 +1304,7 @@ def _table_doc(self, full: str) -> Dict[str, Any]: "data_source_format": "DELTA", "table_id": f"tid-{full}", "metastore_id": "ms-fake", - "owner": _USER, + "owner": t.get("owner", _USER), "columns": cols, "properties": t["properties"], "comment": t.get("comment"), @@ -1100,6 +1317,7 @@ def _delete_table(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: if full not in self.tables: return _err(404, "TABLE_DOES_NOT_EXIST", f"Table '{full}' does not exist.") del self.tables[full] + self.table_rows.pop(full, None) self.grants.pop(("table", full), None) self._drop_tags("tables", full) return 200, {} @@ -1341,6 +1559,115 @@ def _ws_delete(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: self.workspace_dirs.discard(path) return 200, {} + # -- files on volumes (the Files API) + def add_volume(self, full_name: str) -> None: + """Make ``/Volumes///`` exist; creates the catalog and schema.""" + cat, sch, _volume = full_name.split(".") + with self._lock: + self._ensure_schema(cat, sch) + self.volumes.add(full_name) + os.makedirs(self._fs_path("/Volumes/" + full_name.replace(".", "/")), exist_ok=True) + + def _fs_path(self, volume_path: str) -> Optional[str]: + """The temporary-directory path of a ``/Volumes/...`` path; ``None`` outside a volume + the fake holds, or on a path that climbs out of it.""" + parts = volume_path.split("/") + if parts[:2] != ["", "Volumes"] or len(parts) < 5: + return None + if ".." in parts or "." in parts[2:]: + return None + if ".".join(parts[2:5]) not in self.volumes: + return None + return os.path.join(self.files_root, *[p for p in parts[1:] if p]) + + def volume_files(self, prefix: str = "/Volumes") -> List[str]: + """Every file the Files API holds under ``prefix``, as ``/Volumes/...`` paths.""" + found: List[str] = [] + for root, _dirs, names in os.walk(self.files_root): + for name in names: + path = "/" + os.path.relpath(os.path.join(root, name), self.files_root) + if path.startswith(prefix.rstrip("/") + "/") or path == prefix: + found.append(path.replace(os.sep, "/")) + return sorted(found) + + def file_bytes(self, volume_path: str) -> bytes: + path = self._fs_path(volume_path) + assert path is not None, volume_path + with open(path, "rb") as fh: + return fh.read() + + def rows(self, full_name: str) -> List[Dict[str, Any]]: + """The rows the last INSERT OVERWRITE left in the table.""" + return list(self.table_rows.get(full_name, [])) + + def _fs_target(self, method: str, m: Any) -> Tuple[Optional[str], Optional[Tuple[int, Any]]]: + path = m.group(1) + self.file_ops.append((method, path)) + real = self._fs_path(path) + if real is None: + return None, _err(404, NOT_FOUND, f"The path {path} is not in a volume that exists.") + return real, None + + def _fs_mkdir(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + real, failed = self._fs_target("PUT-DIR", m) + if failed: + return failed + os.makedirs(real, exist_ok=True) + return 204, None + + def _fs_put(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + real, failed = self._fs_target("PUT", m) + if failed: + return failed + overwrite = (q.get("overwrite") or ["true"])[0].lower() != "false" + if os.path.exists(real) and not overwrite: + return _err(409, "ALREADY_EXISTS", f"The file {m.group(1)} already exists.") + os.makedirs(os.path.dirname(real), exist_ok=True) + with open(real, "wb") as fh: + fh.write(b.get("_raw") or b"") + return 204, None + + def _fs_delete(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + real, failed = self._fs_target("DELETE", m) + if failed: + return failed + if not os.path.isfile(real): + return _err(404, NOT_FOUND, f"The file {m.group(1)} does not exist.") + os.remove(real) + return 204, None + + def _fs_rmdir(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + real, failed = self._fs_target("DELETE-DIR", m) + if failed: + return failed + if not os.path.isdir(real): + return _err(404, NOT_FOUND, f"The directory {m.group(1)} does not exist.") + if os.listdir(real): + return _err(400, "FAILED_PRECONDITION", f"The directory {m.group(1)} is not empty.") + os.rmdir(real) + return 204, None + + def _fs_list(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: + real, failed = self._fs_target("LIST", m) + if failed: + return failed + if not os.path.isdir(real): + return _err(404, NOT_FOUND, f"The directory {m.group(1)} does not exist.") + base = m.group(1).rstrip("/") + contents = [] + for name in sorted(os.listdir(real)): + full = os.path.join(real, name) + entry: Dict[str, Any] = { + "name": name, + "path": f"{base}/{name}", + "is_directory": os.path.isdir(full), + } + if os.path.isfile(full): + entry["file_size"] = os.path.getsize(full) + entry["last_modified"] = int(os.path.getmtime(full) * 1000) + contents.append(entry) + return 200, {"contents": contents} + # -- warehouse / sql def _get_wh(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: wid = m.group(1) @@ -1361,7 +1688,14 @@ def _exec_sql(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: self.sql_log.append(sql) self.timeline.append(("sql", sql)) injected = next((msg for sub, msg in self.failing if sub in sql), None) - ok, err = (False, injected) if injected is not None else _execute(self, sql) + data = None if injected is not None else _data_statement(self, sql) + if injected is not None: + ok, err, answer = False, injected, {} + elif data is not None: + ok, err, answer = data + else: + ok, err = _execute(self, sql) + answer = {} sid = f"stmt-{len(self.statements) + 1}" if ok: doc: Dict[str, Any] = { @@ -1373,6 +1707,7 @@ def _exec_sql(self, m: Any, q: Any, b: Any) -> Tuple[int, Any]: "total_row_count": 0, }, "result": {"row_count": 0}, + **answer, } else: doc = { diff --git a/tests/iac/conftest.py b/tests/iac/conftest.py index f93a38f91..ee0bb16c2 100644 --- a/tests/iac/conftest.py +++ b/tests/iac/conftest.py @@ -791,13 +791,13 @@ def _localstack_ready() -> tuple[bool, str]: if os.environ.get("FLUID_IAC_LIVE_LOCALSTACK", "").strip().lower() not in _TRUE: return False, "live LocalStack tests are opt-in — set FLUID_IAC_LIVE_LOCALSTACK=1" try: - import requests # noqa: F401 — verify present + import httpx # noqa: F401 — verify present except ImportError: - return False, "the `requests` package is not installed" + return False, "the `httpx` package is not installed" try: - import requests + import httpx - resp = requests.get(f"{LOCALSTACK_ENDPOINT}/_localstack/health", timeout=5) + resp = httpx.get(f"{LOCALSTACK_ENDPOINT}/_localstack/health", timeout=5) if resp.status_code != 200: return False, f"LocalStack health at {LOCALSTACK_ENDPOINT} returned {resp.status_code}" except Exception as exc: # noqa: BLE001 @@ -873,11 +873,11 @@ def _localstack_reset(endpoint: str = LOCALSTACK_ENDPOINT) -> None: (S3 buckets, Glue databases, Lambdas, …) so the next test sees an empty account. The default account ID stays ``000000000000``. """ - import requests + import httpx with contextlib.suppress(Exception): # POST is the documented method; GET works on some Pro versions. - requests.post(f"{endpoint}/_localstack/state/reset", timeout=15) + httpx.post(f"{endpoint}/_localstack/state/reset", timeout=15) class LocalStackProject: diff --git a/tests/iac/test_iac_aws_action_order.py b/tests/iac/test_iac_aws_action_order.py new file mode 100644 index 000000000..b243a5f77 --- /dev/null +++ b/tests/iac/test_iac_aws_action_order.py @@ -0,0 +1,32 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The AWS planner keeps the order it planned actions in. + +``DependencyGraph.nodes`` was a ``set``, so the order of independent actions, and with it +the emitted module, followed Python's hash seed: a Lambda permission or schedule target +named its function by literal ARN in one run and by reference in the next. +""" + +from __future__ import annotations + + +class TestActionOrderIsStable: + def test_independent_actions_keep_the_order_they_were_planned_in(self): + from fluid_build.providers.aws.util.dependencies import order_actions_by_dependencies + + actions = [{"id": f"a{i}", "op": "x.y"} for i in range(40)] + assert [a["id"] for a in order_actions_by_dependencies(actions)] == [ + a["id"] for a in actions + ] diff --git a/tests/iac/test_iac_aws_apply_engine_moto.py b/tests/iac/test_iac_aws_apply_engine_moto.py index dfe22f8bf..7d8d530f2 100644 --- a/tests/iac/test_iac_aws_apply_engine_moto.py +++ b/tests/iac/test_iac_aws_apply_engine_moto.py @@ -82,7 +82,7 @@ def moto_endpoint(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: the backends. The ``/moto-api/reset`` admin endpoint clears all state, giving each test the clean account it expects. """ - import requests + import httpx from moto.server import ThreadedMotoServer server = ThreadedMotoServer(port=0, verbose=False) @@ -94,7 +94,7 @@ def moto_endpoint(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: # Reset before yielding — also covers state leaked in by a previous # test running in the same process. with contextlib.suppress(Exception): - requests.post(f"{endpoint}/moto-api/reset", timeout=5) + httpx.post(f"{endpoint}/moto-api/reset", timeout=5) yield endpoint finally: server.stop() diff --git a/tests/iac/test_iac_aws_schedule_names.py b/tests/iac/test_iac_aws_schedule_names.py new file mode 100644 index 000000000..eac68a908 --- /dev/null +++ b/tests/iac/test_iac_aws_schedule_names.py @@ -0,0 +1,642 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An AWS contract's schedule is never dropped silently, and is honestly labelled. + +``fluid generate iac`` on a contract whose id carried a dot (the schema's own +example is ``gold.hr.employee_360_v1``) logged "Planned EventBridge schedule" +and wrote a module with no ``aws_lambda_function``, ``aws_lambda_permission`` or +``aws_scheduler_schedule``, exit 0: the planner named the Lambda +``fluid-workflow-`` raw, the plan validator refused the dot, and +``native_actions`` swallowed the refusal at DEBUG. ``fluid apply`` shares that +function, so it omitted the schedule the same way. +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import logging +import re +import sys +import types +from pathlib import Path +from typing import Any, Dict + +import pytest +import yaml + +from fluid_build.cli import generate_iac +from fluid_build.cli._apply_opentofu_engine import emit_module +from fluid_build.cli._common import CLIError +from fluid_build.providers._plan_validation import PlanValidationError +from fluid_build.providers.aws.plan.schedule import SchedulePlanner +from fluid_build.providers.aws.util.config import AwsAccountUnresolvedError +from fluid_build.providers.aws.util.names import ( + EVENTBRIDGE_RULE_NAME, + LAMBDA_FUNCTION_NAME, + LAMBDA_STATEMENT_ID, + MWAA_ENVIRONMENT_NAME, + SCHEDULER_SCHEDULE_NAME, + STATE_MACHINE_NAME, + bounded_aws_name, +) +from fluid_build.providers.base import ProviderError + +pytestmark = [pytest.mark.unit, pytest.mark.provider] + +ACCOUNT = "123456789012" +REGION = "us-east-1" +DOTTED = "sales.orders_daily" +LONG = "gold.finance." + "quarterly_revenue_recognition_" * 4 + "v1" + +_LAMBDA_NAME = re.compile(r"[a-zA-Z0-9\-_]{1,64}") +_STATEMENT_ID = re.compile(r"[a-zA-Z0-9\-_]{1,100}") +_SCHEDULE_NAME = re.compile(r"[a-zA-Z0-9\-_.]{1,64}") + + +def _contract(contract_id: str = DOTTED, **orchestration: Any) -> Dict[str, Any]: + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": contract_id, + "name": "Orders", + "metadata": {"layer": "Silver", "owner": {"team": "data", "email": "d@example.com"}}, + "orchestration": {"engine": "none", "schedule": "cron(0 6 * * ? *)", **orchestration}, + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": { + "platform": "aws", + "format": "parquet", + "location": { + "bucket": "offline-lake", + "path": "orders/", + "database": "sales", + "table": "orders", + "region": REGION, + }, + }, + "contract": {"schema": [{"name": "id", "type": "string"}]}, + } + ], + } + + +def _args(contract: Dict[str, Any], tmp_path: Path) -> argparse.Namespace: + path = tmp_path / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(contract, sort_keys=False), encoding="utf-8") + return argparse.Namespace( + contract=str(path), provider="auto", out=str(tmp_path / "out"), env=None + ) + + +@pytest.fixture(autouse=True) +def _account(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AWS_ACCOUNT_ID", ACCOUNT) + + +class TestBoundedAwsName: + @staticmethod + def _digest(raw: str) -> str: + return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:8] + + def test_a_name_the_rule_accepts_is_returned_as_written(self): + # Nothing already deployed under such a name is renamed. + assert bounded_aws_name("fluid-workflow-sales_orders", LAMBDA_FUNCTION_NAME) == ( + "fluid-workflow-sales_orders" + ) + + def test_a_name_that_was_sanitised_ends_in_the_hash_of_the_raw_name(self): + raw = "fluid-workflow-sales.orders" + assert bounded_aws_name(raw, LAMBDA_FUNCTION_NAME) == ( + f"fluid-workflow-sales-orders-{self._digest(raw)}" + ) + + def test_two_ids_that_sanitise_alike_do_not_collide(self): + dotted = bounded_aws_name("fluid-workflow-a.b", LAMBDA_FUNCTION_NAME) + dashed = bounded_aws_name("fluid-workflow-a-b", LAMBDA_FUNCTION_NAME) + assert dashed == "fluid-workflow-a-b" + assert dotted != dashed + + def test_each_service_keeps_the_characters_it_allows(self): + raw = "fluid-x.y_z" + assert bounded_aws_name(raw, SCHEDULER_SCHEDULE_NAME) == raw # `.` is legal here + assert bounded_aws_name(raw, EVENTBRIDGE_RULE_NAME) == raw + assert bounded_aws_name(raw, LAMBDA_FUNCTION_NAME) == f"fluid-x-y_z-{self._digest(raw)}" + assert bounded_aws_name("a:b/c d", STATE_MACHINE_NAME) == ( + f"a-b-c-d-{self._digest('a:b/c d')}" + ) + assert bounded_aws_name("mwaa-a.b", MWAA_ENVIRONMENT_NAME) == ( + f"mwaa-a-b-{self._digest('mwaa-a.b')}" + ) + + @pytest.mark.parametrize( + "rule", + [ + LAMBDA_FUNCTION_NAME, + LAMBDA_STATEMENT_ID, + SCHEDULER_SCHEDULE_NAME, + EVENTBRIDGE_RULE_NAME, + MWAA_ENVIRONMENT_NAME, + STATE_MACHINE_NAME, + ], + ) + def test_a_long_name_is_cut_to_the_limit_deterministically(self, rule): + limit = rule[1] + raw = "fluid-workflow-" + "x" * 300 + name = bounded_aws_name(raw, rule) + assert len(name) == limit + assert bounded_aws_name(raw, rule) == name + + def test_two_long_names_sharing_a_prefix_do_not_collide(self): + a = bounded_aws_name("fluid-workflow-" + "x" * 80 + "-a", LAMBDA_FUNCTION_NAME) + b = bounded_aws_name("fluid-workflow-" + "x" * 80 + "-b", LAMBDA_FUNCTION_NAME) + assert len(a) == len(b) == 64 + assert a != b + + +class TestPlannedScheduleNames: + @pytest.mark.parametrize("contract_id", [DOTTED, LONG], ids=["dotted", "long"]) + def test_every_derived_name_is_valid_and_the_target_matches(self, contract_id): + actions = SchedulePlanner(ACCOUNT, REGION).plan_schedule_actions(_contract(contract_id)) + by_op = {a["op"]: a for a in actions} + function = by_op["lambda.ensure_function"] + schedule = by_op["eventbridge.ensure_schedule"] + permission = by_op["lambda.add_permission"] + + assert _LAMBDA_NAME.fullmatch(function["function_name"]) + assert _SCHEDULE_NAME.fullmatch(schedule["schedule_name"]) + assert _STATEMENT_ID.fullmatch(permission["statement_id"]) + assert schedule["target"]["arn"].endswith(f":function:{function['function_name']}") + assert permission["function_name"] == function["function_name"] + assert permission["source_arn"].endswith(f"/default/{schedule['schedule_name']}") + + def test_the_names_are_deterministic(self): + planner = SchedulePlanner(ACCOUNT, REGION) + assert planner.plan_schedule_actions(_contract(LONG)) == planner.plan_schedule_actions( + _contract(LONG) + ) + + def test_two_long_ids_with_a_shared_prefix_get_distinct_functions(self): + planner = SchedulePlanner(ACCOUNT, REGION) + a = planner.plan_schedule_actions(_contract(LONG + "_a"))[0]["function_name"] + b = planner.plan_schedule_actions(_contract(LONG + "_b"))[0]["function_name"] + assert a != b + + def test_trigger_lambdas_invoke_the_sanitised_workflow_function(self): + contract = _contract( + LONG, + triggers=[ + {"type": "s3", "id": "land.csv", "bucket": "b"}, + {"type": "sqs", "id": "q", "queue": "q"}, + {"type": "eventbridge", "id": "evt.1", "eventPattern": {"source": ["x"]}}, + ], + ) + actions = SchedulePlanner(ACCOUNT, REGION).plan_schedule_actions(contract) + workflow = next(a for a in actions if a["id"].startswith("lambda_workflow_")) + for action in actions: + if action["op"] == "lambda.ensure_function": + assert _LAMBDA_NAME.fullmatch(action["function_name"]) + if action["op"] == "lambda.add_permission": + assert _STATEMENT_ID.fullmatch(action["statement_id"]) + if action["id"].startswith("lambda_") and "_trigger_" in action["id"]: + assert f"FunctionName='{workflow['function_name']}'" in action["code"]["ZipFile"] + + @pytest.mark.parametrize("contract_id", [DOTTED, LONG], ids=["dotted", "long"]) + def test_generate_iac_emits_the_schedule(self, contract_id, tmp_path): + args = _args(_contract(contract_id), tmp_path) + assert generate_iac.run(args, logging.getLogger("t")) == 0 + module = json.loads((tmp_path / "out" / "main.tf.json").read_text(encoding="utf-8")) + resources = module["resource"] + ((fn_key, function),) = resources["aws_lambda_function"].items() + ((_, schedule),) = resources["aws_scheduler_schedule"].items() + ((_, permission),) = resources["aws_lambda_permission"].items() + + assert _LAMBDA_NAME.fullmatch(function["function_name"]) + assert _SCHEDULE_NAME.fullmatch(schedule["name"]) + assert _STATEMENT_ID.fullmatch(permission["statement_id"]) + # The schedule and the permission reference the emitted function, not a literal. + assert schedule["target"]["arn"] == f"${{aws_lambda_function.{fn_key}.arn}}" + assert permission["function_name"] == f"${{aws_lambda_function.{fn_key}.function_name}}" + + +class TestEveryNameStartsWithALetter: + """MWAA requires a leading letter; every name here carries a literal prefix + (``fluid-``, ``mwaa-``, ``Allow...``) that survives both the replacement and + the cut, whatever the id starts with.""" + + HOSTILE = "9 weird/id:" + "v" * 120 + + def test_names_built_from_a_hostile_id(self): + planner = SchedulePlanner(ACCOUNT, REGION) + triggers = [ + {"type": "s3", "id": "1.x", "bucket": "b"}, + {"type": "sqs", "id": "2.y", "queue": "q"}, + {"type": "eventbridge", "id": "3.z", "eventPattern": {}}, + ] + tasks = [{"taskId": "a", "type": "provider_action", "action": "x"}] + actions = planner.plan_schedule_actions( + _contract(self.HOSTILE, triggers=triggers, engine="step-functions", tasks=tasks) + ) + actions += planner.plan_schedule_actions(_contract(self.HOSTILE, engine="mwaa")) + limits = { + "function_name": 64, + "statement_id": 100, + "schedule_name": 64, + "rule_name": 64, + "state_machine_name": 80, + "environment_name": 80, + } + names = [] + for a in actions: + for key, limit in limits.items(): + if key in a: + assert len(a[key]) <= limit, (key, a[key]) + names.append(a[key]) + assert len(names) >= 10 + assert all(n[0].isalpha() for n in names), names + # MWAA: a letter first, then letters, digits, ``-`` and ``_``, up to 80. + env = [a["environment_name"] for a in actions if "environment_name" in a] + assert env and all(re.fullmatch(r"[a-zA-Z][0-9a-zA-Z\-_]{0,79}", n) for n in env) + + +class _Refuses: + """A stand-in native provider whose ``plan`` raises ``exc``.""" + + def __init__(self, exc: BaseException): + self._exc = exc + + def plan(self, contract, **_): + raise self._exc + + +@pytest.fixture +def planner_raises(monkeypatch: pytest.MonkeyPatch): + def install(exc: BaseException) -> None: + monkeypatch.setattr("fluid_build.cli._common.build_provider", lambda *a, **k: _Refuses(exc)) + + return install + + +class TestAPlannerThatRanAndRefusedIsLoud: + def test_a_plan_validation_failure_stops_native_actions(self, planner_raises): + planner_raises(PlanValidationError("Plan validation failed:\nAction 1: bad name")) + with pytest.raises(PlanValidationError): + generate_iac.native_actions(_contract(), logging.getLogger("t"), offline=True) + + def test_it_is_recognised_through_the_cause_chain(self, planner_raises): + try: + raise PlanValidationError("nope") + except PlanValidationError as inner: + try: + raise RuntimeError("wrapped") from inner + except RuntimeError as outer: + assert generate_iac._is_plan_validation_refusal(outer) is True + + def test_the_message_is_not_what_is_matched(self): + assert ( + generate_iac._is_plan_validation_refusal(ProviderError("Plan validation failed: ...")) + is False + ) + + def test_generate_iac_exits_non_zero(self, planner_raises, tmp_path): + planner_raises(PlanValidationError("Plan validation failed:\nAction 1: bad name")) + with pytest.raises(CLIError) as excinfo: + generate_iac.run(_args(_contract(), tmp_path), logging.getLogger("t")) + assert excinfo.value.event == "generate_iac_failed" + assert "Plan validation failed" in excinfo.value.context["error"] + assert not (tmp_path / "out" / "main.tf.json").exists() + + def test_the_apply_native_path_raises(self, planner_raises): + planner_raises(PlanValidationError("Plan validation failed:\nAction 1: bad name")) + with pytest.raises(PlanValidationError): + emit_module(object(), _contract(), object(), logging.getLogger("t")) + + def test_a_real_planner_refusal_stops_generate_iac(self, tmp_path): + contract = _contract() + contract["exposes"][0]["binding"]["location"]["bucket"] = "Not_A_Bucket" + with pytest.raises(CLIError) as excinfo: + generate_iac.run(_args(contract, tmp_path), logging.getLogger("t")) + assert excinfo.value.event == "generate_iac_failed" + assert "Plan validation failed" in excinfo.value.context["error"] + + def test_a_sovereignty_refusal_still_propagates(self, planner_raises): + from fluid_build.cli._errors import SovereigntyViolationError + + planner_raises( + SovereigntyViolationError.for_connector(connector="aws:us-east-1", jurisdiction="EU") + ) + with pytest.raises(SovereigntyViolationError): + generate_iac.native_actions(_contract(), logging.getLogger("t"), offline=True) + + @pytest.mark.parametrize("exc", [ImportError("no boto3"), RuntimeError("no credentials")]) + def test_a_planner_that_could_not_run_is_still_best_effort(self, planner_raises, exc): + planner_raises(exc) + assert generate_iac.native_actions(_contract(), logging.getLogger("t")) == [] + + +class TestThePlaceholderLambdaIsLabelled: + @staticmethod + def _warnings(caplog) -> list: + events = [] + for record in caplog.records: + if record.levelno == logging.WARNING and "schedule_placeholder_lambda" in ( + record.getMessage() + ): + events.append(json.loads(record.getMessage())) + return events + + def test_a_schedule_warns_that_the_lambda_is_a_placeholder(self, caplog): + logger = logging.getLogger("t") + with caplog.at_level(logging.WARNING, logger="t"): + generate_iac.native_actions(_contract(), logger, offline=True) + (event,) = self._warnings(caplog) + assert event["contract_id"] == DOTTED + assert "placeholder" in event["message"] + assert "does not run" in event["message"] + assert f"role/{'fluid-workflow-execution'}" in event["message"] + assert "role/EventBridgeSchedulerRole" in event["message"] + assert "must already exist" in event["message"] + + def test_apply_native_path_warns_too(self, caplog): + """``fluid apply`` plans through ``native_actions`` without ``offline``.""" + with caplog.at_level(logging.WARNING, logger="t"): + generate_iac.native_actions(_contract(), logging.getLogger("t")) + assert len(self._warnings(caplog)) == 1 + + def test_generate_iac_shows_it(self, caplog, tmp_path): + with caplog.at_level(logging.WARNING): + generate_iac.run(_args(_contract(), tmp_path), logging.getLogger("t")) + assert len(self._warnings(caplog)) == 1 + + def test_no_schedule_no_warning(self, caplog): + contract = _contract() + del contract["orchestration"] + with caplog.at_level(logging.WARNING, logger="t"): + generate_iac.native_actions(contract, logging.getLogger("t"), offline=True) + assert self._warnings(caplog) == [] + + def test_an_mwaa_schedule_plans_no_placeholder_lambda(self, caplog): + with caplog.at_level(logging.WARNING, logger="t"): + generate_iac.native_actions( + _contract(engine="mwaa"), logging.getLogger("t"), offline=True + ) + assert self._warnings(caplog) == [] + + +class TestContractValuesNeverBecomeGeneratedLambdaCode: + """A contract value reaches the generated Lambda source only as a literal. + + Spliced inside quotes (``'{contract_id}'``, ``json.loads('''{tasks}''')``), a + task param such as ``'''+__import__('os').system('id')+'''`` closed the + literal and ran on every schedule tick under ``fluid-workflow-execution``, + and an id with a trailing newline left a Lambda that was a SyntaxError. + """ + + EVIL = "'''+__import__('os').system('id')+'''" + HOSTILE_IDS = [ + EVIL, + "a'b\"c'''d\"\"\"e", + "back\\slash\\'", + "line\nbreak", + "trailing.newline\n", + "${aws_lambda_function.x.arn}{braces}{{x}}", + ] + #: Calls the generated sources are expected to make, by final name. + EXPECTED_CALLS = { + "loads", + "dumps", + "getLogger", + "setLevel", + "get", + "info", + "set", + "list", + "all", + "add", + "remove", + "client", + "invoke", + "RuntimeError", + "len", + } + + @staticmethod + def _sources(contract_id: str, param: str) -> Dict[str, str]: + planner = SchedulePlanner(ACCOUNT, REGION) + task = { + "taskId": param, + "type": "provider_action", + "action": "aws.glue.run_job", + "params": {"k": param}, + "dependsOn": [], + } + contract = {"id": contract_id, "name": "n"} + orchestration = {"tasks": [task]} + return { + "workflow": planner._generate_workflow_lambda_code(contract, orchestration)["ZipFile"], + "s3": planner._generate_s3_trigger_lambda_code(contract, {})["ZipFile"], + "dynamodb": planner._generate_dynamodb_trigger_lambda_code(contract, {})["ZipFile"], + "event": planner._generate_event_trigger_lambda_code(contract, {})["ZipFile"], + "sqs": planner._generate_sqs_trigger_lambda_code(contract, {})["ZipFile"], + } + + @staticmethod + def _callee(call: ast.Call) -> str: + func = call.func + return func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", "?") + + @pytest.mark.parametrize("contract_id", HOSTILE_IDS, ids=range(len(HOSTILE_IDS))) + @pytest.mark.parametrize("param", [EVIL, "p'\"\\\n${x}"], ids=["inject", "punctuation"]) + def test_the_source_parses_calls_only_what_it_should_and_round_trips(self, contract_id, param): + for kind, source in self._sources(contract_id, param).items(): + tree = ast.parse(source) # a SyntaxError here is the trailing-newline defect + callees = {self._callee(n) for n in ast.walk(tree) if isinstance(n, ast.Call)} + assert callees <= self.EXPECTED_CALLS, (kind, callees - self.EXPECTED_CALLS) + + strings = set() + for node in ast.walk(tree): + if isinstance(node, ast.Dict): + for key, value in zip(node.keys, node.values, strict=True): + if ( + isinstance(key, ast.Constant) + and key.value == "contract_id" + and isinstance(value, ast.Constant) + ): + strings.add(value.value) + if kind != "workflow": + assert strings == {contract_id}, kind + functions = [ + ast.literal_eval(kw.value) + for n in ast.walk(tree) + if isinstance(n, ast.Call) + for kw in n.keywords + if kw.arg == "FunctionName" + ] + assert functions == [SchedulePlanner.workflow_function_name(contract_id)] + + @pytest.mark.parametrize("contract_id", HOSTILE_IDS, ids=range(len(HOSTILE_IDS))) + def test_the_workflow_lambda_round_trips_its_tasks_and_default_id(self, contract_id): + param = self.EVIL + source = self._sources(contract_id, param)["workflow"] + boto3 = types.ModuleType("boto3") + namespace: Dict[str, Any] = {} + sys.modules["boto3"], saved = boto3, sys.modules.get("boto3") + try: + exec(compile(source, "", "exec"), namespace) # noqa: S102 + finally: + if saved is None: + del sys.modules["boto3"] + else: + sys.modules["boto3"] = saved + assert namespace["TASKS"] == [ + {"taskId": param, "action": "aws.glue.run_job", "params": {"k": param}, "dependsOn": []} + ] + # The handler's fallback id is the exact id, and running it runs no payload. + handler_defaults = [ + ast.literal_eval(n.args[1]) + for n in ast.walk(ast.parse(source)) + if isinstance(n, ast.Call) + and self._callee(n) == "get" + and len(n.args) == 2 + and isinstance(n.args[0], ast.Constant) + and n.args[0].value == "contract_id" + ] + assert handler_defaults == [contract_id] + assert namespace["handler"]({}, None)["statusCode"] == 200 + + +class TestEveryPlanningFailureIsARefusalNotADrop: + def test_an_orchestration_task_the_planner_rejects_stops_generate_iac(self, tmp_path): + contract = _contract( + engine="airflow", + tasks=[ + {"taskId": "a", "type": "provider_action", "action": "gcp.bq.run", "params": {}} + ], + ) + with pytest.raises(CLIError) as excinfo: + generate_iac.run(_args(contract, tmp_path), logging.getLogger("t")) + assert excinfo.value.event == "generate_iac_failed" + assert "Orchestration planning failed" in excinfo.value.context["error"] + + def test_a_schedule_planner_failure_is_a_plan_validation_error(self, monkeypatch): + from fluid_build.providers.aws.provider import AwsProvider + + def boom(*args, **kwargs): + raise ValueError("bad schedule") + + monkeypatch.setattr("fluid_build.providers.aws.plan.schedule.plan_schedule_actions", boom) + provider = AwsProvider(account_id=ACCOUNT, region=REGION) + with pytest.raises(PlanValidationError, match="Schedule planning failed"): + provider._plan_schedule(_contract()) + + +class TestS3BucketNames: + @pytest.fixture + def validate(self): + from fluid_build.providers.aws.util.validation import ResourceValidator + + return ResourceValidator(ACCOUNT, REGION).validate_s3_bucket_name + + @pytest.mark.parametrize( + "bucket", ["acme.orders.lake", "my-bucket.v2", "a1b", "data.2026-lake", "x" * 63] + ) + def test_a_name_s3_allows_is_accepted(self, validate, bucket): + assert validate(bucket) == (True, None) + + @pytest.mark.parametrize( + "bucket", + [ + "a..b", + "a.-b", + "a-.b", + ".abc", + "abc.", + "-abc", + "192.168.0.1", + "Upper.case", + "has_underscore", + "ab", + "x" * 64, + "xn--bucket", + ], + ) + def test_a_name_s3_refuses_is_refused(self, validate, bucket): + assert validate(bucket)[0] is False + + def test_generate_iac_accepts_a_dotted_bucket(self, tmp_path): + contract = _contract() + contract["exposes"][0]["binding"]["location"]["bucket"] = "acme.orders.lake" + assert generate_iac.run(_args(contract, tmp_path), logging.getLogger("t")) == 0 + + +class TestTheUnresolvedAccountMessage: + @staticmethod + def _provider_without_account(monkeypatch): + from fluid_build.providers.aws.provider import AwsProvider + + monkeypatch.setattr( + "fluid_build.providers.aws.util.config.resolve_account_and_region", + lambda account_id=None, region=None: (None, REGION), + ) + return AwsProvider(region=REGION) + + def test_missing_boto3_is_named_with_the_install_command(self, monkeypatch): + provider = self._provider_without_account(monkeypatch) + monkeypatch.setattr("fluid_build.providers.aws.util.config._boto3_missing", lambda: True) + with pytest.raises(AwsAccountUnresolvedError) as excinfo: + provider.plan(_contract()) + message = str(excinfo.value) + assert "boto3 is not installed" in message + assert "pip install 'data-product-forge[aws]'" in message + assert "AWS_ACCOUNT_ID" in message + + def test_with_boto3_present_it_points_at_credentials(self, monkeypatch): + provider = self._provider_without_account(monkeypatch) + monkeypatch.setattr("fluid_build.providers.aws.util.config._boto3_missing", lambda: False) + with pytest.raises(AwsAccountUnresolvedError) as excinfo: + provider.plan(_contract()) + message = str(excinfo.value) + assert "sts:GetCallerIdentity" in message + assert "boto3" not in message + + def test_a_contract_without_orchestration_still_applies_its_exposes(self, monkeypatch): + """Nothing in it needs the native plan, and tofu resolves the account itself, + so the planner only reports that it could not run, as on main.""" + provider = self._provider_without_account(monkeypatch) + contract = _contract() + del contract["orchestration"] + with pytest.raises(ProviderError) as excinfo: + provider.plan(contract) + assert not isinstance(excinfo.value, AwsAccountUnresolvedError) + monkeypatch.setattr("fluid_build.cli._common.build_provider", lambda *a, **k: provider) + assert generate_iac.native_actions(contract, logging.getLogger("t")) == [] + + def test_sovereignty_is_checked_before_the_account(self, monkeypatch): + provider = self._provider_without_account(monkeypatch) + contract = _contract() + del contract["orchestration"] + contract["sovereignty"] = { + "jurisdiction": "EU", + "allowedRegions": ["eu-central-1"], + "enforcementMode": "strict", + } + monkeypatch.setattr("fluid_build.cli._common.build_provider", lambda *a, **k: provider) + with pytest.raises(Exception) as excinfo: + generate_iac.native_actions(contract, logging.getLogger("t")) + assert generate_iac._is_sovereignty_refusal(excinfo.value) diff --git a/tests/iac/test_iac_aws_unemitted_ops.py b/tests/iac/test_iac_aws_unemitted_ops.py index 89343330b..f9757db83 100644 --- a/tests/iac/test_iac_aws_unemitted_ops.py +++ b/tests/iac/test_iac_aws_unemitted_ops.py @@ -300,9 +300,9 @@ def _cyclic_tasks() -> Dict[str, Any]: class TestThePlannerFailing: """The planner failing on another field does not let the environment drop. - ``native_actions`` returns ``[]`` when the native planner raises (it logs - that at DEBUG), so the emitter saw no ``mwaa.ensure_environment`` and wrote - the data plane alone, exit 0. + ``native_actions`` returns ``[]`` when the native planner raises something + that is not a refusal (it logs that at DEBUG), so the emitter saw no + ``mwaa.ensure_environment`` and wrote the data plane alone, exit 0. """ @pytest.mark.parametrize("make", [_upper_case_table, _cyclic_tasks], ids=["glue-name", "cycle"]) @@ -322,7 +322,12 @@ def test_generate_iac_is_refused(self, no_aws, monkeypatch, tmp_path): generate_iac.run( _args(_write(tmp_path, _upper_case_table()), out), logging.getLogger("t") ) - assert excinfo.value.context["kind"] == "mwaa-environment-not-emitted" + # The planner ran and refused its own plan, so ``native_actions`` raises + # it (``PlanValidationError``) before the emitter's backstop is reached; + # the cycle case, a planner failure that is not a refusal of the plan, + # still gets that backstop (above). + assert excinfo.value.event == "generate_iac_failed" + assert "Plan validation failed" in excinfo.value.context["error"] assert not out.exists() @pytest.mark.parametrize( diff --git a/tests/iac/test_iac_column_type_gate.py b/tests/iac/test_iac_column_type_gate.py new file mode 100644 index 000000000..3a1831ae4 --- /dev/null +++ b/tests/iac/test_iac_column_type_gate.py @@ -0,0 +1,540 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""No column type reaches AWS Glue or BigQuery silently degraded or invalid. + +Every type the bundled schemas admit is classified on both clouds: written as a +valid target type, written as a fallback and warned about, or refused. The +refusal is the same one at ``fluid validate``, ``generate iac`` and ``apply``, +and ``validate`` agrees with the emitter on which contracts pass. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import re +from pathlib import Path + +import pytest +import yaml + +import fluid_build +from fluid_build.iac import get_iac_plugin +from fluid_build.iac.base import UnsupportedBindingError +from fluid_build.iac.column_types import EXACT, LOSSY, REFUSED +from fluid_build.iac.governance_validation import validate_governance +from fluid_build.iac.providers.aws import _hive_outcome, _hive_type, validate_aws_column_types +from fluid_build.iac.providers.gcp import ( + _bq_outcome, + validate_bigquery_column_types, + validate_gcp_binding, +) + +pytestmark = [pytest.mark.unit, pytest.mark.provider] + +_SCHEMAS = Path(fluid_build.__file__).parent / "schemas" + +# Every type the BigQuery schema JSON accepts, standard and legacy spellings. +_BQ_VALID = { + "STRING", "BYTES", "INTEGER", "INT64", "FLOAT", "FLOAT64", "NUMERIC", "BIGNUMERIC", + "BOOLEAN", "BOOL", "TIMESTAMP", "DATE", "TIME", "DATETIME", "GEOGRAPHY", "JSON", + "RECORD", "STRUCT", "INTERVAL", "RANGE", +} # fmt: skip +# A Hive/Athena column type this emitter writes. +_HIVE_VALID = re.compile( + r"^(string|tinyint|smallint|int|bigint|float|double|boolean|date|timestamp|binary" + r"|decimal\((?:[1-9]|[12][0-9]|3[0-8]),\d+\))$" +) + + +def _enum(version: str) -> set: + schema = json.loads((_SCHEMAS / f"fluid-schema-{version}.json").read_text(encoding="utf-8")) + return set(schema["$defs"]["column"]["properties"]["type"]["anyOf"][0]["enum"]) + + +_ALL_TYPES = sorted(_enum("0.7.6") | _enum("0.7.7")) + +_STRUCTURED = {"array", "struct", "map", "object", "record", "row"} +_GLUE_LOSSY = {"json", "jsonb", "variant", "super", "time", "interval", "bignumeric"} +_GLUE_REFUSED = _STRUCTURED | {"geography", "geometry", "geom", "point", "hll"} +_BQ_LOSSY = {"geometry", "geom"} +_BQ_REFUSED = _STRUCTURED + + +def test_the_enums_are_the_79_types_this_gate_was_written_against(): + # A new enum member lands in the parametrised tests below and has to be + # classified; this only notes the size so a change is deliberate. + assert len(_ALL_TYPES) == 79 + + +@pytest.mark.parametrize("fluid_type", _ALL_TYPES) +def test_every_schema_type_is_valid_warned_or_refused_on_glue(fluid_type): + outcome = _hive_outcome(fluid_type) + if outcome.status == REFUSED: + assert outcome.target is None + assert outcome.why and outcome.fix + else: + assert _HIVE_VALID.match(outcome.target), (fluid_type, outcome) + assert (outcome.status == LOSSY) == (fluid_type in _GLUE_LOSSY) + assert (outcome.status == REFUSED) == (fluid_type in _GLUE_REFUSED) + + +@pytest.mark.parametrize("fluid_type", _ALL_TYPES) +def test_every_schema_type_is_valid_warned_or_refused_on_bigquery(fluid_type): + outcome = _bq_outcome(fluid_type) + if outcome.status == REFUSED: + assert outcome.target is None + assert outcome.why and outcome.fix + else: + assert outcome.target in _BQ_VALID, (fluid_type, outcome) + assert (outcome.status == LOSSY) == (fluid_type in _BQ_LOSSY) + assert (outcome.status == REFUSED) == (fluid_type in _BQ_REFUSED) + + +@pytest.mark.parametrize( + "fluid_type, expected", + [ + ("int2", "smallint"), + ("int4", "int"), + ("int8", "bigint"), + ("int16", "smallint"), + ("tinyint", "tinyint"), + ("smallint", "smallint"), + ("mediumint", "int"), + ("longint", "bigint"), + ("serial", "int"), + ("bigserial", "bigint"), + ("float4", "float"), + ("float8", "double"), + ("real", "float"), + ("dec", "decimal(38,9)"), + ("number", "decimal(38,9)"), + ("number(12,2)", "decimal(12,2)"), + ("dec(10)", "decimal(10)"), + ("money", "decimal(19,4)"), + ("bit", "boolean"), + ("datetime2", "timestamp"), + ("smalldatetime", "timestamp"), + ("timestamptz", "timestamp"), + ("timestamp_tz", "timestamp"), + ("timestamp_ltz", "timestamp"), + ("timestamp_ntz", "timestamp"), + ("timestampntz", "timestamp"), + ("TIMESTAMP WITH TIME ZONE", "timestamp"), + ("double precision", "double"), + ("varbinary", "binary"), + ("blob", "binary"), + ("bytea", "binary"), + ("raw", "binary"), + ("year", "smallint"), + ("varchar(255)", "string"), + ("float(53)", "double"), + ("float(24)", "float"), + ("decimal(10, 2)", "decimal(10,2)"), + ], +) +def test_a_scalar_with_a_hive_form_gets_it(fluid_type, expected): + outcome = _hive_outcome(fluid_type) + assert (outcome.target, outcome.status) == (expected, EXACT) + assert _hive_type(fluid_type) == expected + + +@pytest.mark.parametrize("fluid_type", ["json", "jsonb", "variant", "super", "time", "interval"]) +def test_a_type_hive_cannot_hold_is_written_as_string_and_warned(fluid_type): + outcome = _hive_outcome(fluid_type) + assert (outcome.target, outcome.status) == ("string", LOSSY) + + +@pytest.mark.parametrize( + "fluid_type", ["decimal(39,2)", "decimal(10,11)", "numeric(0)", "decimal(a,b)", "bit(8)"] +) +def test_a_type_hive_would_reject_is_refused(fluid_type): + assert _hive_outcome(fluid_type).status == REFUSED + + +@pytest.mark.parametrize( + "fluid_type", ["array", "array(string)", "ARRAY", "struct", "map(string,int)", "row(a)"] +) +def test_a_structured_type_is_refused_whatever_its_parameters(fluid_type): + assert _hive_outcome(fluid_type).status == REFUSED + assert _bq_outcome(fluid_type).status == REFUSED + + +def test_a_type_nothing_maps_is_refused_on_both_clouds(): + assert _hive_outcome("geometryz").status == REFUSED + assert _bq_outcome("geometryz").status == REFUSED + + +def test_the_lenient_views_stay_total_for_verify_and_diff(): + """``fluid verify`` and ``fluid diff`` compare with these; they never raise.""" + from fluid_build.iac.providers.gcp import _bq_type + + assert _hive_type("array") == "string" + assert _bq_type("array") == "ARRAY" + assert _bq_type("geometryz") == "GEOMETRYZ" + assert _bq_type(None) == "STRING" + + +# ── The contracts: validate, generate and apply agree ──────────────────────── + +_AWS_LOC = { + "database": "demo", + "table": "orders", + "bucket": "acme-demo", + "path": "orders/", + "region": "us-east-1", +} +_GCP_LOC = {"project": "acme", "dataset": "demo", "table": "orders"} + +_BAD = [ + ("id", "string"), + ("tags", "array"), + ("payload", "json"), + ("small", "smallint"), + ("tz", "timestamptz"), + ("addr", "struct"), +] +_GOOD = [c for c in _BAD if c[1] not in ("array", "struct")] + + +def _contract(platform, columns, binding=None): + fmt, loc = ("parquet", _AWS_LOC) if platform == "aws" else ("bigquery_table", _GCP_LOC) + loc = dict(loc) + return { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "demo.types", + "name": "Types demo", + "description": "column type gate", + "domain": "demo", + "metadata": {"layer": "Silver", "owner": {"team": "data", "email": "data@example.com"}}, + "exposes": [ + { + "exposeId": "orders", + "version": "1.0.0", + "kind": "table", + "binding": binding or {"platform": platform, "format": fmt, "location": loc}, + "contract": {"schema": [{"name": n, "type": t} for n, t in columns]}, + } + ], + } + + +def _gate(contract): + """What ``fluid validate`` reports for the contract's column types, both clouds.""" + gcp_errors, gcp_warnings = validate_gcp_binding(contract) + gov_errors, gov_warnings = validate_governance(contract) + return gcp_errors + gov_errors, gcp_warnings + gov_warnings + + +@pytest.mark.parametrize("platform", ["aws", "gcp"]) +def test_a_structured_column_is_refused_at_validate_and_at_generate(platform): + contract = _contract(platform, _BAD) + errors, _ = _gate(contract) + typed = [e for e in errors if "cannot hold its columns as declared" in e] + assert typed, errors + message = typed[0] + assert "expose 'orders'" in message + assert "'tags' is declared 'array'" in message + assert "'addr' is declared 'struct'" in message + assert "json or string" in message and "planned" in message + with pytest.raises(UnsupportedBindingError) as caught: + get_iac_plugin(platform).emit(contract) + assert caught.value.kind == "column-type" + assert "'tags' is declared 'array'" in str(caught.value) + assert any("json or string" in step for step in caught.value.remediation) + + +def test_the_validate_refusal_is_the_emit_refusal_word_for_word(): + for platform in ("aws", "gcp"): + contract = _contract(platform, _BAD) + errors, _ = _gate(contract) + with pytest.raises(UnsupportedBindingError) as caught: + get_iac_plugin(platform).emit(contract) + raised = f"{caught.value} {' '.join(caught.value.remediation)}" + assert raised in errors + + +def test_without_the_structured_columns_glue_writes_the_faithful_types_and_warns_on_json(): + contract = _contract("aws", _GOOD) + errors, warnings = _gate(contract) + assert errors == [] + assert len(warnings) == 1 and "'payload'" in warnings[0] and "'string'" in warnings[0] + resources = get_iac_plugin("aws").emit(contract) + columns = next(iter(resources["aws_glue_catalog_table"].values()))["storage_descriptor"][ + "columns" + ] + assert [(c["name"], c["type"]) for c in columns] == [ + ("id", "string"), + ("payload", "string"), + ("small", "smallint"), + ("tz", "timestamp"), + ] + + +def test_without_the_structured_columns_bigquery_gets_valid_types_and_no_warning(): + contract = _contract("gcp", _GOOD) + assert _gate(contract) == ([], []) + resources = get_iac_plugin("gcp").emit(contract) + schema = json.loads(next(iter(resources["google_bigquery_table"].values()))["schema"]) + assert [(f["name"], f["type"]) for f in schema] == [ + ("id", "STRING"), + ("payload", "JSON"), + ("small", "INT64"), + ("tz", "TIMESTAMP"), + ] + + +def test_a_geometry_column_on_bigquery_is_warned(): + errors, warnings = _gate(_contract("gcp", [("shape", "geometry")])) + assert errors == [] + assert len(warnings) == 1 and "planar" in warnings[0] + + +def test_a_geometry_column_on_glue_is_refused_not_guessed(): + errors, _ = _gate(_contract("aws", [("shape", "geometry")])) + assert any("'shape' is declared 'geometry'" in e and "WKT" in e for e in errors) + + +@pytest.mark.parametrize("platform", ["aws", "gcp"]) +def test_every_type_validate_lets_through_generates(platform): + """A contract that passes validate must not fail generate on a type, and the + reverse: one column per schema type, each expose judged by both.""" + for fluid_type in _ALL_TYPES: + contract = _contract(platform, [("col", fluid_type)]) + errors, _ = _gate(contract) + try: + get_iac_plugin(platform).emit(contract) + refused = False + except UnsupportedBindingError as exc: + assert exc.kind == "column-type" + refused = True + assert refused == bool(errors), (platform, fluid_type, errors) + + +def test_a_binding_with_no_glue_table_or_a_view_has_no_column_types_to_gate(): + no_table = _contract("aws", _BAD) + del no_table["exposes"][0]["binding"]["location"]["table"] + assert validate_aws_column_types(no_table) == ([], []) + view = _contract("gcp", _BAD) + view["exposes"][0]["binding"]["format"] = "bigquery_view" + view["exposes"][0]["binding"]["location"]["query"] = "SELECT 1" + assert not [e for e in _gate(view)[0] if "cannot hold its columns" in e] + get_iac_plugin("gcp").emit(view) + + +def test_another_clouds_binding_is_not_gated_by_this_ones_types(): + contract = _contract("aws", _BAD) + assert not [e for e in validate_gcp_binding(contract)[0] if "BigQuery" in e] + + +# ── fluid validate, end to end ─────────────────────────────────────────────── + + +def _validate(tmp_path, contract, *flags): + from fluid_build.cli import build_parser + from fluid_build.cli.validate import run + + path = tmp_path / "contract.fluid.yaml" + path.write_text(yaml.safe_dump(contract, sort_keys=False), encoding="utf-8") + args = build_parser().parse_args(["validate", str(path), "--offline", *flags]) + return run(args, logging.getLogger("test")) + + +@pytest.mark.parametrize("platform", ["aws", "gcp"]) +def test_validate_strict_exits_1_for_a_structured_column(tmp_path, platform, capsys): + assert _validate(tmp_path, _contract(platform, _BAD), "--strict") == 1 + out = capsys.readouterr() + assert "cannot hold its columns as declared" in out.out + out.err + + +def test_validate_strict_exits_1_on_the_glue_json_warning_and_0_without_it(tmp_path, capsys): + assert _validate(tmp_path, _contract("aws", _GOOD), "--strict") == 1 + assert _validate(tmp_path, _contract("aws", _GOOD)) == 0 + assert ( + _validate(tmp_path, _contract("aws", [c for c in _GOOD if c[1] != "json"]), "--strict") == 0 + ) + assert _validate(tmp_path, _contract("gcp", _GOOD), "--strict") == 0 + capsys.readouterr() + + +# ── An in-place Glue column type change is held by the data-loss gate ──────── + + +def _glue_change(actions, before_types, after_types, before_raw=None): + def table(types): + if types is None: + return None + columns = [{"name": n, "type": t, "comment": ""} for n, t in types] + return { + "database_name": "demo", + "name": "orders", + "storage_descriptor": [{"columns": columns}], + } + + return { + "type": "aws_glue_catalog_table", + "address": "aws_glue_catalog_table.t", + "change": { + "actions": actions, + "before": before_raw if before_raw is not None else table(before_types), + "after": table(after_types), + }, + } + + +def _hold(*changes): + return get_iac_plugin("aws").plan_data_changes({"resource_changes": list(changes)}) + + +def test_a_glue_update_that_changes_a_column_type_is_held_with_the_whole_story(): + lines = _hold( + _glue_change( + ["update"], + [("id", "string"), ("small", "string"), ("tz", "string")], + [("id", "string"), ("small", "smallint"), ("tz", "timestamp")], + ) + ) + assert len(lines) == 1 + line = lines[0] + assert line.startswith("Glue table demo.orders: column types change in place") + assert "small: string -> smallint; tz: string -> timestamp" in line + assert "id:" not in line + assert "may stop reading until they are rewritten" in line + assert "reverting the column type restores reads" in line + + +def test_a_spacing_only_difference_a_new_column_or_a_create_is_not_held(): + same = [("id", "string"), ("d", "decimal(10, 2)")] + assert _hold(_glue_change(["update"], same, [("id", "string"), ("d", "decimal(10,2)")])) == [] + assert _hold(_glue_change(["update"], same, same + [("new", "int")])) == [] + assert _hold(_glue_change(["update"], same, same[:1])) == [] + assert _hold(_glue_change(["create"], None, same)) == [] + other = _glue_change(["update"], [("a", "string")], [("a", "int")]) + other["type"] = "aws_s3_bucket" + assert _hold(other) == [] + assert get_iac_plugin("aws").plan_data_changes({}) == [] + # A type change next to a spacing change is still held, for the type only. + held = _hold( + _glue_change( + ["update"], + [("a", "string"), ("d", "decimal(10, 2)")], + [("a", "int"), ("d", "decimal(10,2)")], + ) + ) + assert len(held) == 1 and "a: string -> int" in held[0] and "d:" not in held[0] + + +def test_a_malformed_plan_is_read_without_raising(): + assert _hold(_glue_change(["update"], None, [("a", "int")], before_raw="x")) == [] + assert _hold({"type": "aws_glue_catalog_table", "change": "x"}) == [] + + +def _enforce(lines, allow): + from fluid_build.cli import _apply_opentofu_engine as engine + + args = argparse.Namespace(allow_data_loss=allow) + changes = {"add": 0, "change": 1, "remove": 0} + engine._enforce_data_loss_gate( + args, logging.getLogger("test"), "aws", changes, {"remove": 0}, lines + ) + + +def test_the_apply_is_refused_without_the_flag_and_proceeds_with_it(monkeypatch, capsys): + from fluid_build.cli import _apply_opentofu_engine as engine + from fluid_build.cli._common import CLIError + + plan = { + "resource_changes": [ + _glue_change(["update"], [("small", "string")], [("small", "smallint")]) + ] + } + monkeypatch.setattr(engine.runner, "tofu_show_plan", lambda *a, **k: plan) + lines = engine._in_place_changes(get_iac_plugin("aws"), {"change": 1}, "wd", {}) + assert len(lines) == 1 + with pytest.raises(CLIError) as caught: + _enforce(lines, allow=False) + message = str(caught.value.context if hasattr(caught.value, "context") else caught.value) + assert "small: string -> smallint" in message and "--allow-data-loss" in message + _enforce(lines, allow=True) + assert "small: string -> smallint" in capsys.readouterr().out + + +def test_live_values_in_the_gate_list_and_the_notices_print_literally(monkeypatch, capsys): + """A live column type is data: ``string[/]`` must not be read as markup, and a + link tag must not become a hyperlink.""" + from fluid_build.cli import _apply_opentofu_engine as engine + + evil = "string[/] [link=https://evil.example]x[/link]" + lines = _hold(_glue_change(["update"], [("a", evil)], [("a", "int")])) + _enforce(lines, allow=True) + out = capsys.readouterr().out + assert "string[/] [link=https://evil.example]x[/link]" in out + + class Plugin: + def plan_notices(self, contract): + return [evil] + + engine._report_plan_notices(Plugin(), {}, logging.getLogger("test")) + assert evil in capsys.readouterr().out + + +# ── Hardening: non-ASCII parameters, the NUMERIC range, malformed bindings ─── + + +@pytest.mark.parametrize("fluid_type", ["decimal(١٠,٢)", "float(٥٣)", "number(１０)"]) +def test_non_ascii_digits_in_a_type_are_refused_on_both_clouds(fluid_type): + assert _hive_outcome(fluid_type).status == REFUSED + assert _bq_outcome(fluid_type).status == REFUSED + + +@pytest.mark.parametrize( + "fluid_type", ["decimal(38,0)", "decimal(40,10)", "numeric(38,10)", "number(30)"] +) +def test_a_decimal_wider_than_numeric_is_warned_on_bigquery(fluid_type): + outcome = _bq_outcome(fluid_type) + assert (outcome.target, outcome.status) == ("NUMERIC", LOSSY) + assert "bignumeric" in outcome.fix.lower() + errors, warnings = validate_gcp_binding(_contract("gcp", [("amt", fluid_type)])) + assert errors == [] and len(warnings) == 1 and "bignumeric" in warnings[0] + + +@pytest.mark.parametrize( + "fluid_type", ["decimal", "decimal(38,9)", "decimal(29,0)", "numeric(10,2)", "number(18)"] +) +def test_a_decimal_within_numeric_is_exact_on_bigquery(fluid_type): + assert _bq_outcome(fluid_type).status == EXACT + + +@pytest.mark.parametrize("platform", ["aws", "gcp"]) +@pytest.mark.parametrize( + "mutate", + [ + lambda e: e["binding"].update(location="s3://x"), + lambda e: e.update(contract="x"), + lambda e: e["contract"].update(schema="x"), + lambda e: e.update(binding="x"), + lambda e: e["contract"].update(schema=[1, None, "x"]), + ], +) +def test_a_schema_invalid_binding_is_skipped_not_fatal(platform, mutate): + contract = _contract(platform, _GOOD) + mutate(contract["exposes"][0]) + assert validate_aws_column_types(contract) == ([], []) + assert validate_bigquery_column_types(contract) == ([], []) + for exposes in ("x", 5, {"a": 1}): + assert validate_aws_column_types({"exposes": exposes}) == ([], []) + assert validate_bigquery_column_types({"exposes": exposes}) == ([], []) diff --git a/tests/iac/test_iac_databricks.py b/tests/iac/test_iac_databricks.py index 94d462496..72145dd0d 100644 --- a/tests/iac/test_iac_databricks.py +++ b/tests/iac/test_iac_databricks.py @@ -694,7 +694,7 @@ def test_unset_warehouse_is_refused(monkeypatch): {"principal": "group:x", "permissions": ["read"], "conditions": {"ip": "1"}}, "conditions", ), - ({"principal": "group:x", "permissions": ["read"], "resources": ["$.exposes"]}, "scope"), + ({"principal": "group:x", "permissions": ["read"], "resources": ["$.exposes[0]"]}, "scope"), ({"principal": "user:bob@corp.example", "permissions": ["read"]}, "principal-placeholder"), ({"principal": "serviceAccount:sa@p.iam", "permissions": ["read"]}, "principal-unmapped"), ], @@ -1048,6 +1048,58 @@ def test_an_unreachable_workspace(self): refusal = self.plugin.explain_plan_failure(out, _contract(), []) assert refusal.kind == "databricks-workspace-unreachable" + RESIDENCY = ( + "The contract declares binding.location.region eu-central-1, but the workspace " + "DATABRICKS_HOST names is in us-west-2. Residency is checked against the workspace's " + "metastore. Point DATABRICKS_HOST at a workspace in eu-central-1, or correct " + "binding.location.region." + ) + + def _residency_json(self, padding=2000): + diagnostic = { + "severity": "error", + "summary": "Resource precondition failed", + "detail": self.RESIDENCY, + "range": {"filename": "main.tf.json"}, + "snippet": { + "code": "x" * padding, + "values": [ + { + "traversal": "data.databricks_current_metastore.m.metastore_info[0].region", + "statement": 'is "us-west-2"', + } + ], + }, + } + return json.dumps({"@level": "error", "diagnostic": diagnostic}) + + def test_a_residency_failure_is_one_clean_line_with_both_regions_and_the_fix(self): + out = self._residency_json() + assert len(out) > 1600, "the engine's tail would cut the sentence" + refusal = self.plugin.explain_plan_failure(out, _contract(), []) + text = str(refusal) + assert refusal.kind == "databricks-residency-region" + assert "\n" not in text and not refusal.remediation + assert "eu-central-1" in text and "us-west-2" in text + assert "DATABRICKS_HOST" in text and "binding.location.region" in text + + def test_a_residency_failure_in_the_wrapped_human_output(self): + out = "Error: Resource precondition failed\n\n on main.tf.json line 3:\n\n" + "\n".join( + " " + line for line in __import__("textwrap").wrap(self.RESIDENCY, 60) + ) + text = str(self.plugin.explain_plan_failure(out, _contract(), [])) + assert "eu-central-1" in text and "us-west-2" in text and "\n" not in text + + def test_the_live_region_comes_from_the_diagnostic_values_when_the_text_lacks_it(self): + diagnostic = { + "detail": "The contract declares binding.location.region eu-central-1, but the " + "workspace DATABRICKS_HOST names is in (unknown).", + "snippet": {"values": [{"statement": 'is "us-west-2"'}]}, + } + out = json.dumps({"diagnostic": diagnostic}) + text = str(self.plugin.explain_plan_failure(out, _contract(), [])) + assert "us-west-2" in text and "eu-central-1" in text + def test_anything_else_is_left_alone(self): assert self.plugin.explain_plan_failure("Error: 503", _contract(), []) is None @@ -1273,3 +1325,13 @@ def test_the_native_simple_apply_never_takes_the_platform(): _simple_apply_provider(contract, _LOG) assert exc.value.event == "apply_platform_unsupported" assert "exposes[0].binding.platform: databricks" in exc.value.context["error"] + + +@pytest.mark.parametrize( + "value", [{1: "a", "b": "c"}, {__import__("datetime").date(2024, 1, 1): "x"}] +) +def test_a_contract_json_cannot_key_reuses_no_pause_decision(value): + contract = _contract() + contract["metadata"] = {"free_form": value} + assert dbx._contract_key(contract) is None + IAC_PLUGINS["databricks"].emit(contract) # no TypeError from the key diff --git a/tests/iac/test_iac_databricks_enforce.py b/tests/iac/test_iac_databricks_enforce.py index 280e79e4e..9e6a805e7 100644 --- a/tests/iac/test_iac_databricks_enforce.py +++ b/tests/iac/test_iac_databricks_enforce.py @@ -175,7 +175,7 @@ def test_a_second_run_executes_nothing(fake): assert again.said == [] -def test_a_changed_predicate_binds_a_new_function_and_keeps_the_old_one(fake): +def test_a_changed_predicate_binds_a_new_function_and_drops_the_old_one_after_the_apply(fake): seed(fake) Run(fake).pre(contract()) old = live(fake)["row_filter"]["function_name"] @@ -184,14 +184,17 @@ def test_a_changed_predicate_binds_a_new_function_and_keeps_the_old_one(fake): changed = contract(filters=[{**FILTER, "where": "consent_data_analytics = false"}]) run = Run(fake) run.pre(changed) + assert old in fake.functions, "pre_apply drops nothing" run.post(changed) new = live(fake)["row_filter"] assert new["function_name"] != old assert new["input_column_names"] == ["consent_data_analytics"] - assert old in fake.functions, "a superseded function is not dropped" + assert old not in fake.functions, "the superseded function is dropped once the apply is done" assert live_masks(fake) == masks - assert not any(s.startswith("DROP") for s in run.statements()) + assert [s for s in run.statements() if s.startswith("DROP")] == [ + f"DROP FUNCTION IF EXISTS `main`.`cp360`.`{old.rsplit('.', 1)[1]}`" + ] assert len([s for s in run.statements() if s.startswith("CREATE FUNCTION")]) == 1 @@ -1048,6 +1051,158 @@ def test_a_read_failure_before_the_apply_still_stops_the_run(fake): Run(fake).pre(contract()) +# ── orphan functions: what a changed contract leaves behind ───────────────── +OTHER_FILTER = {"principal": ANALYSTS, "where": "is_suppressed = false"} + + +def _fluid_functions(fake: FakeDatabricks) -> set: + return {n for n in fake.functions if n.rsplit(".", 1)[1].startswith("fluid_")} + + +def _changed_contract_applied(fake: FakeDatabricks): + """Apply a contract, then one whose row filter differs: the first filter is superseded.""" + seed(fake) + Run(fake).pre(contract()) + first = live(fake)["row_filter"]["function_name"] + changed = contract(filters=[OTHER_FILTER]) + return first, changed + + +def test_a_superseded_function_is_dropped_after_a_successful_apply_and_said(fake): + first, changed = _changed_contract_applied(fake) + run = Run(fake) + run.pre(changed) + assert first in fake.functions, "pre_apply keeps it until the apply has succeeded" + run.post(changed) + assert first not in fake.functions + now = live(fake)["row_filter"]["function_name"] + assert now in fake.functions and now != first + assert f"DROP FUNCTION IF EXISTS `main`.`cp360`.`{first.rsplit('.', 1)[1]}`" in run.statements() + assert f"dropped superseded function {first}" in run.text + assert all(f in fake.functions for f in live_masks(fake).values()) + + +def test_a_function_bound_to_any_table_in_the_schema_is_never_dropped(fake): + first, changed = _changed_contract_applied(fake) + fake.create_table("main.cp360.other", [("a", "string", True)]) + fake.tables["main.cp360.other"]["row_filter"] = { + "function_name": first, + "input_column_names": [], + } + run = Run(fake) + run.pre(changed) + run.post(changed) + assert first in fake.functions + assert "DROP FUNCTION" not in "\n".join(run.statements()) + + +def test_only_this_tables_fluid_functions_are_dropped(fake): + first, changed = _changed_contract_applied(fake) + keep = [ + "main.cp360.my_helper", # no fluid_ prefix + "main.cp360.fluid_rf_profile_x_deadbeef", # another table's name + "main.cp360.fluid_rf_profile_deadbeef_extra", # not this module's naming + "main.cp360.fluid_mask_profile_nosuchcolumn_deadbeef", # no such column on the table + ] + for name in keep: + fake.add_function(name, [("val", "STRING")], "STRING", "`val`") + run = Run(fake) + run.pre(changed) + run.post(changed) + assert first not in fake.functions + assert all(name in fake.functions for name in keep) + + +def test_an_orphan_not_owned_by_the_deploy_identity_is_left_and_named(fake): + first, changed = _changed_contract_applied(fake) + fake.functions[first]["owner"] = "someone-else@example.test" + run = Run(fake) + run.pre(changed) + run.post(changed) + assert first in fake.functions + assert "DROP FUNCTION" not in "\n".join(run.statements()) + assert f"left function {first}: owned by someone-else@example.test, not by the deploy " in ( + run.text + ) + + +def test_an_orphan_whose_owner_cannot_be_read_is_left_and_named(fake): + first, changed = _changed_contract_applied(fake) + fake.fail_route("GET", rf"/functions/{first}$", 503, "unavailable") + run = Run(fake) + run.pre(changed) + run.post(changed) + assert first in fake.functions + assert f"left function {first}: owner unknown" in run.text + + +def test_nothing_is_dropped_after_a_failed_apply(fake): + first, changed = _changed_contract_applied(fake) + run = Run(fake) + PLUGIN.post_apply( + changed, say=run.said.append, client_factory=fake.client, after_failed_apply=True + ) + assert first in fake.functions + assert "DROP FUNCTION" not in "\n".join(run.statements()) + + +def test_nothing_is_dropped_when_a_bind_fails(fake): + first, changed = _changed_contract_applied(fake) + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") + run = Run(fake) + with pytest.raises(UnsupportedBindingError): + run.post(changed) + assert first in fake.functions + assert "DROP FUNCTION" not in "\n".join(run.statements()) + + +def test_a_schema_whose_tables_cannot_all_be_read_drops_nothing_and_says_so(fake): + first, changed = _changed_contract_applied(fake) + fake.create_table("main.cp360.other", [("a", "string", True)]) + Run(fake).pre(changed) + fake.fail_route("GET", r"/tables/main\.cp360\.other$", 503, "unavailable") + run = Run(fake) + run.post(changed) + assert first in fake.functions + assert "orphan" in run.text and "not dropped" in run.text + + +def test_the_orphan_drop_follows_pages_of_functions(fake): + first, changed = _changed_contract_applied(fake) + fake.list_page_size = 1 + run = Run(fake) + run.pre(changed) + run.post(changed) + assert first not in fake.functions + + +def test_an_orphan_whose_drop_fails_is_kept_named_and_the_apply_still_succeeds(fake): + first, changed = _changed_contract_applied(fake) + fake.fail_on("DROP FUNCTION", "[PERMISSION_DENIED] cannot drop") + run = Run(fake) + run.pre(changed) + run.post(changed) + assert first in fake.functions + assert f"note: superseded function {first} was not dropped" in run.text + assert "PERMISSION_DENIED" in run.text + + +def test_the_orphan_drop_lists_the_tables_it_may_only_browse_too(fake): + first, changed = _changed_contract_applied(fake) + seen: List[Any] = [] + real = fake._list_tables + + def spy(m, q, b): + seen.append(q) + return real(m, q, b) + + fake.routes = [(mth, rx, spy if fn == real else fn) for mth, rx, fn in fake.routes] + run = Run(fake) + run.pre(changed) + run.post(changed) + assert any((q.get("include_browse") or [""])[0] == "true" for q in seen), seen + + # ── the fail-safe: a deny-all filter, then a lock ─────────────────────────── def _holders(fake: FakeDatabricks) -> Dict[str, List[str]]: return {p: v for p, v in fake.grants[("table", TABLE)].items() if v} @@ -1113,6 +1268,7 @@ def test_when_the_deny_all_filter_fails_select_is_revoked_from_every_grantee_but fake.grant("table", TABLE, "power-group", "ALL_PRIVILEGES") fake.grant("table", TABLE, "fake-user@example.test", "SELECT") # the table's owner fake.grant("table", TABLE, "modifier", "MODIFY") + fake.grant("schema", "main.cp360", "browsers", "USE_SCHEMA") # inherited, but not a read fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") run = Run(fake) with pytest.raises(UnsupportedBindingError) as exc: @@ -1129,6 +1285,57 @@ def test_when_the_deny_all_filter_fails_select_is_revoked_from_every_grantee_but assert "No grant inherited from the schema or the catalog was found open" in message +def test_the_lock_never_revokes_from_the_deploy_identity_when_the_owner_is_unreadable(fake): + seed(fake) + fake.grant("table", TABLE, DEPLOY_USER, "SELECT") # this run's own identity + fake.grant("table", TABLE, "other-group", "SELECT") + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") + fake.fail_route("GET", r"/tables/main\.cp360\.profile$", 503, "unavailable") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.post(contract()) + revokes = [s for s in fake.sql_log if s.startswith("REVOKE")] + assert not any(DEPLOY_USER in s for s in revokes), "the run must not lock itself out" + assert any("other-group" in s for s in revokes) + assert fake.grants[("table", TABLE)][DEPLOY_USER] == ["SELECT"] + assert "owner could not be read, so it could not be skipped" in str(exc.value) + + +def test_the_lock_skips_the_deploy_identity_when_it_is_not_the_owner(fake): + seed(fake) + fake.tables[TABLE]["owner"] = "someone-else@example.test" + fake.grant("table", TABLE, DEPLOY_USER, "SELECT") + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") + with pytest.raises(UnsupportedBindingError) as exc: + Run(fake).post(contract()) + assert fake.grants[("table", TABLE)][DEPLOY_USER] == ["SELECT"] + assert "could not be skipped" not in str(exc.value) + + +def test_an_owner_the_contract_restricts_is_named_as_a_path_the_lock_cannot_close(fake): + seed(fake) + fake.tables[TABLE]["owner"] = ANALYSTS_GROUP # the owner is a restricted group + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") + with pytest.raises(UnsupportedBindingError) as exc: + Run(fake).post(contract()) + message = str(exc.value) + assert "Still open:" in message + assert f"{ANALYSTS_GROUP} owns {TABLE}" in message + assert "ALTER TABLE `main`.`cp360`.`profile` OWNER TO" in message + assert "No grant inherited" not in message + + +def test_effective_permissions_with_no_inherited_field_do_not_claim_nothing_is_open(fake): + seed(fake) + fake.fail_on("SET ROW FILTER", "[INTERNAL_ERROR] x") + run = Run(fake) + with pytest.raises(UnsupportedBindingError) as exc: + run.post(contract()) + message = str(exc.value) + assert "could not determine whether a schema or catalog grant is open" in message + assert "No grant inherited" not in message + + def test_a_grant_inherited_from_the_schema_or_catalog_is_named_with_what_to_run(fake): seed(fake) fake.grant("schema", "main.cp360", "schema-readers", "SELECT") @@ -1239,6 +1446,26 @@ def test_a_job_is_unpaused_once_the_live_bindings_match(workspace_client): assert not any("is created paused" in n for n in PLUGIN.plan_notices(_job_contract())) +def test_the_pause_decision_is_made_once_and_the_notice_reuses_it(workspace_client): + fake = workspace_client + seed(fake) + Run(fake).pre(contract()) # bound: a fresh read would say "protected" + fake.fail_route("GET", r"/tables/main\.cp360\.profile$", 503, "unavailable") + assert _emitted_job(_job_contract())["schedule"]["pause_status"] == "PAUSED" + fake.failing_routes.clear() # the read error was transient + notices = "\n".join(PLUGIN.plan_notices(_job_contract())) + assert "is created paused" in notices and "main.cp360.profile" in notices + + +def test_the_notice_is_not_reused_by_a_later_run(workspace_client): + fake = workspace_client + seed(fake) + assert _emitted_job(_job_contract())["schedule"]["pause_status"] == "PAUSED" + assert "is created paused" in "\n".join(PLUGIN.plan_notices(_job_contract())) + Run(fake).pre(contract()) # a later run, with no emit before its notices + assert not any("is created paused" in n for n in PLUGIN.plan_notices(_job_contract())) + + def test_a_job_stays_paused_while_a_binding_is_missing_or_differs(workspace_client): fake = workspace_client seed(fake) diff --git a/tests/iac/test_iac_databricks_fake_e2e.py b/tests/iac/test_iac_databricks_fake_e2e.py index 35bef8651..cbee5c8ff 100644 --- a/tests/iac/test_iac_databricks_fake_e2e.py +++ b/tests/iac/test_iac_databricks_fake_e2e.py @@ -653,7 +653,7 @@ def test_a_dry_run_prints_the_statements_and_executes_none(workspace, fake, capf @requires_sdk -def test_a_changed_predicate_binds_a_new_function_and_keeps_the_old_one(workspace, fake, capfd): +def test_a_changed_predicate_binds_a_new_function_and_drops_the_old_one(workspace, fake, capfd): assert apply_contract(capfd, write_contract(workspace, _restricted()))[0] == 0 old = _live(fake)["row_filter"]["function_name"] masks = _masks(fake) @@ -661,9 +661,12 @@ def test_a_changed_predicate_binds_a_new_function_and_keeps_the_old_one(workspac rc, out = apply_contract(capfd, write_contract(workspace, _restricted(where="region = 'uk'"))) assert rc == 0, out new = _live(fake)["row_filter"]["function_name"] - assert new != old and old in fake.functions and new in fake.functions + assert new != old and old not in fake.functions and new in fake.functions assert _masks(fake) == masks - assert not any(s.startswith("DROP") for s in fake.sql_log) + assert f"dropped superseded function {old}" in out + assert [s for s in fake.sql_log if s.startswith("DROP")] == [ + f"DROP FUNCTION IF EXISTS `main`.`sales`.`{old.rsplit('.', 1)[1]}`" + ] assert not any("REPLACE FUNCTION" in s for s in fake.sql_log) diff --git a/tests/iac/test_iac_databricks_grant_scope.py b/tests/iac/test_iac_databricks_grant_scope.py new file mode 100644 index 000000000..124906110 --- /dev/null +++ b/tests/iac/test_iac_databricks_grant_scope.py @@ -0,0 +1,175 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``accessPolicy.grants[].resources`` scopes a grant to exposes on Databricks. + +The emitter evaluates the expose filters a data product writes, +``$.exposes[?(@.exposeId=='')]``, and ``$.exposes[*]`` / ``$.exposes`` for every expose: +the grant lands on exactly the tables they name. Anything it cannot evaluate, or an expose id +the contract does not have, is refused (``databricks-grant-scope-unmapped``), never widened. +""" + +from __future__ import annotations + +import copy +import json +from typing import Any, Dict, List + +import pytest + +from fluid_build.iac import IAC_PLUGINS, build_module +from fluid_build.iac.access import normalize_access_grants +from fluid_build.iac.base import UnsupportedBindingError +from fluid_build.iac.providers import databricks as dbx + +from .test_iac_databricks import _contract + +PRINCIPAL = "group:auditors" + + +@pytest.fixture(autouse=True) +def _warehouse(monkeypatch): + monkeypatch.setenv(dbx.WAREHOUSE_ENV, "wh-0123456789abcdef") + + +def _two_exposes() -> Dict[str, Any]: + contract = _contract() + refunds = copy.deepcopy(contract["exposes"][0]) + refunds["exposeId"] = "refunds" + refunds["binding"]["location"]["table"] = "refunds" + contract["exposes"].append(refunds) + return contract + + +def _scoped(contract: Dict[str, Any], *resources: str) -> Dict[str, Any]: + contract["accessPolicy"]["grants"].append( + {"principal": PRINCIPAL, "permissions": ["read"], "resources": list(resources)} + ) + return contract + + +def _tables_granted(contract: Dict[str, Any], principal: str = "auditors") -> List[str]: + """The tables (``main.sales.``) a principal is granted on.""" + module = json.loads(build_module(IAC_PLUGINS["databricks"], contract)) + tables = module["resource"]["databricks_sql_table"] + by_ref = {f"${{databricks_sql_table.{key}.id}}": t for key, t in tables.items()} + return sorted( + "main.sales." + by_ref[g["table"]]["name"] + for g in module["resource"]["databricks_grant"].values() + if "table" in g and g["principal"] == principal + ) + + +def _refusal(contract: Dict[str, Any]) -> UnsupportedBindingError: + with pytest.raises(UnsupportedBindingError) as exc: + IAC_PLUGINS["databricks"].emit(contract) + errors, _ = dbx.validate_databricks_binding(contract) + assert any(exc.value.args[0][:60] in e for e in errors), "validate refuses it too" + return exc.value + + +@pytest.mark.parametrize( + "resource", + [ + "$.exposes[?(@.exposeId=='orders')]", + '$.exposes[?(@.exposeId=="orders")]', + "$.exposes[?( @.exposeId == 'orders' )]", + " $.exposes[ ?(@.exposeId=='orders') ] ", + ], +) +def test_a_grant_scoped_to_the_one_expose_lands_on_its_table(resource): + contract = _scoped(_contract(), resource) + assert _tables_granted(contract) == ["main.sales.orders"] + assert dbx.validate_databricks_binding(contract)[0] == [] + + +def test_a_grant_scoped_to_one_of_two_exposes_lands_on_that_table_only(): + contract = _scoped(_two_exposes(), "$.exposes[?(@.exposeId=='refunds')]") + assert _tables_granted(contract) == ["main.sales.refunds"] + tables, _ = dbx.managed_objects(contract) + readers = {t.expose_id: t.readers for t in tables} + assert "auditors" in readers["refunds"] and "auditors" not in readers["orders"] + + +def test_a_grant_scoped_to_both_exposes_lands_on_both(): + contract = _scoped( + _two_exposes(), + "$.exposes[?(@.exposeId=='orders')]", + "$.exposes[?(@.exposeId=='refunds')]", + ) + assert _tables_granted(contract) == ["main.sales.orders", "main.sales.refunds"] + + +@pytest.mark.parametrize("resource", ["$.exposes[*]", "$.exposes", " $.exposes[ * ] "]) +def test_every_expose_selectors_land_on_every_table(resource): + contract = _scoped(_two_exposes(), resource) + assert _tables_granted(contract) == ["main.sales.orders", "main.sales.refunds"] + + +def test_an_unscoped_grant_still_lands_on_every_table(): + contract = _two_exposes() + contract["accessPolicy"]["grants"].append({"principal": PRINCIPAL, "permissions": ["read"]}) + assert _tables_granted(contract) == ["main.sales.orders", "main.sales.refunds"] + + +def test_an_expose_id_the_contract_does_not_have_is_refused(): + contract = _scoped(_contract(), "$.exposes[?(@.exposeId=='ordres')]") + refusal = _refusal(contract) + assert refusal.kind == "databricks-grant-scope-unmapped" + text = str(refusal) + assert "'ordres'" in text and "no expose" in text + assert "reach no" not in text + + +@pytest.mark.parametrize( + "resource", + [ + "$.exposes[0]", + "$.exposes[?(@.kind=='table')]", + "$..orders", + "$.exposes[?(@.exposeId=='orders' || @.exposeId=='refunds')]", + "$.exposes[?(@.exposeId=='orders')].contract", + "orders", + "", + " ", + ], +) +def test_a_jsonpath_the_emitter_cannot_evaluate_is_refused(resource): + refusal = _refusal(_scoped(_two_exposes(), resource)) + assert refusal.kind == "databricks-grant-scope-unmapped" + assert repr(resource) in str(refusal) + assert "reach no" not in str(refusal) + + +def test_one_unevaluable_resource_beside_a_good_one_refuses_the_grant(): + contract = _scoped(_two_exposes(), "$.exposes[?(@.exposeId=='orders')]", "$.exposes[0]") + assert _refusal(contract).kind == "databricks-grant-scope-unmapped" + + +def test_a_scope_refusal_never_emits_a_wider_grant(): + # the grant is refused, and the table privileges for it are computed for no table + contract = _scoped(_two_exposes(), "$.exposes[0]") + refusals: List[UnsupportedBindingError] = [] + compiled = dbx._compile(contract, {dbx.WAREHOUSE_ENV: "wh"}, validating=True) + refusals = compiled.refusals + assert refusals + assert not [g for g in compiled.grants if g[2] == "auditors"] + + +def test_the_grant_model_parses_the_scope_once(): + contract = _scoped(_two_exposes(), "$.exposes[?(@.exposeId=='refunds')]") + grant = next(g for g in normalize_access_grants(contract) if g.principal == "auditors") + assert grant.scope.expose_ids == frozenset({"refunds"}) + assert not grant.scope.all_exposes and grant.scope.unevaluable == () + assert grant.scope.covers("refunds") and not grant.scope.covers("orders") diff --git a/tests/iac/test_iac_databricks_landing_fake_e2e.py b/tests/iac/test_iac_databricks_landing_fake_e2e.py new file mode 100644 index 000000000..c818f5055 --- /dev/null +++ b/tests/iac/test_iac_databricks_landing_fake_e2e.py @@ -0,0 +1,207 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fluid apply`` creates the table, the acquisition build lands bronze into it, against a stand-in. + +The real ``fluid`` CLI, ``tofu`` and the real ``databricks-sdk`` run in-process; only the workspace +is ``_fake_databricks.FakeDatabricks``. A contract exposes a managed Delta table that a DuckDB +acquisition build writes from a local CSV, with ``policy.privacy.masking`` on one column, a column +restriction and a row filter: + +* ``fluid apply`` accepts the masking (the runner applies it) and creates the table, its grants, its + mask and its row filter; +* ``fluid apply --mode amend-and-build`` then hashes the column, writes Parquet locally, uploads it + to ``$DATABRICKS_STAGING_VOLUME`` through the Files API and runs ``INSERT OVERWRITE ... read_files`` + on the warehouse: the rows are in the table with the column hashed, the staged files are deleted, + and the table keeps its definition, grants, row filter and mask (no statement but the load); +* a second run overwrites the rows, it does not append; +* a plan afterwards is empty: loading data does not drift the table. + +Not proven here: what real Databricks accepts of ``read_files``, ``INSERT OVERWRITE`` over a managed +table that has a mask and a row filter, or the Files API's limits; that is the manual live tier. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Any, Dict, Iterator, List, Tuple + +import pytest + +from fluid_build.build_runners import _masking as masking +from fluid_build.iac import runner + +from ._databricks_cli import ( # noqa: F401 — plugin_cache is a fixture + apply_contract, + isolate_workspace, + make_contract, + plugin_cache, + write_contract, +) +from ._fake_databricks import FakeDatabricks + +pytestmark = [ + pytest.mark.integration, + pytest.mark.provider, + pytest.mark.databricks, + pytest.mark.skipif(runner.tofu_path() is None, reason="tofu binary not installed"), +] + +duckdb = pytest.importorskip("duckdb") +pytest.importorskip("databricks.sdk", reason="the [databricks] extra") + +TABLE = "main.sales.orders" +VOLUME = "main.staging.landing" +SALT = "e2e-landing-salt-0123456789" +FIRST: List[Tuple[int, float, str]] = [(1, 10.5, "emea"), (2, 20.0, "apac"), (3, 4.25, "emea")] +SECOND: List[Tuple[int, float, str]] = [(7, 1.5, "amer"), (8, 2.5, "amer")] + + +@pytest.fixture +def fake() -> Iterator[FakeDatabricks]: + server = FakeDatabricks().start() + server.add_volume(VOLUME) + try: + yield server + finally: + server.stop() + + +@pytest.fixture +def workspace(monkeypatch, tmp_path: Path, fake: FakeDatabricks, plugin_cache: str) -> Path: + isolate_workspace( + monkeypatch, + tmp_path, + host=fake.endpoint, + warehouse_id=fake.warehouse_id, + plugin_cache=plugin_cache, + ) + monkeypatch.setenv("DATABRICKS_STAGING_VOLUME", "/Volumes/" + VOLUME.replace(".", "/")) + monkeypatch.setenv(masking.DEFAULT_SALT_ENV, SALT) + # A directory of its own: the runner refuses a contract directory that holds more than the + # product (here the test's HOME and credential file), as it should. + product = tmp_path / "product" + product.mkdir() + return product + + +def _csv(root: Path, rows: List[Tuple[int, float, str]]) -> Path: + path = root / "orders.csv" + path.write_text( + "id,amount,region\n" + "".join(f"{i},{a},{r}\n" for i, a, r in rows), encoding="utf-8" + ) + return path + + +def _contract(root: Path) -> Dict[str, Any]: + contract = make_contract() + expose = contract["exposes"][0] + # varchar, not string: the run-time schema gate compares the declared type with DuckDB's own + for column in expose["contract"]["schema"]: + if column["type"] == "string": + column["type"] = "varchar" + expose["policy"]["privacy"] = {"masking": [{"column": "region", "strategy": "hash"}]} + expose["policy"]["authz"]["columnRestrictions"] = [ + {"principal": "group:analysts", "columns": ["amount"], "access": "deny"} + ] + expose["policy"]["authz"]["rowFilters"] = [{"principal": "group:analysts", "where": "id > 0"}] + contract["builds"] = [ + { + "id": "ingest_orders", + "pattern": "acquisition", + "engine": "duckdb", + "properties": { + "source": { + "kind": "filesystem", + "mode": "full_refresh", + "connection": {"uri": str(root / "orders.csv")}, + "reader": {"format": "csv", "options": {"header": True}}, + "streams": ["orders"], + }, + "sink": {"format": "parquet"}, + }, + "outputs": ["orders"], + } + ] + return contract + + +def _sha(value: str) -> str: + return hashlib.sha256((SALT + value).encode("utf-8")).hexdigest() + + +def _by_id(fake: FakeDatabricks) -> List[Dict[str, Any]]: + return sorted(fake.rows(TABLE), key=lambda r: r["id"]) + + +def test_apply_creates_the_table_then_the_build_lands_masked_bronze_in_it(workspace, fake, capfd): + _csv(workspace, FIRST) + path = write_contract(workspace, _contract(workspace)) + + # 1. apply: the masking is accepted (the runner applies it); the table, grants, mask and filter exist + rc, out = apply_contract(capfd, path) + assert rc == 0, out + assert "databricks-governance-unmapped" not in out + assert fake.rows(TABLE) == [], "apply creates the table empty" + assert fake.tables[TABLE]["row_filter"], "the row filter is bound" + assert any(c.get("mask") for c in fake.tables[TABLE]["columns"]), "and the column mask" + grants = dict(fake.grants[("table", TABLE)]) + definition_before = repr(fake.tables[TABLE]) + sql_before = len(fake.sql_log) + + # 2. the build: hash at landing, upload, INSERT OVERWRITE, delete the staged files + rc, out = apply_contract(capfd, path, "--mode", "amend-and-build") + assert rc == 0, out + rows = _by_id(fake) + assert [(r["id"], r["amount"]) for r in rows] == [(i, a) for i, a, _ in FIRST] + assert [r["region"] for r in rows] == [_sha(r) for _, _, r in FIRST] + assert "emea" not in repr(fake.rows(TABLE)), "the cleartext never left the machine" + assert fake.volume_files() == [], "no landed data is left in the volume" + uploads = [p for m, p in fake.file_ops if m == "PUT"] + assert len(uploads) == 1 and uploads[0].startswith( + "/Volumes/main/staging/landing/dbx_fake_orders/" + ) + assert not (workspace / "out").exists() or list((workspace / "out").glob("*")) == [] + assert fake.unknown == [] + + # the table keeps its definition, grants, row filter and mask: the load is the only statement + assert repr(fake.tables[TABLE]) == definition_before + assert dict(fake.grants[("table", TABLE)]) == grants + assert fake.replaced == [] + after = fake.sql_log[sql_before:] + loads = [s for s in after if "INSERT OVERWRITE" in s] + assert len(loads) == 1 and "read_files(" in loads[0] + assert [s for s in after if "INSERT OVERWRITE" not in s] == [], "nothing else ran" + + # 3. loading data does not drift the table + rc, out = apply_contract(capfd, path, "--mode", "dry-run") + assert rc == 0, out + assert "tofu plan: +0 ~0 -0" in out + + +def test_a_second_run_replaces_the_rows(workspace, fake, capfd): + _csv(workspace, FIRST) + path = write_contract(workspace, _contract(workspace)) + assert apply_contract(capfd, path)[0] == 0 + assert apply_contract(capfd, path, "--mode", "amend-and-build")[0] == 0 + assert len(fake.rows(TABLE)) == 3 + + _csv(workspace, SECOND) + marker = fake.tables[TABLE]["marker"] + rc, out = apply_contract(capfd, path, "--mode", "amend-and-build") + assert rc == 0, out + assert [r["id"] for r in _by_id(fake)] == [7, 8], "overwritten, not appended" + assert fake.volume_files() == [] + assert fake.tables[TABLE]["marker"] == marker, "the table was never replaced" diff --git a/tests/iac/test_iac_databricks_masking_at_landing.py b/tests/iac/test_iac_databricks_masking_at_landing.py new file mode 100644 index 000000000..041b55b97 --- /dev/null +++ b/tests/iac/test_iac_databricks_masking_at_landing.py @@ -0,0 +1,286 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``policy.privacy.masking`` on a Databricks expose: accepted when forge-cli applies it. + +The acquisition runner masks at landing, before the rows leave the machine +(``build_runners/_masking.py``), and lands them in the Databricks table +(``build_runners/_databricks_load.py``). So an expose every build of which is a DuckDB +acquisition build keeps its ``policy.privacy.masking``, as on AWS. Anything else that writes the +table (dbt, another engine, no build at all) applies no mask, and the emitter keeps refusing the +field, with the reason: applied anyway, the table would hold the values the contract promises to +mask. + +``fluid validate`` and ``fluid apply`` read the verdict from the one ``_compile`` pass, so each +case below is checked through both. +""" + +from __future__ import annotations + +import copy +from typing import Any, Dict, List, Optional + +import pytest + +from fluid_build.iac import IAC_PLUGINS +from fluid_build.iac.base import UnsupportedBindingError +from fluid_build.iac.providers import databricks as dbx + +pytestmark = [pytest.mark.unit, pytest.mark.databricks] + +MASKING = [{"column": "msisdn", "strategy": "hash"}] + + +@pytest.fixture(autouse=True) +def _warehouse(monkeypatch): + monkeypatch.setenv(dbx.WAREHOUSE_ENV, "wh-0123456789abcdef") + + +def _build( + build_id: str = "ingest", + *, + engine: str = "duckdb", + pattern: Optional[str] = "acquisition", + outputs: Optional[List[str]] = ("orders",), +) -> Dict[str, Any]: + build: Dict[str, Any] = { + "id": build_id, + "engine": engine, + "properties": { + "source": {"kind": "filesystem", "mode": "full_refresh"}, + "sink": {"format": "parquet"}, + }, + } + if pattern: + build["pattern"] = pattern + if outputs is not None: + build["outputs"] = list(outputs) + return build + + +def _contract( + *builds: Dict[str, Any], masking: Any = MASKING, **expose_extra: Any +) -> Dict[str, Any]: + expose: Dict[str, Any] = { + "exposeId": "orders", + "kind": "table", + "binding": { + "platform": "databricks", + "format": "delta_table", + "location": {"database": "main", "schema": "bronze", "table": "orders"}, + }, + "policy": {"authz": {"readers": ["group:analysts"]}, "privacy": {"masking": masking}}, + "contract": { + "schema": [{"name": "id", "type": "bigint"}, {"name": "msisdn", "type": "string"}] + }, + } + expose.update(expose_extra) + contract: Dict[str, Any] = { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "bronze.orders", + "name": "Orders", + "exposes": [expose], + } + if builds: + contract["builds"] = list(builds) + return contract + + +def _emit(contract: Dict[str, Any]) -> Optional[UnsupportedBindingError]: + try: + IAC_PLUGINS["databricks"].emit(copy.deepcopy(contract)) + except UnsupportedBindingError as exc: + return exc + return None + + +def _validate(contract: Dict[str, Any]) -> List[str]: + return dbx.validate_databricks_binding(copy.deepcopy(contract))[0] + + +def _both(contract: Dict[str, Any]) -> Optional[str]: + """The refusal text, asserting that validate and emit agree on whether there is one.""" + refusal = _emit(contract) + errors = _validate(contract) + assert bool(errors) == (refusal is not None), (errors, refusal) + if refusal is None: + return None + assert refusal.kind == "databricks-governance-unmapped" or refusal.kind == "governance-unmapped" + assert "policy.privacy.masking" in str(refusal) + assert any("policy.privacy.masking" in e for e in errors) + return str(refusal) + + +# ── accepted: every build that writes the expose applies the mask ───────── + + +def test_a_duckdb_acquisition_build_that_writes_the_expose_makes_masking_acceptable(): + assert _both(_contract(_build())) is None + + +def test_a_duckdb_build_that_names_no_outputs_is_taken_to_write_it(): + assert _both(_contract(_build(outputs=None))) is None + + +def test_two_duckdb_acquisition_builds_both_apply_it(): + assert _both(_contract(_build("a"), _build("b"))) is None + + +def test_a_build_that_writes_another_expose_does_not_matter(): + other = _build("other", engine="dbt", pattern="embedded-logic", outputs=["something_else"]) + assert _both(_contract(_build(), other)) is None + + +def test_the_engine_is_read_case_insensitively(): + assert _both(_contract(_build(engine=" DuckDB "))) is None + + +def test_masking_and_column_restrictions_are_both_enforced(): + contract = _contract(_build()) + contract["exposes"][0]["policy"]["authz"]["columnRestrictions"] = [ + {"principal": "group:analysts", "columns": ["msisdn"], "access": "deny"} + ] + assert _both(contract) is None + + +def test_the_table_is_emitted_exactly_as_without_masking(): + """Masking is the runner's, not the table's: the module has no trace of it.""" + with_mask = _contract(_build()) + without = _contract(_build()) + without["exposes"][0]["policy"]["privacy"] = {} + a = IAC_PLUGINS["databricks"].emit(copy.deepcopy(with_mask)) + b = IAC_PLUGINS["databricks"].emit(copy.deepcopy(without)) + assert a == b + + +# ── refused, with the reason ────────────────────────────────────────────── + + +def test_a_contract_with_no_build_that_writes_the_expose_is_refused(): + text = _both(_contract()) + assert "machine. No build writes it" in text and "DuckDB acquisition" in text + + +def test_a_dbt_build_writing_the_expose_is_refused_naming_it(): + text = _both(_contract(_build("models", engine="dbt", pattern="embedded-logic"))) + assert "'models'" in text and "dbt" in text + assert "DuckDB acquisition" in text + + +def test_a_dbt_build_beside_a_duckdb_one_is_still_refused(): + text = _both(_contract(_build(), _build("models", engine="dbt-databricks", pattern=None))) + assert "'models'" in text and "'ingest'" not in text + + +@pytest.mark.parametrize( + "engine", ["dlt", "airbyte", "meltano", "kafka-connect", "debezium", "spark"] +) +def test_another_engine_applies_no_mask_here(engine): + text = _both(_contract(_build("other", engine=engine))) + assert "'other'" in text and engine in text + + +def test_a_duckdb_build_that_is_not_an_acquisition_build_applies_no_mask(): + """Embedded SQL on DuckDB does not run ``_masking`` and cannot land a Databricks table.""" + text = _both(_contract(_build("sql", engine="duckdb", pattern="embedded-logic"))) + assert "'sql'" in text + + +def test_a_build_with_no_engine_applies_no_mask(): + build = _build("plain") + del build["engine"] + assert "'plain'" in _both(_contract(build)) + + +# ── the rest of the governance block is unchanged ───────────────────────── + + +@pytest.mark.parametrize( + "extra, field", + [ + ({"policy": {"privacy": {"rowLevelPolicy": {"a": 1}}}}, "policy.privacy.rowLevelPolicy"), + ( + { + "binding": { + "platform": "databricks", + "format": "delta_table", + "location": {"database": "main", "schema": "bronze", "table": "orders"}, + "governance": {"lakeFormation": {"grants": []}}, + } + }, + "binding.governance", + ), + ], +) +def test_row_level_policy_and_binding_governance_stay_refused(extra, field): + contract = _contract(_build(), masking=[]) + contract["exposes"][0].update(extra) + refusal = _emit(contract) + assert refusal is not None and field in str(refusal) + assert any(field in e for e in _validate(contract)) + + +def test_row_level_policy_beside_accepted_masking_is_still_refused_alone(): + contract = _contract(_build()) + contract["exposes"][0]["policy"]["privacy"]["rowLevelPolicy"] = {"a": 1} + refusal = str(_emit(contract)) + assert "policy.privacy.rowLevelPolicy" in refusal + assert "policy.privacy.masking" not in refusal, "the accepted field is not named as refused" + + +@pytest.mark.parametrize("masking", [None, [], {}]) +def test_an_empty_masking_block_is_not_masking(masking): + assert _both(_contract(masking=masking)) is None + + +@pytest.mark.parametrize( + "source, sink, fragment", + [ + ({"mode": "incremental_append"}, {"format": "parquet"}, "full_refresh"), + ({"mode": "full_refresh"}, {"format": "csv"}, "sink format"), + ({}, {"format": "parquet"}, "modes"), + ], +) +def test_a_landing_the_runner_would_refuse_is_refused_before_the_table_exists( + source, sink, fragment +): + build = _build() + build["properties"] = {"source": {"kind": "filesystem", **source}, "sink": sink} + refusal = _emit(_contract(build)) + assert refusal is not None and refusal.kind == "databricks-landing-unsupported" + assert fragment in str(refusal) and "'ingest'" in str(refusal) + + +def test_a_placeholder_mode_is_left_to_the_emitter(): + build = _build() + build["properties"]["source"]["mode"] = "{{ env.MODE }}" + errors, _ = dbx.validate_databricks_binding(_contract(build)) + assert not [e for e in errors if "landing" in e] + # at apply the contract is substituted; a variable still unset is refused there + refusal = _emit(_contract(build)) + assert refusal is not None and refusal.kind == "databricks-landing-unsupported" + + +@pytest.mark.parametrize( + "masking", + [ + [{"column": "msisdn", "strategy": "k_anonymity"}], + [{"column": "msisdn", "strategy": "no-such-strategy"}], + {"column": "msisdn", "strategy": "hash"}, + ], +) +def test_a_masking_rule_the_runner_cannot_apply_is_refused(masking): + refusal = _emit(_contract(_build(), masking=masking)) + assert refusal is not None and refusal.kind == "databricks-masking-invalid" diff --git a/tests/iac/test_iac_databricks_retention.py b/tests/iac/test_iac_databricks_retention.py new file mode 100644 index 000000000..0d5a86d9c --- /dev/null +++ b/tests/iac/test_iac_databricks_retention.py @@ -0,0 +1,856 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``lifecycle.expire`` on Databricks: accepted for a table rebuilt in full, Delta history limited. + +Pure functions only. The spec is the module docstring of +``iac/providers/databricks_retention.py``: a dbt ``table`` model or forge-cli's DuckDB +acquisition rewrites the whole table, so ``expire`` is accepted when the contract's retention +covers the Delta history (30 days by default); anything else is refused. +""" + +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any, Dict, List, Optional + +import pytest + +from fluid_build.iac import IAC_PLUGINS +from fluid_build.iac.base import UnsupportedBindingError +from fluid_build.iac.providers import databricks as dbx +from fluid_build.iac.providers import databricks_retention as ret +from fluid_build.iac.source import contract_source + +pytestmark = [pytest.mark.unit, pytest.mark.databricks] + +WAREHOUSE = "wh-0123456789abcdef" +DAY = 86400 + + +# ── durations ─────────────────────────────────────────────────────────────── +@pytest.mark.parametrize( + "text,seconds", + [ + ("P30D", 30 * DAY), + ("P365D", 365 * DAY), + ("P6Y", 6 * 365 * DAY), + ("P1Y", 365 * DAY), # the shortest year: never read as longer than it is + ("P1M", 28 * DAY), # the shortest month + ("P2M", 56 * DAY), + ("P1Y2M3W4D", (365 + 56 + 21 + 4) * DAY), + ("P4W", 28 * DAY), + ("PT720H", 30 * DAY), + ("PT36H", 36 * 3600), + ("P1DT12H30M15S", DAY + 12 * 3600 + 30 * 60 + 15), + ("P0D", 0), + ("PT0S", 0), + (" P7D ", 7 * DAY), + ], +) +def test_iso_duration(text, seconds): + assert ret.iso_duration_seconds(text) == seconds + + +@pytest.mark.parametrize( + "text", + [ + "", + "P", + "PT", + "P1DT", # a T with no time part + "30 days", + "p30d", + "P-1D", + "P1.5D", + "P1,5D", + "P1H", # hours need the T + "PT1D", + "P1D1Y", # out of order + "P" + "9" * 13 + "D", + None, + 30, + ], +) +def test_iso_duration_rejects(text): + with pytest.raises(ValueError): + ret.iso_duration_seconds(text) + + +@pytest.mark.parametrize( + "text,seconds", + [ + ("interval 30 days", 30 * DAY), + ("INTERVAL 7 DAYS", 7 * DAY), + ("interval 1 week", 7 * DAY), + ("interval 1 week 12 hours", 7 * DAY + 12 * 3600), + ("interval 48 hours", 48 * 3600), + ("30 days", 30 * DAY), + ("interval 1500 milliseconds", 2), # rounded up, never down + ], +) +def test_interval(text, seconds): + assert ret.interval_seconds(text) == seconds + + +@pytest.mark.parametrize( + "text", ["", "interval", "interval 1 month", "interval 1 year", "interval -1 days", "soon", 5] +) +def test_interval_rejects(text): + with pytest.raises(ValueError): + ret.interval_seconds(text) + + +def test_history_is_the_larger_of_the_two_properties_and_defaults_to_30_days(): + assert ret.history_seconds({}) == 30 * DAY + assert ret.history_seconds({ret.LOG_RETENTION: "interval 10 days"}) == 10 * DAY + assert ret.history_seconds({ret.LOG_RETENTION: "interval 2 days"}) == 7 * DAY + assert ret.history_seconds({ret.DELETED_FILE_RETENTION: "interval 90 days"}) == 90 * DAY + # names match case-insensitively + assert ret.history_seconds({"DELTA.LOGRETENTIONDURATION": "interval 400 days"}) == 400 * DAY + with pytest.raises(ValueError): + ret.history_seconds({ret.LOG_RETENTION: "forever"}) + + +# ── the contract ──────────────────────────────────────────────────────────── +def _set(target: Any, path: str, value: Any) -> None: + keys = path.split(".") + for key in keys[:-1]: + target = target[int(key)] if isinstance(target, list) else target.setdefault(key, {}) + if isinstance(target, list): + target[int(keys[-1])] = value + else: + target[keys[-1]] = value + + +def _contract(**overrides: Any) -> Dict[str, Any]: + """``orders`` on Databricks with ``expire`` after P365D, written by a local dbt build.""" + contract: Dict[str, Any] = { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "sales.orders", + "name": "Orders", + "exposes": [ + { + "exposeId": "orders", + "kind": "table", + "binding": { + "platform": "databricks", + "format": "delta_table", + "location": {"database": "main", "schema": "sales", "table": "orders"}, + }, + "lifecycle": {"retention": "P365D", "expire": True}, + "contract": {"schema": [{"name": "id", "type": "bigint"}]}, + } + ], + "builds": [ + { + "id": "transform", + "engine": "dbt", + "repository": "./dbt", + "outputs": ["orders"], + "properties": {"model": "orders"}, + "execution": {"runtime": {"platform": "databricks"}}, + } + ], + } + for path, value in overrides.items(): + _set(contract, path.replace("__", "."), value) + return contract + + +def _acquisition(mode: Any = "full_refresh", **build: Any) -> Dict[str, Any]: + source = {"kind": "filesystem", **({"mode": mode} if mode is not None else {})} + out = { + "id": "land", + "engine": "duckdb", + "pattern": "acquisition", + "outputs": ["orders"], + **build, + } + out["properties"] = {"source": source, **(build.get("properties") or {})} + return out + + +def _project( + tmp_path: Path, + sql: str = "select 1 as id\n", + *, + project_models: str = "", + yaml_config: Optional[str] = None, + name: str = "orders", + folder: str = "", + extra: Optional[Dict[str, str]] = None, +) -> Path: + root = tmp_path / "dbt" + root.mkdir(exist_ok=True) + (root / "dbt_project.yml").write_text(f"name: shop\n{project_models}", encoding="utf-8") + models = root / "models" / folder + models.mkdir(parents=True, exist_ok=True) + (models / f"{name}.sql").write_text(sql, encoding="utf-8") + if yaml_config is not None: + (models / "schema.yml").write_text( + f"models:\n - name: {name}\n config:\n{yaml_config}", encoding="utf-8" + ) + for rel, text in (extra or {}).items(): + path = root / "models" / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return root + + +def _compile( + contract: Dict[str, Any], tmp_path: Optional[Path] = None, *, validating: bool = False +) -> dbx._Compiled: + source = tmp_path / "contract.fluid.yaml" if tmp_path is not None else None + with contract_source(source): + return dbx._compile(contract, {dbx.WAREHOUSE_ENV: WAREHOUSE}, validating=validating) + + +def _kinds(compiled: dbx._Compiled) -> List[str]: + return [refusal.kind for refusal in compiled.refusals] + + +def _accepted(compiled: dbx._Compiled) -> dbx._Table: + assert not compiled.refusals, [f"{r.kind}: {r}" for r in compiled.refusals] + (table,) = compiled.tables + assert table.retention is not None + return table + + +def _refused(compiled: dbx._Compiled, kind: str) -> UnsupportedBindingError: + assert f"databricks-retention-{kind}" in _kinds(compiled), _kinds(compiled) + assert not compiled.tables, "a refused expose emits no table" + return next(r for r in compiled.refusals if r.kind == f"databricks-retention-{kind}") + + +# ── a dbt model that is rebuilt in full ───────────────────────────────────── +class TestDbtTable: + def test_sql_config_table_is_accepted(self, tmp_path): + _project(tmp_path, "{{ config(materialized='table') }}\nselect 1 as id\n") + table = _accepted(_compile(_contract(), tmp_path)) + assert table.retention.period == "P365D" and table.retention.seconds == 365 * DAY + + def test_double_quotes_whitespace_control_and_other_args(self, tmp_path): + _project( + tmp_path, + '{{- config(\n materialized="table",\n partition_by=["id"],\n tags=["a"]\n) -}}\n' + "select 1 as id\n", + ) + _accepted(_compile(_contract(), tmp_path)) + + def test_yaml_config_table_is_accepted(self, tmp_path): + _project(tmp_path, yaml_config=" materialized: table\n") + _accepted(_compile(_contract(), tmp_path)) + + def test_project_config_with_the_plus_prefix(self, tmp_path): + _project(tmp_path, project_models="models:\n shop:\n +materialized: table\n") + _accepted(_compile(_contract(), tmp_path)) + + def test_project_config_without_the_plus_prefix(self, tmp_path): + _project(tmp_path, project_models="models:\n shop:\n materialized: table\n") + _accepted(_compile(_contract(), tmp_path)) + + def test_root_level_project_config(self, tmp_path): + _project(tmp_path, project_models="models:\n +materialized: table\n") + _accepted(_compile(_contract(), tmp_path)) + + def test_folder_config(self, tmp_path): + _project( + tmp_path, + folder="marts", + project_models="models:\n shop:\n +materialized: view\n marts:\n" + " +materialized: table\n", + ) + _accepted(_compile(_contract(), tmp_path)) + + def test_config_named_for_the_model_itself(self, tmp_path): + _project( + tmp_path, + folder="marts", + project_models="models:\n shop:\n marts:\n +materialized: view\n" + " orders:\n +materialized: table\n", + ) + _accepted(_compile(_contract(), tmp_path)) + + # dbt: config() in the model > the model's YAML config > dbt_project.yml, and the + # most specific path in dbt_project.yml. + @pytest.mark.parametrize( + "project,yaml,sql,accepted", + [ + ("incremental", "table", None, True), # YAML over the project + ("table", "incremental", None, False), + ("view", "view", "table", True), # SQL over both + ("table", "table", "incremental", False), + ("incremental", "incremental", "table", True), + (None, "view", "table", True), + (None, "table", "view", False), + ], + ) + def test_precedence(self, tmp_path, project, yaml, sql, accepted): + _project( + tmp_path, + (f"{{{{ config(materialized='{sql}') }}}}\n" if sql else "") + "select 1 as id\n", + project_models=(f"models:\n shop:\n +materialized: {project}\n" if project else ""), + yaml_config=f" materialized: {yaml}\n" if yaml else None, + ) + compiled = _compile(_contract(), tmp_path) + if accepted: + _accepted(compiled) + else: + assert compiled.refusals and not compiled.tables + + def test_the_more_specific_project_path_wins(self, tmp_path): + _project( + tmp_path, + folder="marts", + project_models="models:\n shop:\n marts:\n +materialized: incremental\n" + " +materialized: table\n", + ) + _refused(_compile(_contract(), tmp_path), "incremental") + + def test_later_config_call_wins(self, tmp_path): + _project( + tmp_path, + "{{ config(materialized='incremental') }}\n{{ config(materialized='table') }}\n" + "select 1 as id\n", + ) + _accepted(_compile(_contract(), tmp_path)) + + def test_a_commented_out_or_raw_config_is_not_read(self, tmp_path): + _project( + tmp_path, + "{# {{ config(materialized='incremental') }} #}\n" + "{% raw %}{{ config(materialized='incremental') }}{% endraw %}\n" + "{{ config(materialized='table') }}\nselect 1 as id\n", + ) + _accepted(_compile(_contract(), tmp_path)) + + def test_the_model_is_found_by_alias(self, tmp_path): + _project( + tmp_path, + "{{ config(materialized='table', alias='orders') }}\nselect 1 as id\n", + name="stg_orders_final", + ) + # dbt selects by the model's name (its file), not its alias. + contract = _contract(builds__0__properties={"select": "stg_orders_final"}) + _accepted(_compile(contract, tmp_path)) + _refused(_compile(_contract(), tmp_path), "not-selected") + + def test_the_model_is_the_one_named_for_the_table_not_its_neighbours(self, tmp_path): + _project( + tmp_path, + "{{ config(materialized='table') }}\nselect 1 as id\n", + extra={"other.sql": "{{ config(materialized='incremental') }}\nselect 1\n"}, + ) + _accepted(_compile(_contract(), tmp_path)) + + def test_a_table_is_found_case_insensitively(self, tmp_path): + _project(tmp_path, "{{ config(materialized='table') }}\nselect 1 as id\n", name="Orders") + # The relation is found without case; dbt's selection is case-sensitive. + _accepted(_compile(_contract(builds__0__properties={"model": "Orders"}), tmp_path)) + _refused(_compile(_contract(), tmp_path), "not-selected") + + +class TestEnvironmentPlaceholders: + """``fluid validate`` reads the contract before ``{{ env.X }}`` is substituted; the emitter + after. A placeholder is left to the emitter, never refused for its own text.""" + + def _check(self, contract, table="orders"): + return ret.check_expire( + "exposes[0]", contract["exposes"][0], table, contract, validating=True + ) + + def test_a_placeholder_table_name_is_left_to_the_emitter(self, tmp_path): + _project(tmp_path, "{{ config(materialized='table') }}\nselect 1 as id\n") + with contract_source(tmp_path / "contract.fluid.yaml"): + retention, refusals = self._check(_contract(), table="{{ env.tbl }}") + assert refusals == [] and retention is not None + + def test_a_placeholder_retention_is_left_to_the_emitter_and_promises_nothing(self): + contract = _contract( + builds=[_acquisition()], + exposes__0__lifecycle={"retention": "{{ env.RETENTION }}", "expire": True}, + ) + assert self._check(contract) == (None, []) + + def test_a_placeholder_source_mode_is_left_to_the_emitter(self): + assert self._check(_contract(builds=[_acquisition("{{ env.MODE }}")]))[1] == [] + + def test_the_emitter_still_refuses_the_value_once_substituted(self): + contract = _contract(builds=[_acquisition("incremental_append")]) + _refused(_compile(contract), "incremental") + + def test_a_retention_of_hours_names_a_whole_day_count_that_covers_it(self, tmp_path): + _project( + tmp_path, + "{{ config(materialized='table', tblproperties=" + "{'delta.logRetentionDuration': 'interval 721 hours'}) }}\nselect 1 as id\n", + ) + contract = _contract(exposes__0__lifecycle={"retention": "P30D", "expire": True}) + refusal = _refused(_compile(contract, tmp_path), "too-short") + assert "at least P31D" in " ".join(refusal.remediation) + + +class TestJinjaScan: + """``sql_config`` reads a model's tags in linear time: a model file is the dbt author's, + but CI runs ``fluid validate`` on a pull request's.""" + + @pytest.mark.parametrize("opener", ["{{", "{%", "{#", "{% raw %}"]) + def test_a_file_of_unclosed_openers_is_read_quickly(self, opener): + import time + + started = time.monotonic() + assert ret.sql_config(opener * 200_000) == {} + assert time.monotonic() - started < 2.0 + + def test_tags_comments_and_raw_blocks_are_read_as_the_regexes_did(self): + text = ( + "{# {{ config(materialized='view') }} #}" + "{% raw %}{{ config(materialized='view') }}{% endraw %}" + "{{ config(materialized='table') }} {{ unclosed" + ) + assert ret.sql_config(text) == {"materialized": "table"} + + +class TestDbtRefused: + @pytest.mark.parametrize("kind", ["incremental", "snapshot"]) + def test_a_model_that_keeps_old_rows(self, tmp_path, kind): + _project(tmp_path, f"{{{{ config(materialized='{kind}') }}}}\nselect 1 as id\n") + refusal = _refused(_compile(_contract(), tmp_path), "incremental") + assert kind in str(refusal) and "materialized='table'" in " ".join(refusal.remediation) + + @pytest.mark.parametrize("kind", ["view", "ephemeral", "materialized_view", "Table", "x"]) + def test_a_model_that_is_not_a_table(self, tmp_path, kind): + _project(tmp_path, f"{{{{ config(materialized='{kind}') }}}}\nselect 1 as id\n") + _refused(_compile(_contract(), tmp_path), "not-table") + + def test_dbts_default_is_a_view(self, tmp_path): + _project(tmp_path) + refusal = _refused(_compile(_contract(), tmp_path), "not-table") + assert "view" in str(refusal) + + @pytest.mark.parametrize( + "sql", + [ + "{{ config(materialized=var('m')) }}\nselect 1", + "{{ config(materialized=mat) }}\nselect 1", + "{% if var('x') %}{{ config(materialized='table') }}{% endif %}\nselect 1", + "{% for x in y %}{{ config(materialized='table') }}{% endfor %}\nselect 1", + "{{ config(**opts) }}\nselect 1", + "{{ config('table') }}\nselect 1", + "{{ config(materialized='table' }}\nselect 1", + "{{ config(schema='other', materialized='table') }}\nselect 1", + "{{ config(database='other', materialized='table') }}\nselect 1", + "{{ config(enabled=false, materialized='table') }}\nselect 1", + ], + ) + def test_what_cannot_be_determined_is_refused_with_the_fix(self, tmp_path, sql): + _project(tmp_path, sql) + refusal = _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + assert "materialized='table'" in " ".join(refusal.remediation) + + def test_a_config_after_a_closed_block_is_still_read(self, tmp_path): + _project( + tmp_path, + "{% set x = 1 %}{% if x %}{% endif %}{{ config(materialized='table') }}\nselect 1", + ) + _accepted(_compile(_contract(), tmp_path)) + + def test_a_project_jinja_value_is_not_a_literal(self, tmp_path): + _project( + tmp_path, + project_models="models:\n shop:\n +materialized: \"{{ var('m') }}\"\n", + ) + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_a_git_repository_is_not_readable_here(self, tmp_path): + contract = _contract(builds__0__repository="https://github.com/o/r?tag=v1&path=dbt") + refusal = _refused(_compile(contract, tmp_path), "dbt-unresolved") + assert "Git repository" in str(refusal) and "?tag" not in str(refusal) + assert "local dbt project" in " ".join(refusal.remediation) + + def test_no_project_at_the_repository(self, tmp_path): + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_no_model_creates_the_table(self, tmp_path): + _project(tmp_path, "{{ config(materialized='table') }}\nselect 1", name="customers") + refusal = _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + assert "'orders'" in str(refusal) + + def test_two_models_creating_it(self, tmp_path): + _project( + tmp_path, + "{{ config(materialized='table') }}\nselect 1", + extra={"b.sql": "{{ config(alias='orders') }}\nselect 1"}, + ) + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_an_alias_that_is_not_a_literal_cannot_be_ruled_out(self, tmp_path): + _project( + tmp_path, + "{{ config(materialized='table') }}\nselect 1", + extra={"b.sql": "{{ config(alias=var('a')) }}\nselect 1"}, + ) + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_a_python_model(self, tmp_path): + _project(tmp_path, name="other", extra={"orders.py": "def model(dbt, s): pass\n"}) + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_a_model_path_outside_the_project(self, tmp_path): + _project(tmp_path, "{{ config(materialized='table') }}\nselect 1") + (tmp_path / "dbt" / "dbt_project.yml").write_text( + "name: shop\nmodel-paths: ['../x']\n", encoding="utf-8" + ) + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_a_broken_dbt_project_yml(self, tmp_path): + _project(tmp_path) + (tmp_path / "dbt" / "dbt_project.yml").write_text("name: [", encoding="utf-8") + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_a_linked_model_is_not_followed(self, tmp_path): + _project(tmp_path, name="other") + outside = tmp_path / "outside.sql" + outside.write_text("{{ config(materialized='table') }}\nselect 1", encoding="utf-8") + (tmp_path / "dbt" / "models" / "orders.sql").symlink_to(outside) + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_a_model_path_through_a_linked_folder_is_not_followed(self, tmp_path): + _project(tmp_path, name="other") + outside = tmp_path / "outside" / "sub" + outside.mkdir(parents=True) + (outside / "orders.sql").write_text( + "{{ config(materialized='table') }}\nselect 1", encoding="utf-8" + ) + (tmp_path / "dbt" / "linked").symlink_to(tmp_path / "outside") + (tmp_path / "dbt" / "dbt_project.yml").write_text( + "name: shop\nmodel-paths: ['linked/sub']\n", encoding="utf-8" + ) + _refused(_compile(_contract(), tmp_path), "dbt-unresolved") + + def test_validate_without_a_contract_file_reads_nothing(self): + # As for the job emitter: nothing to read yet, so nothing refused for the project. + _accepted(_compile(_contract(), None, validating=True)) + # The emit path knows no location, and says so. + _refused(_compile(_contract(), None), "dbt-unresolved") + + +def _selecting(properties: Dict[str, Any], **overrides: Any) -> Dict[str, Any]: + return _contract(builds__0__properties=properties, **overrides) + + +class TestSelection: + """The guarantee needs the build to run the model: its ``--select`` must include it.""" + + @pytest.fixture(autouse=True) + def _project(self, tmp_path): + _project( + tmp_path, + "{{ config(materialized='table') }}\nselect 1 as id\n", + folder="marts", + extra={"other.sql": "select 1\n"}, + ) + + @pytest.mark.parametrize( + "properties", + [ + {"model": "orders"}, # +orders+ + {"select": "orders"}, + {"select": ["orders"]}, + {"select": "orders+"}, + {"select": "+orders"}, + {"select": "2+orders"}, + {"select": "orders+3"}, + {"select": "@orders"}, + {"models": "orders"}, + {"select": "fqn:orders"}, + {"select": "model:orders"}, + {"select": "shop.marts.orders"}, + {"select": "shop.marts"}, + {"select": "shop.*"}, + {"select": "shop"}, # the project's name selects its every node + {"select": "marts"}, # dbt matches the fqn without the project's name too + {"select": "marts.orders"}, + {"select": "marts.*"}, + {"select": "shop.m*.orders"}, + {"select": "*"}, + {"select": "path:models/marts"}, + {"select": "path:models/marts/"}, + {"select": "models/marts/orders.sql"}, + {"select": ["other", "orders"]}, + {"select": "other orders"}, + {"select": "orders,path:models/marts"}, + {"select": "+orders+,shop.marts"}, + {}, # no selector: dbt builds everything + {"model": "other"}, # outputs > 1 : no selector, the whole project + ], + ) + def test_a_selection_that_includes_the_model_is_accepted(self, tmp_path, properties): + contract = _selecting(properties) + if properties == {"model": "other"}: + contract["builds"][0]["outputs"] = ["orders", "other"] + contract["exposes"].append(copy.deepcopy(contract["exposes"][0])) + contract["exposes"][1].update(exposeId="other") + contract["exposes"][1]["binding"]["location"]["table"] = "other" + contract["exposes"][1].pop("lifecycle") + compiled = _compile(contract, tmp_path) + assert not compiled.refusals, [f"{r.kind}: {r}" for r in compiled.refusals] + + @pytest.mark.parametrize( + "properties", + [ + {"select": "other"}, + {"select": "customers"}, + {"select": "shop.orders"}, # dbt's fqn is a prefix: marts is skipped + {"select": "ord*"}, # dbt globs the whole rest of the fqn: shop.marts.orders + {"select": "ma*ts"}, + {"select": "*.core"}, + {"select": "Orders"}, # case-sensitive + {"select": "models"}, # no slash: a name, not the path + {"select": ["other", "customers"]}, + {"select": "shop.other"}, + {"select": "path:models/staging"}, + {"select": "models/other.sql"}, + {"select": "orders,other"}, # intersection of two names + {"select": "stg_*"}, + ], + ) + def test_a_selection_that_leaves_it_out_is_refused(self, tmp_path, properties): + refusal = _refused(_compile(_selecting(properties), tmp_path), "not-selected") + assert "'orders'" in str(refusal) and "model 'orders'" in str(refusal) + assert "does not select it" in str(refusal) + assert "properties.select" in " ".join(refusal.remediation) + + @pytest.mark.parametrize( + "properties", + [ + {"select": "tag:nightly"}, + {"select": "config.materialized:table"}, + {"select": "tag:nightly,orders"}, + {"model": "other"}, # +other+ + {"select": "+other"}, # other's ancestors: only dbt's graph knows + {"select": "other+"}, + {"select": "result:error"}, + ], + ) + def test_a_selection_only_the_graph_or_state_can_answer_is_refused(self, tmp_path, properties): + refusal = _refused(_compile(_selecting(properties), tmp_path), "not-selected") + assert "cannot evaluate" in str(refusal) + + def test_a_flag_like_selector_is_dropped_as_the_runner_drops_it(self, tmp_path): + # build_selectors removes it, leaving no selector: the same set dbt would run. + _accepted_or_not = _compile(_selecting({"select": ["--full-refresh"]}), tmp_path) + assert not _accepted_or_not.refusals + + def test_the_selection_is_named_in_the_message(self, tmp_path): + refusal = _refused( + _compile(_selecting({"select": ["other", "tag:x"]}), tmp_path), "not-selected" + ) + assert "other tag:x" in str(refusal) + + def test_not_selected_is_checked_before_the_materialization(self, tmp_path): + _project(tmp_path, "select 1\n", folder="marts") # a view, and not selected + compiled = _compile(_selecting({"select": "other"}), tmp_path) + assert "databricks-retention-not-selected" in _kinds(compiled) + assert "databricks-retention-not-table" not in _kinds(compiled) + + def test_an_acquisition_has_no_dbt_selection(self): + _accepted(_compile(_contract(builds=[_acquisition(properties={"select": "other"})]))) + + +class TestDeltaProperties: + def _table(self, tmp_path, props: str, retention: str = "P365D"): + _project( + tmp_path, + "{{ config(materialized='table', tblproperties=" + props + ") }}\nselect 1 as id\n", + ) + return _compile( + _contract(exposes__0__lifecycle={"retention": retention, "expire": True}), tmp_path + ) + + def test_a_longer_history_than_the_contract_is_refused(self, tmp_path): + compiled = self._table(tmp_path, "{'delta.logRetentionDuration': 'interval 400 days'}") + refusal = _refused(compiled, "too-short") + assert "interval 400 days (declared)" in str(refusal) + assert "7 days (Delta's default)" in str(refusal) + + def test_a_history_inside_the_contract_is_accepted(self, tmp_path): + compiled = self._table( + tmp_path, + "{'delta.logRetentionDuration': 'interval 400 days'," + " 'delta.deletedFileRetentionDuration': 'interval 7 days'}", + "P2Y", + ) + _accepted(compiled) + + def test_the_deleted_file_retention_counts_too(self, tmp_path): + compiled = self._table( + tmp_path, "{'delta.deletedFileRetentionDuration': 'interval 500 days'}" + ) + _refused(compiled, "too-short") + + def test_a_property_that_is_not_an_interval_is_refused(self, tmp_path): + compiled = self._table(tmp_path, "{'delta.logRetentionDuration': 'a long time'}") + _refused(compiled, "dbt-unresolved") + + +# ── the retention itself ──────────────────────────────────────────────────── +class TestPeriod: + @pytest.fixture(autouse=True) + def _dbt_table(self, tmp_path): + _project(tmp_path, "{{ config(materialized='table') }}\nselect 1 as id\n") + + @pytest.mark.parametrize("period", ["P30D", "P31D", "P365D", "P6Y", "P1Y", "P2M", "PT720H"]) + def test_a_retention_that_covers_the_history_is_accepted(self, tmp_path, period): + contract = _contract(exposes__0__lifecycle={"retention": period, "expire": True}) + _accepted(_compile(contract, tmp_path)) + + @pytest.mark.parametrize("period", ["P7D", "P1D", "P29D", "P4W", "P1M", "PT719H", "P0D"]) + def test_a_retention_shorter_than_30_days_is_refused(self, tmp_path, period): + contract = _contract(exposes__0__lifecycle={"retention": period, "expire": True}) + refusal = _refused(_compile(contract, tmp_path), "too-short") + assert "30 days" in str(refusal) and "P30D" in " ".join(refusal.remediation) + + @pytest.mark.parametrize("period", ["30 days", "P", "p30d", "P1.5D", 30, "P1DT"]) + def test_a_retention_that_is_not_an_iso_duration_is_refused(self, tmp_path, period): + contract = _contract(exposes__0__lifecycle={"retention": period, "expire": True}) + _refused(_compile(contract, tmp_path), "period") + + def test_expire_without_a_retention_is_refused(self, tmp_path): + contract = _contract(exposes__0__lifecycle={"expire": True}) + _refused(_compile(contract, tmp_path), "period") + + @pytest.mark.parametrize( + "lifecycle", [{"retention": "P7D"}, {"retention": "P7D", "expire": False}, {}] + ) + def test_without_expire_nothing_is_checked(self, tmp_path, lifecycle): + compiled = _compile(_contract(exposes__0__lifecycle=lifecycle), tmp_path) + assert not compiled.refusals and compiled.tables[0].retention is None + + +# ── the writers ───────────────────────────────────────────────────────────── +class TestWriters: + def test_a_duckdb_acquisition_is_accepted(self): + contract = _contract(builds=[_acquisition()]) + assert _accepted(_compile(contract)).retention.seconds == 365 * DAY + + def test_an_acquisition_without_outputs_is_still_the_writer(self): + build = _acquisition() + del build["outputs"] + _accepted(_compile(_contract(builds=[build]))) + + @pytest.mark.parametrize( + "mode", ["incremental_append", "incremental_merge", "incremental_dedup", "cdc", "streaming"] + ) + def test_an_acquisition_that_does_not_rewrite_the_table_is_refused(self, mode): + refusal = _refused(_compile(_contract(builds=[_acquisition(mode)])), "incremental") + assert repr(mode) in str(refusal) and "full_refresh" in str(refusal) + + def test_an_acquisition_without_a_source_mode_is_refused(self): + _refused(_compile(_contract(builds=[_acquisition(None)])), "unmapped") + + def test_the_mode_accepted_is_the_one_the_landing_runs(self): + from fluid_build.build_runners import _databricks_load as load + + for mode in load.LOAD_MODES: + assert load.unsupported_reason(mode, "parquet", 1) is None + _accepted(_compile(_contract(builds=[_acquisition(mode)]))) + + def test_an_acquisition_with_a_short_retention_is_still_refused(self): + contract = _contract( + builds=[_acquisition()], exposes__0__lifecycle={"retention": "P7D", "expire": True} + ) + _refused(_compile(contract), "too-short") + + @pytest.mark.parametrize( + "build", + [ + {"engine": "dlt", "pattern": "acquisition"}, + {"engine": "airbyte", "pattern": "acquisition"}, + {"engine": "duckdb", "pattern": "embedded-logic"}, + {"engine": "python"}, + {"engine": "spark"}, + {"engine": "flink"}, + ], + ) + def test_a_writer_not_known_to_rewrite_the_table_is_refused(self, build): + contract = _contract(builds=[{"id": "w", "outputs": ["orders"], **build}]) + refusal = _refused(_compile(contract), "unmapped") + assert "orders" in str(refusal) or "build 'w'" in str(refusal) + + def test_no_build_writes_it(self): + contract = _contract(builds=[]) + _refused(_compile(contract), "unmapped") + contract = _contract(builds=[{**_acquisition(), "outputs": ["something_else"]}]) + _refused(_compile(contract), "unmapped") + + def test_every_writer_must_rewrite_the_table(self, tmp_path): + contract = _contract( + builds=[_acquisition(), {"id": "w", "engine": "python", "outputs": ["orders"]}] + ) + _refused(_compile(contract, tmp_path), "unmapped") + + def test_a_dbt_writer_beside_an_acquisition(self, tmp_path): + _project(tmp_path, "{{ config(materialized='incremental') }}\nselect 1 as id\n") + contract = _contract() + contract["builds"].insert(0, _acquisition()) + _refused(_compile(contract, tmp_path), "incremental") + + +# ── validate and the emitter are one pass ─────────────────────────────────── +CASES = { + "table": ("{{ config(materialized='table') }}\nselect 1 as id\n", {}, None), + "incremental": ("{{ config(materialized='incremental') }}\nselect 1\n", {}, "incremental"), + "view": ("select 1\n", {}, "not-table"), + "var": ("{{ config(materialized=var('m')) }}\nselect 1\n", {}, "dbt-unresolved"), + "short": ( + "{{ config(materialized='table') }}\nselect 1\n", + {"exposes__0__lifecycle": {"retention": "P7D", "expire": True}}, + "too-short", + ), +} + + +@pytest.mark.parametrize("label", sorted(CASES)) +def test_validate_and_the_emitter_agree(tmp_path, monkeypatch, label): + sql, overrides, kind = CASES[label] + monkeypatch.setenv(dbx.WAREHOUSE_ENV, WAREHOUSE) + _project(tmp_path, sql) + contract = _contract(**overrides) + with contract_source(tmp_path / "contract.fluid.yaml"): + errors, _warnings = dbx.validate_databricks_binding(copy.deepcopy(contract)) + if kind is None: + assert not errors + assert "databricks_sql_table" in IAC_PLUGINS["databricks"].emit(contract) + return + assert errors + with pytest.raises(UnsupportedBindingError) as exc: + IAC_PLUGINS["databricks"].emit(contract) + assert exc.value.kind == f"databricks-retention-{kind}" + assert str(exc.value) in errors[0] + + +def test_managed_objects_carry_the_accepted_retention(tmp_path): + _project(tmp_path, "{{ config(materialized='table') }}\nselect 1 as id\n") + with contract_source(tmp_path / "contract.fluid.yaml"): + tables, _jobs = dbx.managed_objects(_contract(), {dbx.WAREHOUSE_ENV: WAREHOUSE}) + (table,) = tables + assert (table.retention_period, table.retention_seconds) == ("P365D", 365 * DAY) + plain = _contract(exposes__0__lifecycle={}) + with contract_source(tmp_path / "contract.fluid.yaml"): + (table,), _ = dbx.managed_objects(plain, {dbx.WAREHOUSE_ENV: WAREHOUSE}) + assert table.retention_seconds is None diff --git a/tests/iac/test_iac_entrypoint_discovery.py b/tests/iac/test_iac_entrypoint_discovery.py index a9e36823b..44f074449 100644 --- a/tests/iac/test_iac_entrypoint_discovery.py +++ b/tests/iac/test_iac_entrypoint_discovery.py @@ -26,9 +26,11 @@ @pytest.fixture(autouse=True) def _isolate_iac_registry(): saved = dict(R.IAC_PLUGINS) + R.reset_iac_plugin_cache() try: yield finally: + R.reset_iac_plugin_cache() R.IAC_PLUGINS.clear() R.IAC_PLUGINS.update(saved) @@ -73,7 +75,7 @@ def test_external_iac_plugin_registered_from_entrypoint(monkeypatch): monkeypatch.delenv("FLUID_PLUGINS_ALLOWLIST", raising=False) monkeypatch.delenv("FLUID_PLUGINS_BLOCKLIST", raising=False) - assert R.get_iac_plugin("mycloud") is None # not registered yet + assert "mycloud" not in R.IAC_PLUGINS # not registered until something asks R.discover_iac_entrypoints() plugin = R.get_iac_plugin("mycloud") assert plugin is not None diff --git a/tests/iac/test_iac_entrypoint_lazy_plugin.py b/tests/iac/test_iac_entrypoint_lazy_plugin.py new file mode 100644 index 000000000..338ccd6be --- /dev/null +++ b/tests/iac/test_iac_entrypoint_lazy_plugin.py @@ -0,0 +1,649 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An outside package ships an OpenTofu emitter through the +``fluid_build.iac_providers`` entry-point group. + +The fixtures build a REAL throwaway distribution (a ``.dist-info`` with an +``entry_points.txt`` plus a module) in a temp dir on ``sys.path``, so +``importlib.metadata`` discovers it the way it discovers an installed package. +The module appends a line to a marker file when imported, which is how these +tests prove a plugin was, or was never, loaded. + +The platform used is ``kubernetes``: the schema's ``binding.platform`` enum +admits it and no in-tree emitter serves it. (``testcloud`` would be rejected by +that closed enum at validation.) +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from fluid_build.cli import generate_iac +from fluid_build.cli._common import CLIError +from fluid_build.iac import cutover +from fluid_build.iac import registry as R + +pytestmark = [pytest.mark.unit, pytest.mark.provider] + +_PLUGIN_SRC = """ +import os +with open(os.environ["LW8_MARKER"], "a") as _f: + _f.write(__name__ + "\\n") + + +class KubePlugin: + name = "kubernetes" + required_providers = {"kubernetes": {"source": "hashicorp/kubernetes", "version": "~> 2.30"}} + credential_env_vars = ("KUBECONFIG",) + REQUIRES = None + + @classmethod + def get_plugin_info(cls): + class _Info: + requires_cli = cls.REQUIRES + + return _Info() + + def emit(self, contract, actions=()): + return {"kubernetes_namespace": {"ns": {"metadata": {"name": "dummy-ns"}}}} + + def emit_data(self, contract, actions=()): + return {} + + def credential_env(self, env): + return {} + + def discover_imports(self, contract, actions=()): + return [] + + def provider_block(self): + return {} + + +class NeedsNewerCli(KubePlugin): + REQUIRES = ">=999.0" + + +class SlowPlugin(KubePlugin): + def __init__(self): + import time + + with open(os.environ["LW8_MARKER"] + ".inst", "a") as _f: + _f.write("inst\\n") + time.sleep(0.3) +""" + + +def _make_dist(root: Path, group_lines: str, name: str = "fluid-testcloud") -> None: + (root / "lw8_plugin.py").write_text(_PLUGIN_SRC) + info = root / f"{name.replace('-', '_')}-1.0.dist-info" + info.mkdir() + (info / "METADATA").write_text(f"Metadata-Version: 2.1\nName: {name}\nVersion: 1.0\n") + (info / "entry_points.txt").write_text( + f"[fluid_build.iac_providers]\n{textwrap.dedent(group_lines)}\n" + ) + + +@pytest.fixture +def dist(tmp_path, monkeypatch): + """Install a dist exposing ``kubernetes`` -> ``KubePlugin``; yield its marker file.""" + root = tmp_path / "site" + root.mkdir() + marker = tmp_path / "loaded.txt" + _make_dist(root, "kubernetes = lw8_plugin:KubePlugin") + monkeypatch.syspath_prepend(str(root)) + monkeypatch.setenv("LW8_MARKER", str(marker)) + monkeypatch.delenv("FLUID_PLUGINS_ALLOWLIST", raising=False) + monkeypatch.delenv("FLUID_PLUGINS_BLOCKLIST", raising=False) + monkeypatch.delenv("FLUID_PLUGIN_STRICT_COMPAT", raising=False) + saved = dict(R.IAC_PLUGINS) + R.reset_iac_plugin_cache() + sys.modules.pop("lw8_plugin", None) + try: + yield marker + finally: + R.reset_iac_plugin_cache() + R.IAC_PLUGINS.clear() + R.IAC_PLUGINS.update(saved) + sys.modules.pop("lw8_plugin", None) + + +def _loaded(marker: Path) -> bool: + return marker.exists() and marker.read_text().strip() != "" + + +def _contract(tmp_path: Path, platform: str = "kubernetes") -> Path: + contract = { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "p", + "name": "P", + "metadata": {"layer": "Bronze", "owner": {"team": "t", "email": "t@x.co"}}, + "exposes": [ + { + "exposeId": "events", + "binding": {"platform": platform, "format": "table", "location": {"name": "e"}}, + "contract": {"schema": [{"name": "id", "type": "string"}]}, + } + ], + } + path = tmp_path / "contract.fluid.yaml" + path.write_text(json.dumps(contract)) # JSON is valid YAML + return path + + +def _generate(tmp_path: Path, contract: Path, provider: str = "auto") -> int: + args = argparse.Namespace( + contract=str(contract), + provider=provider, + out=str(tmp_path / "out"), + env=None, + shadow=False, + validate=False, + allow_empty=False, + ) + return generate_iac.run(args, logging.getLogger("test")) + + +def test_importing_the_iac_package_loads_no_plugin(dist): + code = "import fluid_build.iac, fluid_build.cli.generate_iac" + env = {**os.environ, "LW8_MARKER": str(dist), "PYTHONPATH": str(Path(sys.path[0]))} + subprocess.run([sys.executable, "-c", code], env=env, check=True) + assert not _loaded(dist) + + +def test_help_and_providers_load_no_plugin(dist): + root = next(p for p in sys.path if (Path(p) / "lw8_plugin.py").exists()) + env = {**os.environ, "LW8_MARKER": str(dist), "PYTHONPATH": root} + for argv in (["--help"], ["providers"], ["generate", "iac", "--help"]): + done = subprocess.run( + [sys.executable, "-m", "fluid_build.cli", *argv], + env=env, + capture_output=True, + check=False, + timeout=120, + ) + assert done.returncode == 0, f"`fluid {' '.join(argv)}` failed: {done.stderr[-400:]!r}" + assert not _loaded(dist), f"`fluid {' '.join(argv)}` imported the IaC plugin" + + +def test_contract_for_the_plugin_platform_emits_the_plugins_resources(dist, tmp_path): + contract = _contract(tmp_path) + assert _generate(tmp_path, contract) == 0 + doc = json.loads((tmp_path / "out" / "main.tf.json").read_text()) + assert doc["resource"]["kubernetes_namespace"]["ns"]["metadata"]["name"] == "dummy-ns" + assert doc["terraform"]["required_providers"]["kubernetes"]["source"] == "hashicorp/kubernetes" + assert _loaded(dist) + + +def test_provider_choices_list_the_plugin_platform_but_not_junk(dist, tmp_path): + # The generate-iac subparser's own choices see the plugin (names only); the + # GLOBAL --provider flag is validated elsewhere and accepts built-ins only. + assert "kubernetes" in generate_iac._PROVIDER_CHOICES + assert "nonesuch" not in generate_iac._PROVIDER_CHOICES + assert _generate(tmp_path, _contract(tmp_path), provider="kubernetes") == 0 + + +def test_the_real_cli_autodetects_a_plugin_platform(dist, tmp_path): + root = next(p for p in sys.path if (Path(p) / "lw8_plugin.py").exists()) + env = {**os.environ, "LW8_MARKER": str(dist), "PYTHONPATH": root} + out = tmp_path / "cli-out" + done = subprocess.run( + [sys.executable, "-m", "fluid_build.cli", "generate", "iac", + str(_contract(tmp_path)), "--out", str(out)], + env=env, capture_output=True, text=True, check=False, timeout=180, + ) # fmt: skip + assert done.returncode == 0, done.stderr[-600:] + doc = json.loads((out / "main.tf.json").read_text()) + assert doc["resource"]["kubernetes_namespace"]["ns"]["metadata"]["name"] == "dummy-ns" + assert _loaded(dist) + + +def test_the_apply_engine_routes_the_plugin_platform_through_opentofu(dist): + assert cutover.default_engine("kubernetes") == "opentofu" + assert cutover.resolve_engine(None, "kubernetes") == "opentofu" + assert "kubernetes" in R.IAC_ENTRYPOINT_PLUGINS + + +def test_nothing_loads_until_a_platform_needs_it(dist): + assert R.known_iac_platforms().count("kubernetes") == 1 + assert R.is_iac_entrypoint_platform("kubernetes") + assert cutover.default_engine("aws") == "opentofu" # in-tree: no plugin involved + assert cutover.default_engine("local") == "native" + assert not _loaded(dist) + assert R.get_iac_plugin("kubernetes") is not None + assert _loaded(dist) + + +def test_a_plugin_claiming_a_builtin_is_skipped_before_import_and_warned( + dist, tmp_path, monkeypatch, caplog +): + for claimed in ("aws", "gcp", "snowflake", "databricks", "confluent", "glue", "local"): + root = tmp_path / f"squat-{claimed}" + root.mkdir() + _make_dist(root, f"{claimed} = lw8_plugin:KubePlugin", name=f"squatter-{claimed}") + monkeypatch.syspath_prepend(str(root)) + R.reset_iac_plugin_cache() + original = {k: v for k, v in R.IAC_PLUGINS.items()} + with caplog.at_level(logging.WARNING, logger=R._log.name): + R.discover_iac_entrypoints() + assert R.get_iac_plugin("aws") is original["aws"] + assert cutover.default_engine("glue") == "native" + text = caplog.text + assert "iac_provider_name_reserved" in text + assert "squatter-aws" in text and "'aws'" in text + assert R.IAC_PLUGINS["aws"] is original["aws"] + assert "glue" not in R.IAC_PLUGINS and "local" not in R.IAC_PLUGINS + # Only the legitimate dist was imported; the squatters share lw8_plugin, so + # one import line would mean a squatter ran. The legitimate one imports once. + assert _loaded(dist) and dist.read_text().count("\n") == 1 + + +def test_a_squatter_alone_is_never_imported(tmp_path, monkeypatch): + root = tmp_path / "squat" + root.mkdir() + marker = tmp_path / "m.txt" + _make_dist(root, "aws = lw8_plugin:KubePlugin", name="squatter") + monkeypatch.syspath_prepend(str(root)) + monkeypatch.setenv("LW8_MARKER", str(marker)) + monkeypatch.delenv("FLUID_PLUGINS_ALLOWLIST", raising=False) + monkeypatch.delenv("FLUID_PLUGINS_BLOCKLIST", raising=False) + sys.modules.pop("lw8_plugin", None) + R.reset_iac_plugin_cache() + R.discover_iac_entrypoints() + R.get_iac_plugin("aws") + assert not _loaded(marker) + assert "lw8_plugin" not in sys.modules + + +def test_a_blocked_plugin_is_never_imported_and_the_user_is_told(dist, tmp_path, monkeypatch): + monkeypatch.setenv("FLUID_PLUGINS_BLOCKLIST", "kubernetes") + R.reset_iac_plugin_cache() + assert R.get_iac_plugin("kubernetes") is None + assert cutover.default_engine("kubernetes") == "native" + assert "kubernetes" not in R.known_iac_platforms() + assert "FLUID_PLUGINS_ALLOWLIST/BLOCKLIST" in R.iac_plugin_unavailable_reason("kubernetes") + assert not _loaded(dist) + with pytest.raises(CLIError) as exc: + _generate(tmp_path, _contract(tmp_path), provider="kubernetes") + assert "blocked by FLUID_PLUGINS_ALLOWLIST/BLOCKLIST" in exc.value.context["error"] + + +def test_an_allowlist_that_omits_the_plugin_keeps_it_unloaded(dist, monkeypatch): + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", "aws,gcp") + R.reset_iac_plugin_cache() + assert R.get_iac_plugin("kubernetes") is None + assert not _loaded(dist) + + +def _use_incompatible_plugin(tmp_path, monkeypatch): + root = tmp_path / "newer" + root.mkdir() + _make_dist(root, "kubernetes = lw8_plugin:NeedsNewerCli", name="needs-newer-cli") + monkeypatch.syspath_prepend(str(root)) + monkeypatch.setenv("LW8_MARKER", str(tmp_path / "newer-marker.txt")) + monkeypatch.delenv("FLUID_PLUGINS_ALLOWLIST", raising=False) + monkeypatch.delenv("FLUID_PLUGINS_BLOCKLIST", raising=False) + sys.modules.pop("lw8_plugin", None) + R.reset_iac_plugin_cache() + + +def test_incompatible_requires_cli_warns_by_default(tmp_path, monkeypatch, caplog): + # The registry gate is the provider registry's own: advisory unless strict. + monkeypatch.delenv("FLUID_PLUGIN_STRICT_COMPAT", raising=False) + _use_incompatible_plugin(tmp_path, monkeypatch) + try: + with caplog.at_level(logging.WARNING): + assert R.get_iac_plugin("kubernetes") is not None + assert "provider_version_warning" in caplog.text + finally: + R.reset_iac_plugin_cache() + sys.modules.pop("lw8_plugin", None) + + +def test_incompatible_requires_cli_is_refused_under_strict_compat(tmp_path, monkeypatch, caplog): + monkeypatch.setenv("FLUID_PLUGIN_STRICT_COMPAT", "1") + _use_incompatible_plugin(tmp_path, monkeypatch) + try: + with caplog.at_level(logging.WARNING): + assert R.get_iac_plugin("kubernetes") is None + assert "provider_version_warning" in caplog.text + assert "iac_plugin_compat_rejected" in caplog.text + assert cutover.default_engine("kubernetes") == "native" + assert "refused" in R.iac_plugin_unavailable_reason("kubernetes") + # A refusal is remembered: a second ask does not re-import or re-warn. + caplog.clear() + assert R.get_iac_plugin("kubernetes") is None + assert caplog.text == "" + finally: + R.reset_iac_plugin_cache() + sys.modules.pop("lw8_plugin", None) + + +# ── operator opt-in: a distribution-qualified allowlist entry grants an override ── + + +def _squat(tmp_path, monkeypatch, dist_name="acme-fluid-aws", platform="aws"): + root = tmp_path / f"squat-{platform}" + root.mkdir() + _make_dist(root, f"{platform} = lw8_plugin:KubePlugin", name=dist_name) + monkeypatch.syspath_prepend(str(root)) + R.reset_iac_plugin_cache() + + +def test_a_qualified_grant_lets_the_plugin_replace_a_builtin(dist, tmp_path, monkeypatch, caplog): + _squat(tmp_path, monkeypatch) + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", "acme-fluid-aws:aws") + builtin = R.IAC_PLUGINS["aws"] + with caplog.at_level(logging.INFO, logger=R._log.name): + plugin = R.get_iac_plugin("aws") + assert plugin is not builtin and type(plugin).__name__ == "KubePlugin" + assert cutover.default_engine("aws") == "opentofu" + assert "iac_provider_override_granted" in caplog.text + assert "acme-fluid-aws" in caplog.text and "'aws'" in caplog.text + assert "iac_provider_name_reserved" not in caplog.text + + +def test_without_a_grant_the_claim_is_skipped_and_the_warning_says_how_to_opt_in( + dist, tmp_path, monkeypatch, caplog +): + _squat(tmp_path, monkeypatch) + builtin = R.IAC_PLUGINS["aws"] + with caplog.at_level(logging.WARNING, logger=R._log.name): + R.discover_iac_entrypoints() + assert R.IAC_PLUGINS["aws"] is builtin + assert "add 'acme-fluid-aws:aws' to FLUID_PLUGINS_ALLOWLIST" in caplog.text + + +def test_a_bare_name_in_the_allowlist_does_not_grant(dist, tmp_path, monkeypatch, caplog): + _squat(tmp_path, monkeypatch) + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", "aws") + builtin = R.IAC_PLUGINS["aws"] + with caplog.at_level(logging.WARNING, logger=R._log.name): + R.discover_iac_entrypoints() + assert R.get_iac_plugin("aws") is builtin + assert "iac_provider_name_reserved" in caplog.text + + +@pytest.mark.parametrize("spelling", ["ACME_Fluid.AWS", "acme.fluid_aws", "Acme-Fluid--AWS"]) +def test_distribution_name_variants_match_the_grant(dist, tmp_path, monkeypatch, spelling): + _squat(tmp_path, monkeypatch, dist_name="Acme_Fluid.AWS") + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", f"{spelling}:AWS") + assert type(R.get_iac_plugin("aws")).__name__ == "KubePlugin" + + +def test_a_grant_for_another_distribution_does_not_apply(dist, tmp_path, monkeypatch): + _squat(tmp_path, monkeypatch) + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", "other-dist:aws") + builtin = R.IAC_PLUGINS["aws"] + assert R.get_iac_plugin("aws") is builtin + + +def test_a_qualified_only_allowlist_does_not_restrict_unrelated_plugins(dist, monkeypatch): + from fluid_build import plugin_manager as PM + + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", "acme-fluid-aws:aws") + assert PM.is_allowed("kubernetes") and PM.is_allowed("anything") + assert R.get_iac_plugin("kubernetes") is not None + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", "acme-fluid-aws:aws,gcp") + assert PM.is_allowed("gcp") and not PM.is_allowed("kubernetes") + + +@pytest.mark.parametrize("retired", ["redshift", "datamesh-manager", "datamesh_manager"]) +def test_a_grant_never_lifts_a_retired_name(tmp_path, monkeypatch, caplog, retired): + root = tmp_path / "squat" + root.mkdir() + marker = tmp_path / "m.txt" + _make_dist(root, f"{retired} = lw8_plugin:KubePlugin", name="acme-fluid-aws") + monkeypatch.syspath_prepend(str(root)) + monkeypatch.setenv("LW8_MARKER", str(marker)) + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", f"acme-fluid-aws:{retired}") + monkeypatch.delenv("FLUID_PLUGINS_BLOCKLIST", raising=False) + sys.modules.pop("lw8_plugin", None) + R.reset_iac_plugin_cache() + with caplog.at_level(logging.INFO, logger=R._log.name): + R.discover_iac_entrypoints() + R.get_iac_plugin(retired) + assert not _loaded(marker) + assert "lw8_plugin" not in sys.modules + assert "the retired platform" in caplog.text + assert "iac_provider_override_granted" not in caplog.text + assert "FLUID_PLUGINS_ALLOWLIST" not in caplog.text + + +# ── hardening: grant identity, policy spelling, escaping, duplicates, locking ── + + +def _extra_dist(tmp_path, monkeypatch, tag, lines, name): + root = tmp_path / f"extra-{tag}" + root.mkdir() + _make_dist(root, lines, name=name) + monkeypatch.syspath_prepend(str(root)) + R.reset_iac_plugin_cache() + + +def test_a_self_declared_name_with_a_suffix_cannot_satisfy_a_grant( + dist, tmp_path, monkeypatch, caplog +): + # METADATA ``Name: acme-fluid-aws spoof`` used to pass for ``acme-fluid-aws`` + # (the first word of the " " display string). + _squat(tmp_path, monkeypatch, dist_name="acme-fluid-aws spoof") + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", "acme-fluid-aws:aws") + builtin = R.IAC_PLUGINS["aws"] + with caplog.at_level(logging.WARNING, logger=R._log.name): + assert R.get_iac_plugin("aws") is builtin + assert not _loaded(dist) + assert "iac_provider_name_reserved" in caplog.text + assert "to allow this override" not in caplog.text # no grant can work for that name + + +def test_a_mixed_case_entry_point_is_blocked_by_a_lowercase_blocklist(tmp_path, monkeypatch): + root = tmp_path / "mixed" + root.mkdir() + marker = tmp_path / "mixed-marker.txt" + _make_dist(root, "Kubernetes = lw8_plugin:KubePlugin", name="mixed-case") + monkeypatch.syspath_prepend(str(root)) + monkeypatch.setenv("LW8_MARKER", str(marker)) + monkeypatch.setenv("FLUID_PLUGINS_BLOCKLIST", "kubernetes") + monkeypatch.delenv("FLUID_PLUGINS_ALLOWLIST", raising=False) + sys.modules.pop("lw8_plugin", None) + R.reset_iac_plugin_cache() + try: + assert R.get_iac_plugin("Kubernetes") is None + assert not _loaded(marker) + assert not R.is_iac_entrypoint_platform("kubernetes") + assert "kubernetes" not in R.known_iac_platforms() + finally: + R.reset_iac_plugin_cache() + sys.modules.pop("lw8_plugin", None) + + +@pytest.fixture +def onprem(tmp_path, monkeypatch): + root = tmp_path / "onprem" + root.mkdir() + marker = tmp_path / "onprem-loaded.txt" + _make_dist(root, "on-prem = lw8_plugin:KubePlugin", name="onprem-pkg") + monkeypatch.syspath_prepend(str(root)) + monkeypatch.setenv("LW8_MARKER", str(marker)) + for var in ("FLUID_PLUGINS_ALLOWLIST", "FLUID_PLUGINS_BLOCKLIST", "FLUID_PLUGIN_STRICT_COMPAT"): + monkeypatch.delenv(var, raising=False) + sys.modules.pop("lw8_plugin", None) + R.reset_iac_plugin_cache() + yield marker + R.reset_iac_plugin_cache() + sys.modules.pop("lw8_plugin", None) + + +def test_hyphenated_entry_point_is_blocked_by_the_underscore_spelling(onprem, monkeypatch): + monkeypatch.setenv("FLUID_PLUGINS_BLOCKLIST", "on_prem") + assert R.get_iac_plugin("on-prem") is None + assert not _loaded(onprem) + assert "blocked" in R.iac_plugin_unavailable_reason("on_prem") + + +def test_hyphenated_entry_point_loads_under_a_hyphenated_allowlist(onprem, monkeypatch): + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", "on-prem") + assert R.get_iac_plugin("on_prem") is not None + assert _loaded(onprem) + + +def test_lookup_spelling_does_not_change_the_answer(onprem): + first = R.get_iac_plugin("On-Prem") + second = R.get_iac_plugin("On-Prem") + assert first is not None and first is second + assert R.get_iac_plugin("on_prem") is first + assert cutover.default_engine("On-Prem") == cutover.default_engine("on_prem") == "opentofu" + assert R.is_iac_entrypoint_platform("ON-PREM") + + +@pytest.mark.parametrize("claimed", ["AWS", "S3", "BigQuery"]) +def test_a_mixed_case_squatter_is_still_reserved(tmp_path, monkeypatch, claimed): + root = tmp_path / "squat" + root.mkdir() + marker = tmp_path / "m.txt" + _make_dist(root, f"{claimed} = lw8_plugin:KubePlugin", name="squatter") + monkeypatch.syspath_prepend(str(root)) + monkeypatch.setenv("LW8_MARKER", str(marker)) + monkeypatch.delenv("FLUID_PLUGINS_ALLOWLIST", raising=False) + monkeypatch.delenv("FLUID_PLUGINS_BLOCKLIST", raising=False) + sys.modules.pop("lw8_plugin", None) + R.reset_iac_plugin_cache() + R.discover_iac_entrypoints() + R.get_iac_plugin(claimed) + assert not _loaded(marker) + R.reset_iac_plugin_cache() + + +def test_control_characters_in_a_distribution_name_never_reach_the_log( + dist, tmp_path, monkeypatch, caplog +): + _squat(tmp_path, monkeypatch, dist_name="inj\x1b[31mRED\x1b[0m", platform="gcp") + with caplog.at_level(logging.INFO, logger=R._log.name): + R.discover_iac_entrypoints() + assert "iac_provider_name_reserved" in caplog.text + assert "\x1b" not in caplog.text + # and the user-facing reason for a refused non-reserved plugin + monkeypatch.setenv("FLUID_PLUGINS_BLOCKLIST", "inj_platform") + root = tmp_path / "inj2" + root.mkdir() + _make_dist(root, "inj-platform = lw8_plugin:KubePlugin", name="x\x1b[2Jy") + monkeypatch.syspath_prepend(str(root)) + R.reset_iac_plugin_cache() + reason = R.iac_plugin_unavailable_reason("inj-platform") + assert reason and "\x1b" not in reason + + +def test_a_bare_allowlist_that_omits_the_name_cancels_the_grant_and_says_so( + dist, tmp_path, monkeypatch, caplog +): + _squat(tmp_path, monkeypatch) + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", "kubernetes,acme-fluid-aws:aws") + with caplog.at_level(logging.WARNING, logger=R._log.name): + R.get_iac_plugin("aws") + cancelled = [r for r in caplog.records if "iac_provider_grant_cancelled" in r.getMessage()] + assert len(cancelled) == 1 + assert "do not include 'aws'" in cancelled[0].getMessage() + assert "cancelled" in R.iac_plugin_unavailable_reason("aws") + assert not _loaded(dist) + + +def test_two_distributions_shipping_one_plugin_name_are_both_refused( + dist, tmp_path, monkeypatch, caplog +): + _extra_dist(tmp_path, monkeypatch, "b", "kubernetes = lw8_plugin:KubePlugin", "other-kube") + with caplog.at_level(logging.WARNING, logger=R._log.name): + assert R.get_iac_plugin("kubernetes") is None + assert not _loaded(dist) + warned = [r.getMessage() for r in caplog.records if "name_ambiguous" in r.getMessage()] + assert len(warned) == 1 + assert "fluid-testcloud" in warned[0] and "other-kube" in warned[0] + assert "several distributions" in R.iac_plugin_unavailable_reason("kubernetes") + assert cutover.default_engine("kubernetes") == "native" + + +def test_with_two_distributions_on_a_reserved_name_the_granted_one_loads( + dist, tmp_path, monkeypatch, caplog +): + _extra_dist(tmp_path, monkeypatch, "a", "aws = lw8_plugin:KubePlugin", "ungranted-aws") + _extra_dist(tmp_path, monkeypatch, "b", "aws = lw8_plugin:KubePlugin", "acme-fluid-aws") + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", "acme-fluid-aws:aws") + builtin = R.IAC_PLUGINS["aws"] + with caplog.at_level(logging.WARNING, logger=R._log.name): + plugin = R.get_iac_plugin("aws") + assert plugin is not builtin and type(plugin).__name__ == "KubePlugin" + assert "ungranted-aws" in caplog.text and "iac_provider_name_reserved" in caplog.text + assert "acme-fluid-aws" not in caplog.text.replace("'acme-fluid-aws:aws'", "") + + +def test_reset_restores_the_builtin_a_granted_override_replaced(dist, tmp_path, monkeypatch): + _squat(tmp_path, monkeypatch) + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", "acme-fluid-aws:aws") + builtin = R.IAC_PLUGINS["aws"] + assert R.get_iac_plugin("aws") is not builtin + R.reset_iac_plugin_cache() + assert R.IAC_PLUGINS["aws"] is builtin + + +def test_concurrent_first_requests_instantiate_the_plugin_once(tmp_path, monkeypatch): + import threading + + root = tmp_path / "slow" + root.mkdir() + marker = tmp_path / "slow-marker.txt" + _make_dist(root, "slowcloud = lw8_plugin:SlowPlugin", name="slow-pkg") + monkeypatch.syspath_prepend(str(root)) + monkeypatch.setenv("LW8_MARKER", str(marker)) + for var in ("FLUID_PLUGINS_ALLOWLIST", "FLUID_PLUGINS_BLOCKLIST"): + monkeypatch.delenv(var, raising=False) + sys.modules.pop("lw8_plugin", None) + R.reset_iac_plugin_cache() + results = [] + gate = threading.Barrier(4) + + def ask(): + gate.wait() + results.append(R.get_iac_plugin("slowcloud")) + + threads = [threading.Thread(target=ask) for _ in range(4)] + for th in threads: + th.start() + for th in threads: + th.join() + try: + assert len(results) == 4 and all(r is results[0] and r is not None for r in results) + assert Path(str(marker) + ".inst").read_text().count("inst") == 1 + finally: + R.reset_iac_plugin_cache() + sys.modules.pop("lw8_plugin", None) + + +def test_the_state_guard_sees_a_plugin_cloud_that_was_never_loaded(dist): + from fluid_build.iac import state_migration as sm + + res = [{"provider": 'provider["registry.opentofu.org/hashicorp/kubernetes"]'}] + assert not _loaded(dist) # lazy: nothing loaded yet + verdict, _why = sm.classify(res, "aws") + assert verdict == "other" + assert sm.other_clouds(res, "aws") == frozenset({"kubernetes"}) diff --git a/tests/iac/test_iac_generate_offline.py b/tests/iac/test_iac_generate_offline.py index d432c5a95..3dc3a2f34 100644 --- a/tests/iac/test_iac_generate_offline.py +++ b/tests/iac/test_iac_generate_offline.py @@ -43,6 +43,7 @@ from fluid_build.cli import generate_iac, main from fluid_build.cli._common import CLIError +from fluid_build.providers.aws.util.config import AwsAccountUnresolvedError pytestmark = [pytest.mark.unit, pytest.mark.provider] @@ -202,7 +203,10 @@ class TestApplyStillResolvesTheAccount: (``cli/_apply_opentofu_engine.py``); only ``generate iac`` passes ``offline``.""" def test_the_apply_call_asks_sts_for_a_missing_account(self, clients): - generate_iac.native_actions(_scheduled(), logging.getLogger("t")) + # The ``clients`` guard makes the lookup fail, so no account resolves: + # refused by name, before any resource name is derived from ``None``. + with pytest.raises(AwsAccountUnresolvedError, match="AWS_ACCOUNT_ID"): + generate_iac.native_actions(_scheduled(), logging.getLogger("t")) assert clients == ["sts"] def test_the_offline_call_plans_without_asking(self, clients): diff --git a/tests/iac/test_iac_moto_e2e.py b/tests/iac/test_iac_moto_e2e.py index 6df41b33b..a8c683d2f 100644 --- a/tests/iac/test_iac_moto_e2e.py +++ b/tests/iac/test_iac_moto_e2e.py @@ -131,7 +131,7 @@ def moto_endpoint(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: ``/moto-api/reset`` admin endpoint clears them, giving each test the clean account it expects. """ - import requests + import httpx from moto.server import ThreadedMotoServer server = ThreadedMotoServer(port=0, verbose=False) @@ -141,7 +141,7 @@ def moto_endpoint(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: endpoint = f"http://127.0.0.1:{port}" monkeypatch.setenv("AWS_ENDPOINT_URL", endpoint) with contextlib.suppress(Exception): - requests.post(f"{endpoint}/moto-api/reset", timeout=5) + httpx.post(f"{endpoint}/moto-api/reset", timeout=5) yield endpoint finally: server.stop() diff --git a/tests/iac/test_iac_runner.py b/tests/iac/test_iac_runner.py index 6d952853c..0e3b45ba0 100644 --- a/tests/iac/test_iac_runner.py +++ b/tests/iac/test_iac_runner.py @@ -16,6 +16,9 @@ from __future__ import annotations +import signal +import subprocess + import pytest from fluid_build.iac import build_module, get_iac_plugin, runner @@ -131,24 +134,74 @@ def test_resolve_timeout_ignores_non_positive(self, monkeypatch): monkeypatch.setenv("FLUID_TOFU_TIMEOUT_SECONDS", "-100") assert runner._resolve_timeout() == runner._DEFAULT_TOFU_TIMEOUT_SECONDS - def test_run_returns_124_on_timeout(self, monkeypatch, tmp_path): - """When ``subprocess.run`` raises ``TimeoutExpired``, ``_run`` - synthesises a ``TofuResult`` with returncode 124 (mirrors - coreutils ``timeout``) and a clear stderr — never re-raises.""" - import subprocess as _subprocess - - monkeypatch.setattr(runner, "_require_tofu", lambda: "/usr/local/bin/tofu") - - def _raise_timeout(*_args, **kwargs): - raise _subprocess.TimeoutExpired(cmd="tofu apply", timeout=kwargs["timeout"]) - - monkeypatch.setattr(runner.subprocess, "run", _raise_timeout) - result = runner._run(["apply"], workdir=str(tmp_path), env=None, command="apply") + def _fake_tofu(self, monkeypatch, tmp_path, body): + import os + import stat + import sys + + script = tmp_path / "fake-tofu" + script.write_text(f"#!{sys.executable}\n{body}") + script.chmod(script.stat().st_mode | stat.S_IXUSR) + monkeypatch.setattr(runner, "_require_tofu", lambda: str(script)) + monkeypatch.setenv("FLUID_TOFU_TIMEOUT_SECONDS", "5") + return os.fspath(tmp_path) + + class _Child: + """A ``Popen`` stand-in: times out once, then answers SIGINT (or not).""" + + def __init__(self, answers_interrupt): + self.args = ["tofu", "apply"] + self.returncode = None + self.signals = [] + self.killed = False + self._answers = answers_interrupt + self._calls = 0 + + def communicate(self, timeout=None): + self._calls += 1 + if self._calls == 1 or (self._calls == 2 and not self._answers): + raise subprocess.TimeoutExpired(self.args, timeout) + self.returncode = 130 if self._answers else -9 + return ("released lock\n" if self._answers else "", "") + + def send_signal(self, sig): + self.signals.append(sig) + + def kill(self): + self.killed = True + + def _run_with(self, monkeypatch, child): + monkeypatch.setattr(runner, "_require_tofu", lambda: "tofu") + monkeypatch.setattr(runner.subprocess, "Popen", lambda *a, **k: child) + return runner._run(["apply"], workdir=".", env=None, command="apply") + + def test_run_returns_124_on_timeout_after_a_graceful_interrupt(self, monkeypatch): + """A timeout sends SIGINT, so tofu releases its lock and exits; the result + keeps the 124 shape, keeps tofu's output and never re-raises.""" + child = self._Child(answers_interrupt=True) + result = self._run_with(monkeypatch, child) + assert child.signals == [signal.SIGINT] + assert child.killed is False assert result.ok is False assert result.returncode == 124 + assert "released lock" in result.stdout assert "exceeded" in result.stderr assert "FLUID_TOFU_TIMEOUT_SECONDS" in result.stderr + def test_run_kills_only_after_the_interrupt_is_ignored(self, monkeypatch): + child = self._Child(answers_interrupt=False) + result = self._run_with(monkeypatch, child) + assert child.signals == [signal.SIGINT] + assert child.killed is True + assert result.returncode == 124 + + def test_run_kills_a_tofu_that_ignores_the_interrupt(self, monkeypatch, tmp_path): + body = "import signal, time\nsignal.signal(signal.SIGINT, signal.SIG_IGN)\ntime.sleep(60)\n" + wd = self._fake_tofu(monkeypatch, tmp_path, body) + monkeypatch.setattr(runner, "_TIMEOUT_GRACE_SECONDS", 1) + result = runner._run(["apply"], workdir=wd, env=None, command="apply") + assert result.returncode == 124 + @pytest.mark.integration @pytest.mark.skipif(runner.tofu_path() is None, reason="tofu binary not installed") @@ -193,3 +246,154 @@ def test_state_list_empty_when_no_state(self, monkeypatch): runner, "_run", lambda *a, **k: runner.TofuResult("state-list", 1, "", "no state") ) assert runner.tofu_state_list("/wd") == [] + + +class TestS3Lockfile: + """S3 state locks via ``-backend-config=use_lockfile=true`` at init (tofu >= 1.10).""" + + FLAG = "-backend-config=use_lockfile=true" + + def _init_args(self, monkeypatch, tmp_path, backend, version): + import json as _json + + doc = {"terraform": {"backend": backend} if backend else {}} + (tmp_path / "main.tf.json").write_text(_json.dumps(doc)) + seen = {} + monkeypatch.setattr(runner, "tofu_version", lambda: version) + monkeypatch.setattr(runner, "_run", lambda args, **kw: seen.setdefault("args", args)) + runner.tofu_init(str(tmp_path)) + return seen["args"] + + S3 = {"s3": {"bucket": "b", "key": "k"}} + + def test_added_on_new_tofu(self, monkeypatch, tmp_path): + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE", raising=False) + assert self.FLAG in self._init_args(monkeypatch, tmp_path, self.S3, (1, 10, 0)) + + def test_absent_and_warns_on_old_tofu(self, monkeypatch, tmp_path, caplog): + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE", raising=False) + with caplog.at_level("WARNING"): + args = self._init_args(monkeypatch, tmp_path, self.S3, (1, 9, 9)) + assert self.FLAG not in args + assert "s3_state_unlocked" in caplog.text and "1.10" in caplog.text + + def test_opt_out(self, monkeypatch, tmp_path): + monkeypatch.setenv("FLUID_TOFU_S3_LOCKFILE", "0") + assert self.FLAG not in self._init_args(monkeypatch, tmp_path, self.S3, (1, 12, 1)) + + def test_gcs_and_local_untouched(self, monkeypatch, tmp_path): + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE", raising=False) + assert self.FLAG not in self._init_args( + monkeypatch, tmp_path, {"gcs": {"bucket": "b"}}, (1, 12, 1) + ) + assert self.FLAG not in self._init_args(monkeypatch, tmp_path, None, (1, 12, 1)) + + def test_emitted_module_has_no_lockfile(self): + from fluid_build.iac.backend import parse_backend + + assert parse_backend("s3://b/k") == {"s3": {"bucket": "b", "key": "k"}} + + def test_unreadable_version_skips_flag_and_warns(self, monkeypatch, tmp_path, caplog): + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE", raising=False) + with caplog.at_level("WARNING"): + args = self._init_args(monkeypatch, tmp_path, self.S3, None) + assert self.FLAG not in args + assert "s3_state_unlocked" in caplog.text and "could not be read" in caplog.text + + +class TestLockfileReconfigure: + """``-reconfigure`` joins the lockfile flag only for a kept workdir on this same state.""" + + def _args(self, monkeypatch, tmp_path, recorded, module=None): + import json as _json + + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE", raising=False) + s3 = module or {"bucket": "b", "key": "k", "region": "eu-west-1"} + (tmp_path / "main.tf.json").write_text(_json.dumps({"terraform": {"backend": {"s3": s3}}})) + if recorded is not None: + (tmp_path / ".terraform").mkdir() + (tmp_path / ".terraform" / "terraform.tfstate").write_text( + _json.dumps({"backend": {"type": "s3", "config": recorded}}) + ) + seen = {} + monkeypatch.setattr(runner, "tofu_version", lambda: (1, 12, 1)) + monkeypatch.setattr(runner, "_run", lambda args, **kw: seen.setdefault("args", args)) + runner.tofu_init(str(tmp_path)) + return seen["args"] + + def test_same_backend_without_lockfile_reconfigures(self, monkeypatch, tmp_path): + rec = {"bucket": "b", "key": "k", "region": "eu-west-1", "use_lockfile": None} + args = self._args(monkeypatch, tmp_path, rec) + assert "-reconfigure" in args and "-backend-config=use_lockfile=true" in args + + def test_different_key_does_not(self, monkeypatch, tmp_path): + rec = {"bucket": "b", "key": "other", "region": "eu-west-1", "use_lockfile": None} + assert "-reconfigure" not in self._args(monkeypatch, tmp_path, rec) + + def test_different_region_does_not(self, monkeypatch, tmp_path): + rec = {"bucket": "b", "key": "k", "region": "us-east-1", "use_lockfile": None} + assert "-reconfigure" not in self._args(monkeypatch, tmp_path, rec) + + def test_already_locking_does_not(self, monkeypatch, tmp_path): + rec = {"bucket": "b", "key": "k", "region": "eu-west-1", "use_lockfile": True} + assert "-reconfigure" not in self._args(monkeypatch, tmp_path, rec) + + def test_nothing_recorded_does_not(self, monkeypatch, tmp_path): + assert "-reconfigure" not in self._args(monkeypatch, tmp_path, None) + + +class TestLockfileOptOutOnKeptWorkdir: + """A workdir that saved use_lockfile=true needs -reconfigure for the flag to go away.""" + + SAVED = {"bucket": "b", "key": "k", "region": "eu-west-1", "use_lockfile": True} + + def _args( + self, + monkeypatch, + tmp_path, + *, + version=(1, 12, 1), + env=None, + force_copy=False, + recorded=None, + ): + import json as _json + + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE", raising=False) + if env is not None: + monkeypatch.setenv("FLUID_TOFU_S3_LOCKFILE", env) + s3 = {"bucket": "b", "key": "k"} + (tmp_path / "main.tf.json").write_text(_json.dumps({"terraform": {"backend": {"s3": s3}}})) + (tmp_path / ".terraform").mkdir(exist_ok=True) + (tmp_path / ".terraform" / "terraform.tfstate").write_text( + _json.dumps({"backend": {"type": "s3", "config": recorded or self.SAVED}}) + ) + seen = {} + monkeypatch.setattr(runner, "tofu_version", lambda: version) + monkeypatch.setattr(runner, "_run", lambda args, **kw: seen.setdefault("args", args)) + runner.tofu_init(str(tmp_path), force_copy=force_copy) + return seen["args"] + + def test_opt_out_reconfigures(self, monkeypatch, tmp_path): + args = self._args(monkeypatch, tmp_path, env="0") + assert "-reconfigure" in args and "-backend-config=use_lockfile=true" not in args + + def test_old_tofu_reconfigures(self, monkeypatch, tmp_path): + assert "-reconfigure" in self._args(monkeypatch, tmp_path, version=(1, 9, 0)) + + def test_unreadable_version_reconfigures(self, monkeypatch, tmp_path): + assert "-reconfigure" in self._args(monkeypatch, tmp_path, version=None) + + def test_different_key_is_left_to_tofu(self, monkeypatch, tmp_path): + rec = dict(self.SAVED, key="other") + assert "-reconfigure" not in self._args(monkeypatch, tmp_path, env="0", recorded=rec) + + def test_lockfile_not_saved_adds_nothing(self, monkeypatch, tmp_path): + rec = dict(self.SAVED, use_lockfile=None) + assert "-reconfigure" not in self._args(monkeypatch, tmp_path, env="0", recorded=rec) + + def test_force_copy_never_gets_the_lockfile_reconfigure(self, monkeypatch, tmp_path): + assert "-reconfigure" not in self._args(monkeypatch, tmp_path, env="0", force_copy=True) + rec = dict(self.SAVED, use_lockfile=None) + args = self._args(monkeypatch, tmp_path, force_copy=True, recorded=rec) + assert "-reconfigure" not in args and "-force-copy" in args diff --git a/tests/iac/test_iac_state_drift.py b/tests/iac/test_iac_state_drift.py index e8c1f686f..04203c46e 100644 --- a/tests/iac/test_iac_state_drift.py +++ b/tests/iac/test_iac_state_drift.py @@ -26,7 +26,6 @@ from __future__ import annotations import json -import subprocess from typing import Any, Dict, List, Optional import pytest @@ -340,18 +339,26 @@ def test_format_path(): # --------------------------------------------------------------------------- -def _fake_run(calls: List[List[str]], *, returncode: int = 0, stdout: str = ""): - def _run(argv, **kwargs): - calls.append(list(argv)) - return subprocess.CompletedProcess(argv, returncode, stdout=stdout, stderr="") +def _fake_popen(calls: List[List[str]], *, returncode: int = 0, stdout: str = ""): + """A ``subprocess.Popen`` stand-in: ``runner._run`` starts tofu with Popen so a + timeout can interrupt it before killing it.""" - return _run + class _Child: + def __init__(self, argv, **kwargs): + calls.append(list(argv)) + self.args = argv + self.returncode = returncode + + def communicate(self, timeout=None): + return stdout, "" + + return _Child def test_tofu_plan_detailed_asks_for_the_detailed_exit_code(monkeypatch, tmp_path): calls: List[List[str]] = [] monkeypatch.setattr(runner, "tofu_path", lambda: "/bin/tofu") - monkeypatch.setattr(runner.subprocess, "run", _fake_run(calls, returncode=2)) + monkeypatch.setattr(runner.subprocess, "Popen", _fake_popen(calls, returncode=2)) result = runner.tofu_plan_detailed(str(tmp_path), out_file="fluid-drift.tfplan") assert calls[0][1:] == [ "plan", @@ -377,7 +384,7 @@ def test_tofu_show_plan(monkeypatch, tmp_path, returncode, stdout, expected): calls: List[List[str]] = [] monkeypatch.setattr(runner, "tofu_path", lambda: "/bin/tofu") monkeypatch.setattr( - runner.subprocess, "run", _fake_run(calls, returncode=returncode, stdout=stdout) + runner.subprocess, "Popen", _fake_popen(calls, returncode=returncode, stdout=stdout) ) assert runner.tofu_show_plan(str(tmp_path), plan_file="p") == expected assert calls[0][1:] == ["show", "-json", "p"] diff --git a/tests/iac/test_iac_state_drift_moto.py b/tests/iac/test_iac_state_drift_moto.py index f7628a67b..61d8fc801 100644 --- a/tests/iac/test_iac_state_drift_moto.py +++ b/tests/iac/test_iac_state_drift_moto.py @@ -81,7 +81,7 @@ def plugin_cache(tmp_path_factory: pytest.TempPathFactory) -> str: @pytest.fixture def moto(monkeypatch, tmp_path: Path, plugin_cache: str) -> Iterator[str]: """A fresh moto server that both ``tofu`` and boto3 are pointed at.""" - import requests + import httpx from moto.server import ThreadedMotoServer server = ThreadedMotoServer(port=0, verbose=False) @@ -90,7 +90,7 @@ def moto(monkeypatch, tmp_path: Path, plugin_cache: str) -> Iterator[str]: _, port = server.get_host_and_port() endpoint = f"http://127.0.0.1:{port}" with contextlib.suppress(Exception): - requests.post(f"{endpoint}/moto-api/reset", timeout=5) + httpx.post(f"{endpoint}/moto-api/reset", timeout=5) for var in ("AWS_PROFILE", "AWS_SESSION_TOKEN", "FLUID_STATE_BACKEND", "FLUID_PROVIDER"): monkeypatch.delenv(var, raising=False) monkeypatch.setenv("AWS_ENDPOINT_URL", endpoint) diff --git a/tests/iac/test_iac_state_key_migration_moto.py b/tests/iac/test_iac_state_key_migration_moto.py index f342b8b4e..3e10eee32 100644 --- a/tests/iac/test_iac_state_key_migration_moto.py +++ b/tests/iac/test_iac_state_key_migration_moto.py @@ -95,7 +95,7 @@ def plugin_cache(tmp_path_factory: pytest.TempPathFactory) -> str: @pytest.fixture def moto(monkeypatch, tmp_path: Path, plugin_cache: str) -> Iterator[str]: """A fresh moto server holding the state bucket; tofu and boto3 aim at it.""" - import requests + import httpx from moto.server import ThreadedMotoServer server = ThreadedMotoServer(port=0, verbose=False) @@ -104,7 +104,7 @@ def moto(monkeypatch, tmp_path: Path, plugin_cache: str) -> Iterator[str]: _, port = server.get_host_and_port() endpoint = f"http://127.0.0.1:{port}" with contextlib.suppress(Exception): - requests.post(f"{endpoint}/moto-api/reset", timeout=5) + httpx.post(f"{endpoint}/moto-api/reset", timeout=5) for var in ("AWS_PROFILE", "AWS_SESSION_TOKEN", "FLUID_STATE_BACKEND", "FLUID_PROVIDER"): monkeypatch.delenv(var, raising=False) monkeypatch.setenv("AWS_ENDPOINT_URL", endpoint) @@ -418,3 +418,37 @@ def test_a_read_only_caller_reads_the_old_key_until_the_apply_moves_it(moto, tmp assert outcome.outcome == mig.PENDING assert outcome.read_from == _block(_LEGACY_KEY) assert _AWS_KEY not in _keys(moto) + + +def test_a_workdir_kept_from_before_the_lockfile_upgrades_in_place(moto, tmp_path, monkeypatch): + """Old release's init (no use_lockfile) then the new one on the same kept workdir.""" + contract = _write(tmp_path, _contract()) + monkeypatch.setenv("FLUID_TOFU_S3_LOCKFILE", "0") + _apply_as_the_old_release(contract, tmp_path) + before = _object(moto, _LEGACY_KEY) + assert before["resources"] + + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE") + _apply_as_the_old_release(contract, tmp_path) # must not raise opentofu_init_failed + + after = _object(moto, _LEGACY_KEY) + assert _addresses(after) == _addresses(before) + (workdir,) = [p.parent.parent for p in tmp_path.rglob(".terraform/terraform.tfstate")] + assert mig.recorded_backend(workdir)["s3"]["use_lockfile"] is True + + +def test_the_opt_out_takes_effect_on_a_workdir_that_had_locking_on(moto, tmp_path, monkeypatch): + """Apply with locking, then with FLUID_TOFU_S3_LOCKFILE=0 on the same kept workdir.""" + contract = _write(tmp_path, _contract()) + monkeypatch.delenv("FLUID_TOFU_S3_LOCKFILE", raising=False) + _apply_as_the_old_release(contract, tmp_path) + before = _object(moto, _LEGACY_KEY) + assert before["resources"] + (workdir,) = [p.parent.parent for p in tmp_path.rglob(".terraform/terraform.tfstate")] + assert mig.recorded_backend(workdir)["s3"]["use_lockfile"] is True + + monkeypatch.setenv("FLUID_TOFU_S3_LOCKFILE", "0") + _apply_as_the_old_release(contract, tmp_path) + + assert mig.recorded_backend(workdir)["s3"]["use_lockfile"] in (None, False) + assert _addresses(_object(moto, _LEGACY_KEY)) == _addresses(before) diff --git a/tests/integration/test_iac_cross_account_localstack.py b/tests/integration/test_iac_cross_account_localstack.py index c74e221f5..dcfc45df6 100644 --- a/tests/integration/test_iac_cross_account_localstack.py +++ b/tests/integration/test_iac_cross_account_localstack.py @@ -140,15 +140,15 @@ def _gate() -> tuple[bool, str]: return False, f"fluid_build.iac unavailable: {exc}" if runner.tofu_path() is None: return False, "the `tofu` (OpenTofu) binary is not on PATH" - for mod in ("boto3", "requests"): + for mod in ("boto3", "httpx"): try: __import__(mod) except ImportError: return False, f"the `{mod}` package is not installed" - import requests + import httpx try: - resp = requests.get(f"{ENDPOINT}/_localstack/health", timeout=5) + resp = httpx.get(f"{ENDPOINT}/_localstack/health", timeout=5) except Exception as exc: # noqa: BLE001 return False, f"LocalStack not reachable at {ENDPOINT}: {exc}" if resp.status_code != 200: @@ -188,10 +188,10 @@ def _client(service: str, key: str = KEY_A): def _reset() -> None: """Wipe every emulator backend so each test starts from empty.""" - import requests + import httpx try: - requests.post(f"{ENDPOINT}/_localstack/state/reset", timeout=20) + httpx.post(f"{ENDPOINT}/_localstack/state/reset", timeout=20) except Exception: # noqa: BLE001 - best-effort isolation pass diff --git a/tests/observability/test_redirect_hook_matches_httpx.py b/tests/observability/test_redirect_hook_matches_httpx.py new file mode 100644 index 000000000..d966dd19a --- /dev/null +++ b/tests/observability/test_redirect_hook_matches_httpx.py @@ -0,0 +1,142 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The redirect hook must judge the URL httpx will actually follow. + +``refuse_foreign_redirect`` resolves a ``Location`` itself; httpx resolves it +again in ``Client._redirect_url`` before following it. If the two ever +disagree, a hop the hook judged same-origin can land somewhere else with the +credential (an ``https:/x`` Location once did: httpx keeps the host and drops +the port). This compares the hook's verdict with where httpx goes, for every +pair below. ``_redirect_url`` is private on purpose: the test is a canary for +an httpx upgrade that changes redirect resolution, and should fail loudly then. +""" + +from __future__ import annotations + +import httpx +import pytest + +from fluid_build.observability.reporter import CredentialRedirectRefused, refuse_foreign_redirect + +pytestmark = [pytest.mark.unit] + +_DEFAULT_PORT = {"http": 80, "https": 443} + +_SOURCES = [ + "http://127.0.0.1:8001/a/b", + "https://cc.example.com:8443/a", + "http://cc.example.com/a", + "https://cc.example.com/a", + "http://[::1]:8001/a", + "https://[2001:db8::1]:8443/a", + "http://user:pw@cc.example.com:8080/a", # pragma: allowlist secret +] + +_LOCATIONS = [ + # hostless forms httpx completes with the request's host and no port + "http:/x", + "http:", + "http:///x", + "HTTP:/x", + "https:/x", + "https:", + "https:///x", + "hTtPs:", + "http:?q", + "http:#f", + "http:x", + "http:8002/x", + # backslashes, scheme-relative and userinfo tricks + "http:\\\\evil.com/x", + "/\\evil.com", + "\\\\evil.com", + "//evil.com", + "///evil.com", + "////evil.com", + "https://cc.example.com:8443@evil/", + "https://cc.example.com:8443@evil.com:8443/", + "http://evil.com\\@cc.example.com/", + "http://cc.example.com:80@evil.com/", + "//cc.example.com:8443@evil.com/", + # IPv6 and numeric host spellings + "http://[::1]:8001/", + "http://[::1]/", + "http://[0:0:0:0:0:0:0:1]:8001/", + "http://[::ffff:127.0.0.1]:8001/", + "http://[::ffff:a9fe:a9fe]/", + "http://[fe80::1%25en0]:8001/", + "http://127.0.0.1:8001/", + "http://127.1:8001/", + "http://0x7f000001:8001/", + "http://2130706433:8001/", + # case, trailing dot, ports and scheme changes + "http://CC.EXAMPLE.COM/", + "http://cc.example.com./", + "https://cc.example.com:443/", + "https://cc.example.com:8443/", + "https://cc.example.com:08443/", + "http://cc.example.com/", + "http://cc.example.com:8443/", + "https://cc.example.com/", + "http://cc.example.com:8080/", + "http://cc.example.com:80/", + "http://cc.example.com:0080/", + # whitespace, relative and odd schemes + "https://cc.example.com:8443\t.evil/", + " //evil.com", + "\t//evil.com", + "", + "?q=1", + "#x", + ".", + "..", + "../../../", + "ftp://cc.example.com/", + "file:///etc/passwd", + "javascript:alert(1)", + "data:,x", + "http://xn--e1afmkfd.com/", + "http://cc.example.com%2F@evil/", + "https://cc%2eexample%2ecom:8443/", + "https://cc.example.com:8443%40evil/", +] + + +def _origin(url: httpx.URL) -> tuple: + return (url.scheme, url.host, url.port or _DEFAULT_PORT.get(url.scheme)) + + +@pytest.mark.parametrize("source", _SOURCES) +def test_an_allowed_redirect_lands_where_the_hook_judged(source: str) -> None: + client = httpx.Client() + request = httpx.Request("GET", source) + src = _origin(request.url) + escaped = [] + for location in _LOCATIONS: + response = httpx.Response( + 302, headers=[(b"Location", location.encode("latin-1"))], request=request + ) + try: + refuse_foreign_redirect(response) + except CredentialRedirectRefused: + continue + try: + lands = _origin(client._redirect_url(request, response)) + except Exception: # noqa: BLE001 - httpx refuses to follow it either + continue + upgrade = src[0] == "http" and src[2] == 80 and lands == ("https", src[1], 443) + if lands != src and not upgrade: + escaped.append((location, lands)) + assert not escaped, f"hook allowed hops that httpx sends elsewhere: {escaped}" diff --git a/tests/observability/test_reporter_redirects.py b/tests/observability/test_reporter_redirects.py index 373a1b7d3..2b8314fc2 100644 --- a/tests/observability/test_reporter_redirects.py +++ b/tests/observability/test_reporter_redirects.py @@ -15,7 +15,7 @@ """A credential-bearing request never follows a redirect off its own origin. The Command Center reporter sends ``X-API-Key`` (and, for a bearer credential, -``Authorization``) as session headers. ``requests`` follows redirects by +``Authorization``) as session headers. ``httpx`` follows redirects by default and drops only ``Authorization`` when the host changes, so a Command Center (or anything in front of it) answering ``307 Location: `` used to receive the run, and the key, at the second address. Two real loopback @@ -28,13 +28,13 @@ from __future__ import annotations -import io import json import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, Dict, Iterator, List, Optional, Tuple import pytest +import respx pytestmark = pytest.mark.unit @@ -168,38 +168,18 @@ def test_the_reporter_still_follows_a_redirect_on_its_own_origin(two_origins): assert reporter.stats == {"sent": 2, "failed": 0} -class _TlsRedirect: - """A requests transport adapter standing in for the TLS hop: answers 307 to ``location``.""" - - def __init__(self, location: str) -> None: - self.location = location - - def send(self, request: Any, **_kwargs: Any) -> Any: - import requests - - response = requests.Response() - response.status_code = 307 - response.reason = "Temporary Redirect" - response.headers["Location"] = self.location - response.url = request.url - response.request = request - response.raw = io.BytesIO(b"") - response.connection = self - return response - - def close(self) -> None: - pass - - def test_the_reporter_does_not_follow_https_down_to_http_on_the_same_host(two_origins): _, plain = two_origins port = plain.url.rsplit(":", 1)[1] reporter = _reporter(f"https://127.0.0.1:{port}") # Only the TLS answer is staged; the http hop it redirects to is the real listener. - reporter.session.mount(f"https://127.0.0.1:{port}", _TlsRedirect(f"{plain.url}/api/v1/x")) - - _report_a_run(reporter) + with respx.mock(assert_all_called=False, assert_all_mocked=False) as router: + tls = router.route(scheme="https", host="127.0.0.1", port=int(port)).respond( + 307, headers={"Location": f"{plain.url}/api/v1/x"} + ) + _report_a_run(reporter) + assert tls.called assert plain.carrying(_KEY, _TOKEN) == [] assert plain.requests == [] assert reporter.stats["sent"] == 0 diff --git a/tests/providers/test_retired_providers.py b/tests/providers/test_retired_providers.py new file mode 100644 index 000000000..1805407b9 --- /dev/null +++ b/tests/providers/test_retired_providers.py @@ -0,0 +1,269 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``redshift`` and ``datamesh-manager`` are no longer providers. + +``redshift`` only forwarded to ``aws`` (its rollback DDL is reached through the +snapshot's provider name) and Data Mesh Manager is a catalog. A retired name is +not "unknown": it exits 2 with the replacement, from ``--provider`` and +``FLUID_PROVIDER`` alike. What they did keeps working: the Redshift rollback +dispatch and ``fluid publish --target datamesh-manager``. +""" + +from __future__ import annotations + +import logging + +import pytest + +from fluid_build import providers as P +from fluid_build._contract_loader import CLIError +from fluid_build.cli import main +from fluid_build.cli._common import build_provider +from fluid_build.cli.plan import _gate_provider_flag + +_BUILTINS = {"aws", "databricks", "gcp", "local", "snowflake"} + +_REDSHIFT_HINT = ( + "Provider 'redshift' was retired: Redshift is part of the aws provider; " "use --provider aws" +) +_DMM_HINT = ( + "Provider '{name}' was retired: Data Mesh Manager is a catalog, not a provider; " + "use `fluid publish --target datamesh-manager` (or `fluid datamesh-manager`)" +) + + +def test_built_in_provider_set_is_exactly_five(): + P.discover_providers(force=True) + assert set(P.list_providers()) == _BUILTINS + + +def test_retired_packages_stay_importable_but_do_not_autoregister(): + import fluid_build.providers.datamesh_manager as dmm + from fluid_build.providers.aws import RedshiftProvider + + assert RedshiftProvider and dmm.DataMeshManagerProvider + assert getattr(dmm, "__fluid_no_autoregister__", False) is True + assert not hasattr(dmm, "NAME") and not hasattr(dmm, "Provider") + + +_RETIRED = [ + ("redshift", _REDSHIFT_HINT), + ("datamesh-manager", _DMM_HINT.format(name="datamesh-manager")), + ("datamesh_manager", _DMM_HINT.format(name="datamesh_manager")), +] + + +def _main_output(argv, caplog, capsys): + with caplog.at_level(logging.ERROR): + rc = main(argv) + out = capsys.readouterr() + return rc, caplog.text + out.err + out.out + + +@pytest.mark.parametrize("name, expected", _RETIRED) +def test_flag_exits_2_with_the_hint(name, expected, caplog, capsys, monkeypatch): + monkeypatch.delenv("FLUID_PROVIDER", raising=False) + rc, text = _main_output(["--provider", name, "providers"], caplog, capsys) + assert rc == 2 + assert expected in text + + +@pytest.mark.parametrize("name, expected", _RETIRED) +def test_env_var_behaves_like_the_flag(name, expected, caplog, capsys, monkeypatch): + monkeypatch.setenv("FLUID_PROVIDER", name) + rc, text = _main_output(["providers"], caplog, capsys) + assert rc == 2 + assert expected in text + + +@pytest.mark.parametrize("name, expected", _RETIRED) +def test_build_provider_and_plan_gate_raise_provider_retired(name, expected): + for call in ( + lambda: build_provider(name, None, None, logging.getLogger("t")), + lambda: _gate_provider_flag(name, logging.getLogger("t")), + ): + with pytest.raises(CLIError) as exc: + call() + assert exc.value.event == "provider_retired" + assert exc.value.exit_code == 2 + # Both normalise "-" to "_" before the lookup, so they echo that spelling. + assert exc.value.context["error"].split(": ", 1)[1] == expected.split(": ", 1)[1] + + +def test_an_unknown_name_is_still_just_unknown(capsys, monkeypatch, caplog): + monkeypatch.delenv("FLUID_PROVIDER", raising=False) + with caplog.at_level(logging.ERROR): + assert main(["--provider", "nosuchcloud", "providers"]) == 2 + out = capsys.readouterr() + assert "Unknown provider 'nosuchcloud'" in caplog.text + out.err + out.out + assert "was removed" not in caplog.text + out.err + out.out + + +def test_retired_hint_is_none_for_a_live_provider(): + assert P.retired_provider_hint("aws") is None + assert P.retired_provider_hint(None) is None + + +def test_redshift_rollback_dispatch_still_emits_ddl(): + from fluid_build.cli._rollback_writer import _restore_ddl_for_snapshot + + ddl = _restore_ddl_for_snapshot( + { + "location": { + "database": "db", + "schema": "sch", + "table": "t", + "backup_table": "t_bak", + } + }, + "redshift", + ) + assert ddl == [ + "BEGIN", + "DROP TABLE IF EXISTS db.sch.t", + "CREATE TABLE db.sch.t AS SELECT * FROM db.sch.t_bak", + "COMMIT", + ] + + +@pytest.mark.parametrize("alias", ["datamesh_manager", "datamesh-manager", "entropy-data", "dmm"]) +def test_datamesh_manager_still_resolves_as_a_publish_target(alias): + import fluid_build.build_runners.catalog_registrars # noqa: F401 (registers backends) + from fluid_build.api.catalog_backend import get_catalog_backend + + spec = get_catalog_backend(alias) + assert spec is not None and spec.name == "datamesh_manager" + + +class _FakeEntryPoint: + value = "evil_pkg.provider:Evil" + + def __init__(self, name): + self.name = name + self.loaded = False + + def load(self): + self.loaded = True + return type("Evil", (), {}) + + +class _FakeEntryPoints: + def __init__(self, *eps): + self._eps = eps + + def select(self, group): + return list(self._eps) if group == "fluid_build.providers" else [] + + def get(self, group, default=()): + return self.select(group) or list(default) + + +@pytest.fixture +def _restore_registry(): + before = dict(P.PROVIDERS) + done = P._DISCOVERY_DONE + yield + P.PROVIDERS.clear() + P.PROVIDERS.update(before) + P._DISCOVERY_DONE = done + + +@pytest.mark.parametrize("name", ["redshift", "datamesh-manager"]) +def test_a_plugin_cannot_claim_a_retired_name(name, monkeypatch, caplog, _restore_registry): + ep = _FakeEntryPoint(name) + monkeypatch.setattr(P.importlib.metadata, "entry_points", lambda: _FakeEntryPoints(ep)) + with caplog.at_level(logging.WARNING): + P._discover_entrypoints(logging.getLogger("t")) + assert not ep.loaded + assert P._normalize_name(name) not in P.PROVIDERS + records = [ + r for r in caplog.records if getattr(r, "evt", "") == "provider_retired_name_reserved" + ] + assert records and records[0].plugin == _FakeEntryPoint.value + # The CLI prints the message only, so the plugin must be named in it. + assert repr(_FakeEntryPoint.value) in records[0].getMessage() + + +def test_a_plugin_claiming_redshift_does_not_replace_the_retirement_hint( + monkeypatch, caplog, capsys, _restore_registry +): + monkeypatch.delenv("FLUID_PROVIDER", raising=False) + monkeypatch.setattr( + P.importlib.metadata, "entry_points", lambda: _FakeEntryPoints(_FakeEntryPoint("redshift")) + ) + P.PROVIDERS.clear() + P.discover_providers(force=True) + rc, text = _main_output(["--provider", "redshift", "providers"], caplog, capsys) + assert rc == 2 + assert _REDSHIFT_HINT in text + + +def test_an_explicit_registration_of_a_retired_name_stays_allowed(_restore_registry): + P.register_provider("redshift", object) + assert P.PROVIDERS["redshift"] is object + + +def test_the_hint_does_not_depend_on_the_registry_loading(monkeypatch, caplog, capsys): + # A registry that failed to load (``_known_provider_names`` -> []) must not hide it. + from fluid_build import cli + + monkeypatch.delenv("FLUID_PROVIDER", raising=False) + monkeypatch.setattr(cli, "_known_provider_names", lambda: []) + rc, text = _main_output(["--provider", "redshift", "providers"], caplog, capsys) + assert rc == 2 + assert _REDSHIFT_HINT in text + + +def test_a_blocking_allowlist_does_not_hide_the_hint(monkeypatch, caplog, capsys): + monkeypatch.delenv("FLUID_PROVIDER", raising=False) + monkeypatch.setenv("FLUID_PLUGINS_ALLOWLIST", "zzz") + rc, text = _main_output(["--provider", "redshift", "providers"], caplog, capsys) + assert rc == 2 + assert _REDSHIFT_HINT in text + + +def test_a_contract_naming_redshift_gets_the_aws_native_plan(monkeypatch): + # generate iac's native planner resolved a contract platform through build_provider + # and swallowed provider_retired, so a redshift contract lost its native actions. + from fluid_build.cli import generate_iac + + seen = {} + + def _build(name, project, region, logger): + seen["name"] = name + return type("P", (), {"plan": lambda self, c: []})() + + monkeypatch.setattr("fluid_build.cli._common.build_provider", _build) + contract = {"exposes": [{"binding": {"platform": "redshift", "location": {}}}]} + generate_iac.native_actions(contract, logging.getLogger("t")) + assert seen["name"] == "aws" + + +def test_policy_apply_bindings_resolve_a_retired_platform(): + from fluid_build.cli.policy_apply import _resolve_from_bindings + + assert _resolve_from_bindings({"bindings": [{"provider": "redshift"}]})[0] == "aws" + + +def test_a_non_string_platform_passes_through_unchanged(): + # LLM-drafted contracts can carry a non-string platform; the successor + # lookup must hand it on as before, not raise AttributeError. + from fluid_build.cli._common import resolve_provider_from_contract + from fluid_build.providers import _claims_retired_name, resolve_retired_platform + + assert resolve_retired_platform(123) == 123 + assert resolve_retired_platform(None) is None + assert _claims_retired_name(123, "plugin", None) is False + assert resolve_provider_from_contract({"exposes": [{"binding": {"platform": 123}}]})[0] == 123 diff --git a/tests/security/test_trailing_newline_anchors.py b/tests/security/test_trailing_newline_anchors.py new file mode 100644 index 000000000..445209b87 --- /dev/null +++ b/tests/security/test_trailing_newline_anchors.py @@ -0,0 +1,429 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Whole-string validators refuse a value that ends in a newline. + +Python's ``$`` matches at the end of the string and also just before a final +``\\n``, so ``re.match(r"^[a-z]+$", "orders\\n")`` succeeds and the caller goes on +to use ``"orders\\n"``: as a path segment, a SQL identifier or literal, a +Terraform attribute, a cache directory. Contract YAML yields such values easily +(a ``|`` block scalar ends in one), and so do ``.env`` files and percent-encoded +URL queries. The validators below end in ``\\Z`` (or call ``fullmatch``), which +matches only at the end of the string. + +Each test hands one validator a value with a trailing newline, next to the same +value without it. It fails while that validator's pattern still ends in ``$``. +""" + +from __future__ import annotations + +import logging +import os +import sqlite3 +from types import SimpleNamespace + +import pytest + +pytestmark = [pytest.mark.unit] + + +# --- SQL identifiers and literals -------------------------------------------- + + +def test_quality_engine_refuses_a_selector_ending_in_a_newline(): + from fluid_build.providers import quality_engine + + assert '"amount"' in quality_engine._completeness_sql("t", "amount") + with pytest.raises(ValueError, match="Invalid SQL identifier"): + quality_engine._completeness_sql("t", "amount\n") + + +def test_catalog_identifier_quoting_refuses_a_trailing_newline(): + from fluid_build.copilot.catalog._patterns import validate_and_quote_identifier + from fluid_build.copilot.catalog.base import CatalogConfigError + + assert validate_and_quote_identifier("SALES", kind="database") == '"SALES"' + with pytest.raises(CatalogConfigError): + validate_and_quote_identifier("SALES\n", kind="database") + + +@pytest.mark.parametrize( + ("kind", "value"), [("project", "acme-proj"), ("dataset", "sales"), ("table", "orders")] +) +def test_bigquery_catalog_identifiers_refuse_a_trailing_newline(kind, value): + from fluid_build.copilot.catalog.base import CatalogConfigError + from fluid_build.copilot.catalog.bigquery import _quote_ident + + assert _quote_ident(value, kind=kind) == f"`{value}`" + with pytest.raises(CatalogConfigError): + _quote_ident(f"{value}\n", kind=kind) + + +@pytest.mark.parametrize( + ("kind", "value"), [("project", "acme-proj"), ("dataset", "sales"), ("table", "orders")] +) +def test_bigquery_catalog_name_check_refuses_a_trailing_newline(kind, value): + # The unquoted check behind `table_name IN ('...')` and FQN parsing. + from fluid_build.copilot.catalog.base import CatalogConfigError + from fluid_build.copilot.catalog.bigquery import _validate_ident + + assert _validate_ident(value, kind=kind) == value + with pytest.raises(CatalogConfigError): + _validate_ident(f"{value}\n", kind=kind) + + +def test_jdbc_schema_filter_refuses_a_trailing_newline(monkeypatch, tmp_path): + pytest.importorskip("duckdb") + from fluid_build.cli.discover import _jdbc_introspect + from fluid_build.providers import _duckdb_sandbox + + def _no_connect(*_args, **_kwargs): + raise AssertionError("introspection connected with an invalid schema filter") + + monkeypatch.setattr(_duckdb_sandbox, "secure_duckdb_connect", _no_connect) + db_path = tmp_path / "source.sqlite" + sqlite3.connect(str(db_path)).close() + with pytest.raises(ValueError, match="Invalid schema filter"): + _jdbc_introspect.introspect_jdbc( + source="sqlite", uri=f"sqlite:///{db_path}", schema_filter="main\n" + ) + + +def test_bigquery_project_quoting_refuses_a_trailing_newline(): + # bigquery_validation imports google.cloud.bigquery at module load, and the + # GCP SDK is a provider extra the minimal unit lanes do not install. + pytest.importorskip("google.cloud.bigquery") + from fluid_build.providers.bigquery_validation import _quote_bq_project + + assert _quote_bq_project("acme-analytics") == "`acme-analytics`" + with pytest.raises(ValueError, match="Invalid BigQuery project id"): + _quote_bq_project("acme-analytics\n") + + +def test_gcp_planner_project_refuses_a_trailing_newline(): + from fluid_build.providers.gcp.plan.planner import _validate_bq_project + + assert _validate_bq_project("acme-analytics") == "acme-analytics" + with pytest.raises(ValueError, match="Invalid GCP project ID"): + _validate_bq_project("acme-analytics\n") + + +def test_snowflake_account_segment_refuses_a_trailing_newline(): + from fluid_build.providers.snowflake.util import config + + assert config._normalize_account_identifier("xy12345.eu-west-1") == "xy12345.eu-west-1" + with pytest.raises(ValueError, match="Invalid Snowflake account identifier"): + config._normalize_account_identifier("xy12345\n.eu-west-1") + + +def test_snowflake_account_region_pattern_refuses_a_trailing_newline(): + # The segment rule runs first and now refuses this input on its own; the + # region rule must hold without it. + from fluid_build.providers.snowflake.util import config + + assert config._ACCOUNT_REGION_SEGMENT_RE.match("eu-west-1") + assert config._ACCOUNT_REGION_SEGMENT_RE.match("eu-west-1\n") is None + + +# --- Names that become paths, files and cache directories --------------------- + + +def test_build_identifier_refuses_a_trailing_newline(): + from fluid_build.build_runners._ids import IdentifierViolation, validate_identifier + + assert validate_identifier("orders", kind="contract.id") == "orders" + with pytest.raises(IdentifierViolation): + validate_identifier("orders\n", kind="contract.id") + + +def test_federation_workspace_id_refuses_a_trailing_newline(): + from fluid_build.forge.federation import FederatedWorkspace + + row = {"id": "telco", "kind": "git_registry", "endpoint": "https://github.com/acme/telco"} + assert FederatedWorkspace.from_dict(row).id == "telco" + with pytest.raises(ValueError, match="federation workspace id"): + FederatedWorkspace.from_dict({**row, "id": "telco\n"}) + + +def test_bundle_recorded_env_refuses_a_trailing_newline(tmp_path): + from fluid_build.forge.core.bundle import resolve_bundle_source_contract + + tgz = tmp_path / "bundle.tgz" + _, why = resolve_bundle_source_contract(tgz, {"contract": "contract.fluid.yaml", "env": "aws"}) + assert "environment name" not in why + path, why = resolve_bundle_source_contract( + tgz, {"contract": "contract.fluid.yaml", "env": "aws\n"} + ) + assert path is None + assert "is not an environment name" in why + + +def test_consumes_overlay_env_refuses_a_trailing_newline(): + from fluid_build.build_runners._embedded_sql_io import ConsumesResolutionError, _validated_env + + assert _validated_env("aws") == "aws" + with pytest.raises(ConsumesResolutionError): + _validated_env("aws\n") + + +@pytest.mark.skipif(os.name == "nt", reason="a Windows file name cannot hold a newline") +@pytest.mark.parametrize( + ("resolver", "name"), + [("_resolve_tap_binary", "tap-postgres"), ("_resolve_target_binary", "target-jsonl")], +) +def test_singer_binary_lookup_refuses_a_name_ending_in_a_newline( + monkeypatch, tmp_path, resolver, name +): + from fluid_build.build_runners.meltano import runner + + for filename in (name, f"{name}\n"): + binary = tmp_path / filename + binary.write_text("#!/bin/sh\n", encoding="utf-8") + binary.chmod(0o755) + monkeypatch.setenv("PATH", str(tmp_path)) + resolve = getattr(runner, resolver) + + assert resolve(name) == str(tmp_path / name) + assert resolve(f"{name}\n") is None + + +def test_prompt_profile_name_refuses_a_trailing_newline(): + from fluid_build.cli.forge_copilot_prompts import PromptProfileError, _resolve_profile_dir + + assert _resolve_profile_dir("eu-gdpr-strict").name == "eu-gdpr-strict" + with pytest.raises(PromptProfileError, match="invalid prompt profile name"): + _resolve_profile_dir("eu-gdpr-strict\n") + + +@pytest.mark.parametrize( + ("module", "name"), + [ + ("fluid_build.cli.forge_domain_enrichment", "_SAFE_DOMAIN_RE"), + ("fluid_build.cli._ai_setup_storage", "_DOMAIN_SLUG_RE"), + ], +) +def test_domain_slug_patterns_refuse_a_trailing_newline(module, name): + # Fed a stripped or on-disk value, so pinned on the pattern: their comments + # say they mirror the profile-name guard, and they now do. + import importlib + + pattern = getattr(importlib.import_module(module), name) + assert pattern.match("finance") + assert pattern.match("finance\n") is None + + +def test_prompt_overlay_name_refuses_a_trailing_newline(): + from fluid_build.cli.forge_prompt_overlays import PromptOverlayError, _resolve_overlay_path + + assert _resolve_overlay_path("pii-lockdown").name == "pii-lockdown.yaml" + with pytest.raises(PromptOverlayError, match="invalid prompt overlay name"): + _resolve_overlay_path("pii-lockdown\n") + + +# --- Identifiers written into a contract ------------------------------------- + + +def test_jdbc_forged_product_id_drops_a_trailing_newline(monkeypatch, tmp_path): + import yaml + + from fluid_build.cli import _forge_data_model_jdbc + from fluid_build.cli.discover import _jdbc_introspect + + table = _jdbc_introspect.IntrospectedTable( + schema="public", + name="orders", + columns=[_jdbc_introspect.IntrospectedColumn(name="id", type_name="integer")], + ) + db = _jdbc_introspect.IntrospectedDatabase( + source_kind="postgres", database="orders", tables=[table] + ) + monkeypatch.setattr(_jdbc_introspect, "introspect_jdbc", lambda **_kwargs: db) + + def _forged_id(name): + out = tmp_path / "out.fluid.yaml" + args = SimpleNamespace( + source="postgres", + uri="postgresql://forge@db.example/orders", + schema_name=None, + tables=None, + name=name, + output=str(out), + ) + assert _forge_data_model_jdbc._run_from_jdbc_source(args, logging.getLogger("test")) == 0 + return yaml.safe_load(out.read_text(encoding="utf-8"))["id"] + + assert _forged_id("orders") == "orders" + assert _forged_id("orders\n") == "orders" + + +def test_dbt_importer_slugifies_an_identifier_ending_in_a_newline(): + from fluid_build.cli.import_workflow.dbt import _safe_identifier + + assert _safe_identifier("orders") == "orders" + assert _safe_identifier("orders\n") == "orders" + + +def test_odcs_import_slugifies_an_id_ending_in_a_newline(): + # The bundled schema's identifier pattern ends in `$`; it is matched whole. + from fluid_build.providers.odcs.mappers.normalize import _root_identifier + + assert _root_identifier("orders", "0.7.5") == ("orders", None) + assert _root_identifier("orders\n", "0.7.5")[0] == "orders" + + +def test_odps_import_semver_check_refuses_a_trailing_newline(): + from fluid_build.providers.odps_standard.provider import _is_semver + + assert _is_semver("1.2.3", "0.7.5") is True + assert _is_semver("1.2.3\n", "0.7.5") is False + + +def test_mission_predicate_path_refuses_a_trailing_newline(): + from fluid_build.copilot.missions.spec import MissionSpecError, validate_predicate_path + + # A stripped path can still carry one at the end of an inner segment. + validate_predicate_path("exposes[*].contract.dq", label="checks[0]") + with pytest.raises(MissionSpecError, match=r"invalid path segment 'contract\\n'"): + validate_predicate_path("exposes[*].contract\n.dq", label="checks[0]") + + +# --- Values emitted into OpenTofu --------------------------------------------- + + +def test_row_filter_name_refuses_a_trailing_newline(): + from fluid_build.iac.base import UnsupportedBindingError + from fluid_build.iac.row_access import row_filters_for + + def _expose(name): + row_filter = { + "principal": "group:analysts@northwind.example", + "name": name, + "where": "consent = true", + } + return { + "exposeId": "profile", + "contract": {"schema": [{"name": "consent", "type": "boolean"}]}, + "policy": {"authz": {"rowFilters": [row_filter]}}, + } + + assert row_filters_for(_expose("analysts_consented"))[0].name == "analysts_consented" + with pytest.raises(UnsupportedBindingError, match="must be lowercase letters"): + row_filters_for(_expose("analysts_consented\n")) + + +def _aws_contract(region): + location = { + "bucket": "acme-lake", + "path": "bronze/orders/", + "database": "bronze", + "table": "orders", + "region": region, + } + expose = { + "exposeId": "orders", + "binding": {"platform": "aws", "format": "parquet", "location": location}, + "contract": {"schema": [{"name": "id", "type": "string"}]}, + } + return {"id": "bronze.orders", "exposes": [expose]} + + +def test_aws_binding_region_ending_in_a_newline_is_refused(monkeypatch): + # Neither dropped (the provider would take the environment's region) nor + # stripped (sovereignty checks the value as written, so a stripped pin + # could apply a region its deniedRegions names). + from fluid_build.iac import get_iac_plugin, provider_config + from fluid_build.iac.base import UnsupportedBindingError + + for name in [k for k in os.environ if k.startswith("AWS_ENDPOINT_URL")]: + monkeypatch.delenv(name) + plugin = get_iac_plugin("aws") + + assert provider_config(plugin, _aws_contract("eu-west-1"))["region"] == "eu-west-1" + with pytest.raises(UnsupportedBindingError, match="whitespace around it"): + provider_config(plugin, _aws_contract("eu-west-1\n")) + + +def test_aws_recorded_region_ending_in_a_newline_is_not_a_region(): + from fluid_build.iac.providers.aws import recorded_regions + + assert recorded_regions([{"values": {"region": "eu-west-1"}}]) == {"eu-west-1"} + assert recorded_regions([{"values": {"region": "eu-west-1\n"}}]) == set() + + +@pytest.mark.parametrize( + "query", ["branch=main%0A", "tag=v1.2%0A", "commit=0a1b2c3%0A", "branch=main&path=dbt%0A"] +) +def test_databricks_git_source_refuses_a_ref_or_path_ending_in_a_newline(query): + from fluid_build.iac.providers.databricks_jobs import _git_source + + clean, refused = [], [] + repository = f"https://github.com/acme/jobs?{query}" + assert _git_source(repository.replace("%0A", ""), "builds[0]", clean) is not None + assert not clean + assert _git_source(repository, "builds[0]", refused) is None + assert refused + + +# --- Name validators other code calls --------------------------------------- + + +@pytest.mark.parametrize( + ("method", "value"), + [ + ("validate_s3_bucket_name", "acme-lake"), + ("validate_glue_database_name", "bronze"), + ("validate_glue_table_name", "orders"), + ("validate_lambda_function_name", "ingest-orders"), + ], +) +def test_aws_resource_names_refuse_a_trailing_newline(method, value): + from fluid_build.providers.aws.util.validation import ResourceValidator + + check = getattr(ResourceValidator("123456789012", "eu-west-1"), method) + assert check(value) == (True, None) + valid, message = check(f"{value}\n") + assert valid is False + assert f"{value}\\n" in message # the refused newline is shown, not printed + + +@pytest.mark.parametrize( + ("resource_type", "name"), + [("dataset", "sales"), ("bucket", "acme-lake"), ("topic", "orders-events"), ("job", "nightly")], +) +def test_gcp_resource_names_refuse_a_trailing_newline(resource_type, name): + from fluid_build.providers.gcp.util.names import validate_name + + assert validate_name(name, resource_type) is True + assert validate_name(f"{name}\n", resource_type) is False + + +def test_project_name_check_refuses_a_trailing_newline(): + from fluid_build.cli.security import InputSanitizer + + assert InputSanitizer.validate_project_name("my-project") is True + assert InputSanitizer.validate_project_name("my-project\n") is False + + +def test_forge_project_name_refuses_a_trailing_newline(): + from fluid_build.cli.forge_validation import validate_project_name + + assert validate_project_name("orders-product") == (True, None) + assert validate_project_name("orders-product\n")[0] is False + + +def test_forge_template_name_refuses_a_trailing_newline(): + from fluid_build.cli.forge_validation import validate_template_name + + assert validate_template_name("starter") == (True, None) + assert validate_template_name("starter\n")[0] is False diff --git a/tests/streaming/test_standards.py b/tests/streaming/test_standards.py index 42f55e479..583b583ac 100644 --- a/tests/streaming/test_standards.py +++ b/tests/streaming/test_standards.py @@ -78,12 +78,13 @@ def test_array_element_nullability_tightening_is_breaking(contract): def test_a_container_child_carrying_its_own_name_is_still_diffed_at_its_slot(): - # A `name` inside element/key/value (schema-invalid, but the diff never - # validates) must not drop the child or move it to another path. + # A `name` inside element/key/value is schema-invalid, so this goes through + # the unvalidated diff (validate=False, for a contract still being + # repaired); it must not drop the child or move it to another path. def contract(value_type): child = {"name": 7, "type": value_type} column = {"name": "tags", "type": "map", "key": {"type": "string"}, "value": child} return {"exposes": [{"exposeId": "e", "contract": {"schema": [column]}}]} - report = compare_contracts(contract("int"), contract("string")) + report = compare_contracts(contract("int"), contract("string"), validate=False) assert [c.path for c in report.breaking] == ["exposes[0].contract.schema.tags.value.type"] diff --git a/tests/test_catalog_auth_schema.py b/tests/test_catalog_auth_schema.py index 30a9527b0..06275bb65 100644 --- a/tests/test_catalog_auth_schema.py +++ b/tests/test_catalog_auth_schema.py @@ -164,6 +164,8 @@ def test_ga_075_refuses_the_key(): ({**_OAUTH2, "credentialEnv": "LK_CRED\n"}, "credentialEnv: does not match"), ({"type": "bearer", "tokenEnv": "UC_TOKEN\n"}, "tokenEnv: does not match"), ({"type": "bearer", "tokenEnv": _PAT}, "tokenEnv: must not match"), + # The token check's "$" no longer matches before the "\n"; the name pattern refuses it. + ({"type": "bearer", "tokenEnv": _PAT + "\n"}, "tokenEnv: does not match"), ( {**_OAUTH2, "credentialEnv": "AKIAIOSFODNN7EXAMPLE"}, # pragma: allowlist secret "credentialEnv: must not match", @@ -193,9 +195,10 @@ def test_076_refuses_a_malformed_catalog_auth(catalog_auth, expected): ({"type": "bearer", "tokenEnv": "T", "credentialEnv": _SECRET}, _SECRET), ({"type": _SECRET}, _SECRET), ({"type": "bearer", "tokenEnv": _PAT}, _PAT), + ({"type": "bearer", "tokenEnv": _PAT + "\n"}, _PAT), ({**_OAUTH2, "oauth2ServerUri": _USERINFO_URI}, "2OR3eRvYfSZ"), ], - ids=["credentialEnv", "tokenEnv", "wrong-type-field", "type", "pat", "userinfo"], + ids=["credentialEnv", "tokenEnv", "wrong-type-field", "type", "pat", "pat-lf", "userinfo"], ) def test_a_schema_error_never_quotes_the_value(catalog_auth, secret): errors = _errors("0.7.6", catalog_auth) @@ -224,6 +227,7 @@ def test_schema_and_parser_share_the_variable_name_rule(): {**_OAUTH2, "credentialEnv": "LK_CRED\n"}, {"type": "bearer", "tokenEnv": "UC_TOKEN\n"}, {"type": "bearer", "tokenEnv": _PAT}, + {"type": "bearer", "tokenEnv": _PAT + "\n"}, {"type": "bearer", "tokenEnv": "S3_CREDENTIAL_V2"}, {"type": "bearer", "tokenEnv": "LAKEKEEPERCATALOGCREDENTIALPROD"}, {**_OAUTH2, "oauth2ServerUri": _USERINFO_URI}, diff --git a/tests/test_ci_extras_alongside_base.py b/tests/test_ci_extras_alongside_base.py new file mode 100644 index 000000000..97de9c30e --- /dev/null +++ b/tests/test_ci_extras_alongside_base.py @@ -0,0 +1,111 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The extras that will carry litellm, mcp and openlineage once base drops them. + +Slice 2 of the dependency diet adds ``[litellm]``, ``[mcp]``, ``[lineage]``, +``[standard]`` and ``[ai]`` while base still lists all three packages. These +tests keep that layout from drifting before the removal release. +""" + +from __future__ import annotations + +import pytest + +from tests.ci_guards import extras_alongside_base as g + +PROJECT = g.load() +BASE = g.base_requirements(PROJECT) +EXTRAS = g.extras(PROJECT) + +#: Extras that ``all`` deliberately leaves out, each with its reason. A new +#: extra must either join ``all`` or be added here: that is the point. +NOT_IN_ALL = { + "all": "itself", + "sdk": "plugin-author tooling, not a runtime feature", + "odcs-strict": "opt-in second validation pass", + "databricks": "reachable through [catalogs]", + "datahub": "reachable through [catalogs]", + "keyring": "empty alias; keyring is a base dependency", + "catalogs": "umbrella of adapters, installed on request", + "opds": "back-compat alias of [odps]", + "standard": "alias group, covered through [litellm] and [mcp]", + "ai": "alias of [litellm]", + "test-emulators": "CI-only emulators", + "test-emulators-heavy": "CI-only emulators", +} + + +def _base_lines(package: str) -> list[str]: + return [r for r in BASE if g.requirement_name(r) == package] + + +def _names(extra: str) -> set[str]: + return {g.requirement_name(r) for r in g.closure(extra, EXTRAS)} + + +class TestLitellmMarkerSplit: + def test_base_still_has_the_split(self): + # If this fails the split moved or was collapsed in base; fix the + # guard and the extra together, never one alone. + assert len(_base_lines("litellm")) == 2 + + def test_extra_copies_both_marker_lines_verbatim(self): + assert sorted(EXTRAS["litellm"]) == sorted(_base_lines("litellm")) + + def test_python_310_keeps_the_cap(self): + capped = [r for r in EXTRAS["litellm"] if "python_version < '3.11'" in r] + assert len(capped) == 1 + assert "<1.98" in capped[0] + + +class TestFilledExtras: + def test_mcp_matches_base_and_declares_pyjwt(self): + assert [r for r in _base_lines("mcp")][0] in EXTRAS["mcp"] + assert "pyjwt" in _names("mcp") + + def test_lineage_matches_base(self): + assert EXTRAS["lineage"] == _base_lines("openlineage-python") + + def test_standard_covers_litellm_and_mcp(self): + assert g.closure("litellm", EXTRAS) <= g.closure("standard", EXTRAS) + assert g.closure("mcp", EXTRAS) <= g.closure("standard", EXTRAS) + + def test_ai_equals_litellm(self): + assert g.closure("ai", EXTRAS) == g.closure("litellm", EXTRAS) + + +class TestUmbrellas: + @pytest.mark.parametrize("extra", ["litellm", "mcp", "lineage"]) + def test_all_covers_the_new_extras(self, extra): + assert g.closure(extra, EXTRAS) <= g.closure("all", EXTRAS) + + def test_all_covers_every_user_facing_extra(self): + covered = g.closure("all", EXTRAS) + missing = [ + name + for name in EXTRAS + if name not in NOT_IN_ALL and not g.closure(name, EXTRAS) <= covered + ] + assert not missing, f"extras missing from [all] (or add to NOT_IN_ALL): {missing}" + + def test_the_exclusion_list_names_only_real_extras(self): + assert set(NOT_IN_ALL) <= set(EXTRAS) + + def test_all_keeps_dev(self): + assert g.closure("dev", EXTRAS) <= g.closure("all", EXTRAS) + + @pytest.mark.parametrize("extra", ["litellm", "mcp", "lineage"]) + def test_dev_covers_the_new_extras(self, extra): + assert g.closure(extra, EXTRAS) <= g.closure("dev", EXTRAS) diff --git a/tests/test_circuit_breaker.py b/tests/test_circuit_breaker.py index 330d4f08b..fcba9f72e 100644 --- a/tests/test_circuit_breaker.py +++ b/tests/test_circuit_breaker.py @@ -149,10 +149,9 @@ def test_send_logs_noop_when_disabled(self): def test_send_logs_noop_when_empty_logs(self): config = self._make_config(configured=True) - with patch("fluid_build.observability.reporter.requests", MagicMock()): - reporter = CommandCenterReporter(config) - reporter.send_logs("123", []) - assert reporter.queue.empty() + reporter = CommandCenterReporter(config) + reporter.send_logs("123", []) + assert reporter.queue.empty() def test_send_metrics_noop_when_disabled(self): config = self._make_config(configured=False) @@ -160,8 +159,7 @@ def test_send_metrics_noop_when_disabled(self): reporter.send_metrics("123", [{"name": "duration", "value": 1.0}]) assert reporter.queue.empty() - @patch("fluid_build.observability.reporter.requests") - def test_enqueue_adds_to_queue(self, _mock_requests): + def test_enqueue_adds_to_queue(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) reporter._enqueue("POST", "/api/v1/test", {"key": "val"}) @@ -170,8 +168,7 @@ def test_enqueue_adds_to_queue(self, _mock_requests): assert event["method"] == "POST" assert event["endpoint"] == "/api/v1/test" - @patch("fluid_build.observability.reporter.requests") - def test_enqueue_drops_when_full(self, _mock_requests): + def test_enqueue_drops_when_full(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) reporter.queue = queue.Queue(maxsize=1) @@ -179,8 +176,7 @@ def test_enqueue_drops_when_full(self, _mock_requests): reporter._enqueue("POST", "/test2", {}) # should be dropped assert reporter.queue.qsize() == 1 - @patch("fluid_build.observability.reporter.requests") - def test_register_execution_with_git_info(self, _mock_requests): + def test_register_execution_with_git_info(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) reporter.register_execution( @@ -193,8 +189,7 @@ def test_register_execution_with_git_info(self, _mock_requests): assert event["payload"]["git_repo"] == "test" assert event["payload"]["git_commit"] == "abc123" - @patch("fluid_build.observability.reporter.requests") - def test_update_execution_payload(self, _mock_requests): + def test_update_execution_payload(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) reporter.update_execution( @@ -207,15 +202,13 @@ def test_update_execution_payload(self, _mock_requests): assert event["payload"]["status"] == "success" assert event["payload"]["progress"] == 100.0 - @patch("fluid_build.observability.reporter.requests") - def test_update_execution_empty_payload_not_enqueued(self, _mock_requests): + def test_update_execution_empty_payload_not_enqueued(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) reporter.update_execution(execution_id="abc") assert reporter.queue.empty() - @patch("fluid_build.observability.reporter.requests") - def test_send_event_post(self, _mock_requests): + def test_send_event_post(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) mock_response = MagicMock() @@ -227,8 +220,7 @@ def test_send_event_post(self, _mock_requests): reporter._send_event(event) reporter.session.post.assert_called_once() - @patch("fluid_build.observability.reporter.requests") - def test_send_event_patch(self, _mock_requests): + def test_send_event_patch(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) mock_response = MagicMock() @@ -239,8 +231,7 @@ def test_send_event_patch(self, _mock_requests): reporter._send_event(event) reporter.session.patch.assert_called_once() - @patch("fluid_build.observability.reporter.requests") - def test_send_event_unsupported_method_raises(self, _mock_requests): + def test_send_event_unsupported_method_raises(self): config = self._make_config(configured=True) reporter = CommandCenterReporter(config) reporter.session = MagicMock() diff --git a/tests/test_command_center_client.py b/tests/test_command_center_client.py index b7ba3b99e..81eed64ee 100644 --- a/tests/test_command_center_client.py +++ b/tests/test_command_center_client.py @@ -16,7 +16,9 @@ from unittest.mock import MagicMock, mock_open, patch +import httpx import pytest +import respx from fluid_build.cli._command_center import ( CommandCenterClient, @@ -97,40 +99,27 @@ def test_detect_url_from_config_file(self, mock_path_cls, _mock_check): @patch.object(CommandCenterClient, "_detect_url", return_value="http://localhost:8000") def test_check_availability_success(self, _mock_detect): - mock_health_response = MagicMock() - mock_health_response.status_code = 200 - - mock_head_response = MagicMock() - mock_head_response.status_code = 200 - - mock_version_response = MagicMock() - mock_version_response.status_code = 200 - mock_version_response.json.return_value = {"version": "2.0.0"} - - with ( - patch( - "fluid_build.cli._command_center.requests.get", - side_effect=[mock_health_response, mock_version_response], - ), - patch( - "fluid_build.cli._command_center.requests.head", - return_value=mock_head_response, - ), - ): + with respx.mock(assert_all_called=False) as router: + router.get("http://localhost:8000/health").respond(200) + router.head(url__regex=r"http://localhost:8000/api/.*").respond(200) + router.get("http://localhost:8000/api/v1/version").respond( + 200, json={"version": "2.0.0"} + ) client = CommandCenterClient() assert client.available is True @patch.object(CommandCenterClient, "_detect_url", return_value="http://localhost:8000") def test_check_availability_failure(self, _mock_detect): - with patch( - "fluid_build.cli._command_center.requests.get", - side_effect=Exception("connection refused"), - ): + with respx.mock() as router: + router.get("http://localhost:8000/health").mock( + side_effect=httpx.ConnectError("connection refused") + ) client = CommandCenterClient() assert client.available is False @patch.object(CommandCenterClient, "_detect_url", return_value="http://localhost:8000") - def test_check_availability_requests_not_available(self, _mock_detect): - with patch("fluid_build.cli._command_center.REQUESTS_AVAILABLE", False): + def test_check_availability_health_not_ok(self, _mock_detect): + with respx.mock() as router: + router.get("http://localhost:8000/health").respond(503) client = CommandCenterClient() assert client.available is False diff --git a/tests/test_command_center_client_gate.py b/tests/test_command_center_client_gate.py index 9115db7a6..249a0e235 100644 --- a/tests/test_command_center_client_gate.py +++ b/tests/test_command_center_client_gate.py @@ -32,8 +32,8 @@ from typing import Any, Iterator, List, Tuple from unittest.mock import patch +import httpx import pytest -import requests from fluid_build.cli._command_center import CommandCenterClient @@ -347,7 +347,7 @@ def test_a_blueprint_get_does_not_follow_a_redirect_to_another_origin( cc, other = cc_redirecting_its_data api_url = _marketplace_url() - with pytest.raises(requests.exceptions.RequestException) as raised: + with pytest.raises(httpx.HTTPError) as raised: marketplace._marketplace_get(f"{api_url}{path}", logging.getLogger("t")) assert ("GET", f"{_MARKETPLACE}{path}") in cc.recorded @@ -389,7 +389,7 @@ def test_the_instantiate_post_does_not_resend_the_parameters_to_another_origin( cc, other = cc_redirecting_its_data api_url = _marketplace_url() - with pytest.raises(requests.exceptions.RequestException): + with pytest.raises(httpx.HTTPError): marketplace._marketplace_post( f"{api_url}/customer-360/instantiate", logging.getLogger("t"), @@ -441,7 +441,7 @@ async def _run(): try: return await connector._get_catalog_stats_impl() finally: - await connector.session.close() + await connector.session.aclose() stats = asyncio.run(_run()) @@ -460,10 +460,10 @@ def test_every_request_on_the_connector_session_refuses_the_foreign_redirect( async def _run(): connector = await _connected_catalog_connector() try: - async with connector.session.get(f"{connector.base_url}/search") as response: - return response.status + response = await connector.session.get(f"{connector.base_url}/search") + return response.status_code finally: - await connector.session.close() + await connector.session.aclose() with pytest.raises(CredentialRedirectRefused, match="another origin"): asyncio.run(_run()) @@ -488,7 +488,7 @@ async def _run(): try: return await connector._get_catalog_stats_impl() finally: - await connector.session.close() + await connector.session.aclose() try: assert asyncio.run(_run()) == {"total_products": 7} @@ -499,8 +499,8 @@ async def _run(): # ── a refused hop is never sent: no request, not even a connection ─────── # # The refusal is raised from the hook each library runs on a redirect response -# before it follows it: aiohttp's ``on_request_redirect`` trace signal for the -# catalog connector's session, requests' response hook for the marketplace. +# before it follows it: the ``response`` event hook (``refuse_foreign_redirect``) +# of the catalog connector's httpx AsyncClient and of the marketplace's client. # The second origin must see nothing at all, also for a 307 / 308, which # resends the method and the body (the operator's parameters). @@ -554,12 +554,12 @@ async def _run(): connector = await _connected_catalog_connector() try: kwargs = {} if method == "GET" else {"json": {"parameters": _PARAMETERS}} - async with connector.session.request( + response = await connector.session.request( method, f"{connector.base_url}/instantiate", **kwargs - ) as response: - return response.status + ) + return response.status_code finally: - await connector.session.close() + await connector.session.aclose() with pytest.raises(CredentialRedirectRefused, match="another origin"): asyncio.run(_run()) @@ -579,10 +579,8 @@ def test_a_marketplace_request_sends_nothing_to_the_redirect_target( url = f"{_marketplace_url()}/customer-360/instantiate" kwargs = {} if method == "get" else {"json": {"parameters": _PARAMETERS}} - with pytest.raises(requests.exceptions.RequestException, match="another origin"): - marketplace._marketplace_request( - getattr(requests, method), url, logging.getLogger("t"), **kwargs - ) + with pytest.raises(httpx.HTTPError, match="another origin"): + marketplace._marketplace_request(method.upper(), url, logging.getLogger("t"), **kwargs) assert (method.upper(), f"{_MARKETPLACE}/customer-360/instantiate") in cc.recorded assert other.recorded == [] and other.bodies == [] @@ -593,7 +591,7 @@ def test_the_second_origin_is_reachable_so_the_zero_above_is_real(cc_redirecting # The counting listener does see a connection and a request sent to it. _cc, other, accepted, _status = cc_redirecting_every_hop - assert requests.post(f"{other.url}/collect", json={"x": 1}, timeout=5).status_code == 200 + assert httpx.post(f"{other.url}/collect", json={"x": 1}, timeout=5).status_code == 200 assert other.recorded == [("POST", "/collect")] assert len(accepted) == 1 @@ -758,7 +756,7 @@ def test_a_named_registry_request_with_a_header_of_its_own_refuses_another_origi old, new = moved_registry - with pytest.raises(requests.exceptions.RequestException, match="another origin"): + with pytest.raises(httpx.HTTPError, match="another origin"): marketplace._marketplace_get( f"{old.url}{_MARKETPLACE}/x", logging.getLogger("t"), @@ -775,9 +773,9 @@ def test_a_named_registry_request_with_a_body_refuses_another_origin(moved_regis old, new = moved_registry - with pytest.raises(requests.exceptions.RequestException, match="another origin"): + with pytest.raises(httpx.HTTPError, match="another origin"): marketplace._marketplace_request( - getattr(requests, method), + method.upper(), f"{old.url}{_MARKETPLACE}/x/instantiate", logging.getLogger("t"), named_registry=True, @@ -890,7 +888,7 @@ def test_a_named_registry_request_of_the_operators_still_stops_after_a_same_orig old, new = sticky_moved_registry - with pytest.raises(requests.exceptions.RequestException, match="another origin"): + with pytest.raises(httpx.HTTPError, match="another origin"): marketplace._marketplace_get( f"{old.url}{_MARKETPLACE}/x", logging.getLogger("t"), named_registry=True, **kwargs ) @@ -952,7 +950,7 @@ def test_a_named_registry_through_an_authenticated_proxy_follows_its_move(authen def test_a_named_registry_request_with_a_key_through_the_proxy_still_stops(authenticated_proxy): from fluid_build.cli import marketplace - with pytest.raises(requests.exceptions.RequestException, match="another origin"): + with pytest.raises(httpx.HTTPError, match="another origin"): marketplace._marketplace_get( f"{_PROXIED_OLD}{_MARKETPLACE}/x", logging.getLogger("t"), diff --git a/tests/test_command_center_ssrf.py b/tests/test_command_center_ssrf.py index fdb72bfa1..be319f3a5 100644 --- a/tests/test_command_center_ssrf.py +++ b/tests/test_command_center_ssrf.py @@ -27,6 +27,8 @@ from unittest.mock import MagicMock, patch +import respx + from fluid_build.cli._command_center import ( CommandCenterClient, ) @@ -152,31 +154,35 @@ def test_reporter_allowlist_opts_internal_back_in(self, monkeypatch): class TestClientCheckAvailabilityGate: def test_check_availability_refuses_metadata_url(self, monkeypatch): - """``_check_availability`` must bail before any ``requests`` - call when the configured URL is the metadata service.""" + """``_check_availability`` must bail before any HTTP request + when the configured URL is the metadata service.""" monkeypatch.setenv("FLUID_COMMAND_CENTER_URL", "http://169.254.169.254") - with patch("fluid_build.cli._command_center.requests") as mock_requests: + with respx.mock(assert_all_called=False) as router: + # A catch-all that answers 200: any attempted request is recorded + # (an unmocked request would be an error the client swallows, so + # ``router.calls`` would stay empty and prove nothing). + router.route().respond(200) client = CommandCenterClient() assert client.available is False # No HTTP probe should have been issued. - assert not mock_requests.get.called - assert not mock_requests.head.called + assert not router.calls def test_check_availability_refuses_rfc1918_url(self, monkeypatch): monkeypatch.setenv("FLUID_COMMAND_CENTER_URL", "http://10.0.0.9:8000") - with patch("fluid_build.cli._command_center.requests") as mock_requests: + with respx.mock(assert_all_called=False) as router: + router.route().respond(200) # records any attempted request client = CommandCenterClient() assert client.available is False - assert not mock_requests.get.called + assert not router.calls def test_check_availability_probes_localhost_default(self, monkeypatch): """Localhost (the default) is allowed — the probe IS issued.""" monkeypatch.delenv("FLUID_COMMAND_CENTER_URL", raising=False) monkeypatch.setenv("FLUID_DISABLE_CC_DETECTION", "false") - with patch("fluid_build.cli._command_center.requests") as mock_requests: - mock_requests.get.return_value = MagicMock(status_code=500) - mock_requests.exceptions = __import__("requests").exceptions # real exception classes + with respx.mock(assert_all_called=False) as router: + health = router.get(url__regex=r"http://[^/]+/health").respond(500) client = CommandCenterClient() # Localhost default must reach the probe (status 500 → unavailable, # but the request WAS attempted, proving the gate let it through). - assert mock_requests.get.called + assert health.called + assert client.available is False diff --git a/tests/test_datamesh_manager_publish_spec.py b/tests/test_datamesh_manager_publish_spec.py index 1d5084047..f0f9d6b6b 100644 --- a/tests/test_datamesh_manager_publish_spec.py +++ b/tests/test_datamesh_manager_publish_spec.py @@ -15,8 +15,9 @@ from types import SimpleNamespace from unittest.mock import MagicMock, patch +import httpx import pytest -import requests +import respx from fluid_build.cli.datamesh_manager import _cmd_publish, _publish_exit_code from fluid_build.providers.base import ProviderError @@ -704,14 +705,14 @@ def test_cmd_publish_fail_on_contract_error_returns_non_zero(): def test_request_wraps_retry_error_as_provider_error(): provider = DataMeshManagerProvider(api_key="dummy", api_url="https://api.entropy-data.com") - session = MagicMock() - session.request.side_effect = requests.exceptions.RetryError("too many 500 responses") - provider._session_instance = session - with pytest.raises(ProviderError) as excinfo: - provider._request("PUT", "/api/datacontracts/x.y", json_body={"id": "x.y"}) + with respx.mock() as router: + route = router.put("https://api.entropy-data.com/api/datacontracts/x.y").respond(500) + with pytest.raises(ProviderError) as excinfo: + provider._request("PUT", "/api/datacontracts/x.y", json_body={"id": "x.y"}) assert "HTTP request failed" in str(excinfo.value) + assert route.call_count > 1 # the 500s were retried before giving up def test_provider_rejects_plain_http_remote_api_url(): @@ -723,13 +724,13 @@ def test_provider_rejects_plain_http_remote_api_url(): def test_request_redacts_secret_like_error_body(): provider = DataMeshManagerProvider(api_key="dummy", api_url="https://api.entropy-data.com") - response = MagicMock(status_code=403, text='{"api_key":"ed_live_supersecret"}') - session = MagicMock() - session.request.return_value = response - provider._session_instance = session - with pytest.raises(ProviderError) as excinfo: - provider._request("PUT", "/api/dataproducts/x", json_body={"id": "x"}) + with respx.mock() as router: + router.put("https://api.entropy-data.com/api/dataproducts/x").mock( + return_value=httpx.Response(403, text='{"api_key":"ed_live_supersecret"}') + ) + with pytest.raises(ProviderError) as excinfo: + provider._request("PUT", "/api/dataproducts/x", json_body={"id": "x"}) message = str(excinfo.value) assert "ed_live_supersecret" not in message diff --git a/tests/test_diff_changelog.py b/tests/test_diff_changelog.py index 56ab7b21a..f4e03f6cc 100644 --- a/tests/test_diff_changelog.py +++ b/tests/test_diff_changelog.py @@ -45,7 +45,12 @@ @pytest.fixture def v1_v2(): - """The on-disk fixture pair loaded as parsed contracts.""" + """The on-disk fixture pair loaded as parsed contracts. + + They declare 0.7.3 but keep the pre-0.7 shapes the legacy rules read (a + flat ``schema``, ``quality.tests``, a root ``agentPolicy``), which no + bundled schema accepts, so they are diffed with ``validate=False``. + """ import yaml v1 = yaml.safe_load((FIXTURES / "v1.fluid.yaml").read_text()) @@ -60,14 +65,14 @@ def v1_v2(): def test_same_contract_produces_empty_diff(v1_v2): v1, _ = v1_v2 - report = compare_contracts(v1, v1) + report = compare_contracts(v1, v1, validate=False) assert report.total == 0 assert not report.has_breaking def test_report_total_matches_section_sums(v1_v2): v1, v2 = v1_v2 - report = compare_contracts(v1, v2) + report = compare_contracts(v1, v2, validate=False) assert report.total == (len(report.breaking) + len(report.non_breaking) + len(report.info)) @@ -254,7 +259,7 @@ def test_quality_severity_unchanged_no_change(): def test_fixture_v1_to_v2_classification(v1_v2): """Full pipeline against the on-disk fixture pair.""" v1, v2 = v1_v2 - report = compare_contracts(v1, v2) + report = compare_contracts(v1, v2, validate=False) # v2 vs v1 should detect: # - email column removed (breaking) @@ -296,7 +301,7 @@ def test_fixture_v1_to_v2_classification(v1_v2): def test_render_text_contains_summary_line(v1_v2): v1, v2 = v1_v2 - report = compare_contracts(v1, v2) + report = compare_contracts(v1, v2, validate=False) out = render_text(report) assert "Summary:" in out assert "breaking" in out.lower() @@ -309,7 +314,7 @@ def test_render_text_empty_report_says_no_changes(): def test_render_markdown_includes_section_headings(v1_v2): v1, v2 = v1_v2 - report = compare_contracts(v1, v2) + report = compare_contracts(v1, v2, validate=False) md = render_markdown(report) assert "# Contract changelog" in md if report.breaking: @@ -320,7 +325,7 @@ def test_render_markdown_includes_section_headings(v1_v2): def test_report_to_dict_round_trips_through_json(v1_v2): v1, v2 = v1_v2 - report = compare_contracts(v1, v2) + report = compare_contracts(v1, v2, validate=False) data = report.to_dict() # Must be JSON-serialisable s = json.dumps(data) @@ -334,14 +339,26 @@ def test_report_to_dict_round_trips_through_json(v1_v2): # --------------------------------------------------------------------------- +def _cli_contracts(tmp_path): + """A valid 0.7.6 pair for the CLI, which validates both: ``new`` drops a column.""" + import yaml + + old = tmp_path / "old.fluid.yaml" + new = tmp_path / "new.fluid.yaml" + old.write_text(yaml.safe_dump(_contract_076([_ID_COLUMN, {"name": "note", "type": "string"}]))) + new.write_text(yaml.safe_dump(_contract_076([_ID_COLUMN]))) + return str(old), str(new) + + def test_cli_diff_baseline_mode_smoke(tmp_path, monkeypatch): """End-to-end: ``fluid diff --baseline `` produces JSON and exits 0 by default.""" from fluid_build.cli import diff as diff_mod + old, new = _cli_contracts(tmp_path) out_path = tmp_path / "diff.json" args = argparse.Namespace( - contract=str(FIXTURES / "v2.fluid.yaml"), - baseline=str(FIXTURES / "v1.fluid.yaml"), + contract=new, + baseline=old, env=None, state=None, out=str(out_path), @@ -365,9 +382,10 @@ def test_cli_diff_baseline_fail_on_breaking(tmp_path): """``--fail-on-breaking`` returns 1 when breaking changes exist.""" from fluid_build.cli import diff as diff_mod + old, new = _cli_contracts(tmp_path) args = argparse.Namespace( - contract=str(FIXTURES / "v2.fluid.yaml"), - baseline=str(FIXTURES / "v1.fluid.yaml"), + contract=new, + baseline=old, env=None, state=None, out=str(tmp_path / "diff.json"), @@ -384,9 +402,10 @@ def test_cli_diff_baseline_idempotent(tmp_path): """``--fail-on-breaking`` returns 0 when comparing a contract to itself.""" from fluid_build.cli import diff as diff_mod + old, _ = _cli_contracts(tmp_path) args = argparse.Namespace( - contract=str(FIXTURES / "v1.fluid.yaml"), - baseline=str(FIXTURES / "v1.fluid.yaml"), + contract=old, + baseline=old, env=None, state=None, out=str(tmp_path / "diff.json"), @@ -860,14 +879,14 @@ def test_moving_columns_from_schema_to_contract_schema_is_not_a_change(): old = _contract_flat([{"name": "id", "type": "STRING", "nullable": False}]) new = _contract_076([{"name": "id", "type": "string", "required": True}]) - assert compare_contracts(old, new).total == 0 + assert compare_contracts(old, new, validate=False).total == 0 def test_a_column_dropped_while_moving_to_contract_schema_is_still_found(): old = _contract_flat([{"name": "id", "type": "STRING"}, {"name": "note", "type": "STRING"}]) new = _contract_076([{"name": "id", "type": "string"}]) - report = compare_contracts(old, new) + report = compare_contracts(old, new, validate=False) assert report.total == 1 assert [(c.kind, c.path) for c in report.breaking] == [ @@ -937,9 +956,10 @@ def _fake_traced_span(name, attributes=None): monkeypatch.setattr(diff_mod, "_traced_span", _fake_traced_span) + old, new = _cli_contracts(tmp_path) args = argparse.Namespace( - contract=str(FIXTURES / "v2.fluid.yaml"), - baseline=str(FIXTURES / "v1.fluid.yaml"), + contract=new, + baseline=old, env=None, state=None, out=str(tmp_path / "diff.json"), diff --git a/tests/test_diff_changelog_hardening.py b/tests/test_diff_changelog_hardening.py new file mode 100644 index 000000000..6ed160dce --- /dev/null +++ b/tests/test_diff_changelog_hardening.py @@ -0,0 +1,821 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``fluid diff --baseline``: what reaches the gate, how it is printed, what 0.7.x carries. + +Once 0.7.x columns reached ``--fail-on-breaking`` (PR #789), its reviews found +four gaps. The diff ran on contracts no schema accepts, so a value of the wrong +type hid a change. It printed contract names raw to a terminal and to Markdown. +It ignored the quality, sensitivity and primary-key information a 0.7.x contract +keeps under ``exposes[].contract``. And it read no columns from a +``{"fields": [...]}`` schema. +""" + +from __future__ import annotations + +import argparse +import json +import logging + +import pytest +import yaml + +from fluid_build.api.changelog import ( + ContractValidationError, + compare_contracts, + render_markdown, + render_text, +) +from fluid_build.api.changelog_rules import diff_columns, diff_quality_severity +from fluid_build.api.changelog_types import Change, ChangelogReport +from tests.test_diff_changelog import _ID_COLUMN as _ID +from tests.test_diff_changelog import _contract_076 + + +def _contract(columns, **section): + """``_contract_076`` with ``section`` (``dq``, ``quality``) beside the schema too.""" + contract = _contract_076(columns) + contract["exposes"][0]["contract"].update(section) + return contract + + +def _run_cli(tmp_path, old, new, *, fmt="json", fail_on_breaking=False): + """``fluid diff --baseline `` on two contracts written to ``tmp_path``.""" + from fluid_build.cli import diff as diff_mod + + old_path, new_path = tmp_path / "old.fluid.yaml", tmp_path / "new.fluid.yaml" + old_path.write_text(yaml.safe_dump(old), encoding="utf-8") + new_path.write_text(yaml.safe_dump(new), encoding="utf-8") + args = argparse.Namespace( + contract=str(new_path), + baseline=str(old_path), + env=None, + state=None, + out=str(tmp_path / "diff.json"), + exit_on_drift=False, + fail_on_breaking=fail_on_breaking, + format=fmt, + cmd="diff", + ) + return diff_mod.run(args, logging.getLogger("test.diff")) + + +# --------------------------------------------------------------------------- +# Both contracts are schema-validated before they are diffed +# --------------------------------------------------------------------------- + + +def test_every_contract_below_validates_against_the_bundled_schema(): + """The shapes these tests diff: if 0.7.6 stops accepting them they prove nothing.""" + from fluid_build.schema_manager import FluidSchemaManager + + shapes = [ + _contract([_ID]), + _contract([{**_ID, "tags": ["primary-key"]}, {"name": "email", "type": "string"}]), + _contract([{"name": "email", "type": "string", "sensitivity": "pii"}]), + _contract( + [ + { + "name": "email", + "type": "string", + "quality": [{"id": "email-valid", "type": "text", "severity": "warning"}], + } + ], + dq={"rules": [{"id": "complete", "type": "completeness", "severity": "critical"}]}, + quality=[{"rule": "positive", "expression": "amount > 0", "severity": "warning"}], + ), + ] + for shape in shapes: + result = FluidSchemaManager().validate_contract(shape, offline_only=True) + assert result.is_valid, result.errors + + +def test_a_value_of_the_wrong_type_cannot_hide_a_change_from_the_diff(): + """A ``type`` that is a number was never compared, so the retype passed the gate.""" + old = _contract([_ID, {"name": "amount", "type": "decimal(18,2)"}]) + new = _contract([_ID, {"name": "amount", "type": 18}]) + + with pytest.raises(ContractValidationError) as excinfo: + compare_contracts(old, new) + + assert set(excinfo.value.errors) == {"new"} + assert any("exposes[0].contract.schema[1].type" in e for e in excinfo.value.errors["new"]) + + +def test_every_invalid_side_is_reported_at_once(): + old = _contract([_ID]) + del old["metadata"]["owner"] + new = _contract([{"name": "id"}]) + + with pytest.raises(ContractValidationError) as excinfo: + compare_contracts(old, new) + + assert set(excinfo.value.errors) == {"baseline", "new"} + assert "baseline contract" in str(excinfo.value) and "new contract" in str(excinfo.value) + + +@pytest.mark.parametrize("version", ["0.6.0", "0.5.7", "0.4"]) +def test_a_pre_07_contract_is_refused_with_the_upgrade_message(version): + old = _contract([_ID]) + old["fluidVersion"] = version + + with pytest.raises(ContractValidationError) as excinfo: + compare_contracts(old, _contract([_ID])) + + (message,) = excinfo.value.errors["baseline"] + assert "no longer supported" in message and "0.7.x" in message + + +def test_a_contract_without_a_fluid_version_is_refused(): + new = _contract([_ID]) + del new["fluidVersion"] + + with pytest.raises(ContractValidationError) as excinfo: + compare_contracts(_contract([_ID]), new) + + assert set(excinfo.value.errors) == {"new"} + + +@pytest.mark.parametrize("version", ["garbage", 7]) +def test_a_fluid_version_that_is_not_a_version_is_refused_not_raised(tmp_path, version): + """The validator raised on it, and the CLI exited 1 (``diff_failed``), which reads as breaking.""" + from fluid_build.cli._common import CLIError + + new = _contract([_ID]) + new["fluidVersion"] = version + + with pytest.raises(ContractValidationError) as excinfo: + compare_contracts(_contract([_ID]), new) + assert set(excinfo.value.errors) == {"new"} + with pytest.raises(CLIError) as cli_error: + _run_cli(tmp_path, _contract([_ID]), new, fail_on_breaking=True) + assert (cli_error.value.exit_code, cli_error.value.event) == (2, "diff_contract_invalid") + + +def test_the_cli_names_the_baseline_when_only_the_baseline_is_invalid(tmp_path): + from fluid_build.cli._common import CLIError + + old = _contract([_ID, {"name": "amount", "type": 18}]) + + with pytest.raises(CLIError) as excinfo: + _run_cli(tmp_path, old, _contract([_ID])) + + assert excinfo.value.exit_code == 2 + assert excinfo.value.context["invalid"] == ["baseline"] + assert str(tmp_path / "old.fluid.yaml") in excinfo.value.context["message"] + + +def test_validate_false_diffs_what_it_is_given(): + """The refine preview's opt-out: an advisory diff of a contract still being repaired.""" + old = {"exposes": [{"id": "x", "schema": [{"name": "a", "type": "STRING"}]}]} + new = {"exposes": [{"id": "x", "schema": []}]} + + report = compare_contracts(old, new, validate=False) + + assert [c.kind for c in report.breaking] == ["column_removed"] + + +@pytest.mark.parametrize("fail_on_breaking", [True, False]) +def test_the_cli_exits_2_on_an_invalid_contract_whatever_fail_on_breaking_says( + tmp_path, fail_on_breaking +): + """diff(1)'s 2 is "trouble": an invalid contract never reads as 0 (clean) or 1 (breaking).""" + from fluid_build.cli._common import CLIError + + old = _contract([_ID, {"name": "amount", "type": "decimal(18,2)"}]) + new = _contract([_ID, {"name": "amount", "type": 18}]) + + with pytest.raises(CLIError) as excinfo: + _run_cli(tmp_path, old, new, fail_on_breaking=fail_on_breaking) + + assert excinfo.value.exit_code == 2 + assert excinfo.value.event == "diff_contract_invalid" + context = excinfo.value.context + assert context["invalid"] == ["new"] + assert str(tmp_path / "new.fluid.yaml") in context["message"] + assert "fluid validate" in context["message"] + assert not (tmp_path / "diff.json").exists() + + +# --------------------------------------------------------------------------- +# Names are printed as data: escaped for a terminal and inert in Markdown +# --------------------------------------------------------------------------- + + +def _report(*names): + """A report naming each of ``names`` as an added column, as ``diff_columns`` words it.""" + report = ChangelogReport() + for name in names: + report.add( + Change( + path=f"exposes[0].contract.schema.{name}", + kind="column_added", + severity="non_breaking", + description=f"column '{name}' added to expose 'orders'", + ) + ) + return report + + +@pytest.mark.parametrize( + "name, spelled", + [ + pytest.param( + "x\x1b]8;;https://evil.example\x07click\x1b]8;;\x07", + "x\\x1b]8;;https://evil.example\\x07click\\x1b]8;;\\x07", + id="osc8-hyperlink", + ), + pytest.param("x\x1b[2J", "x\\x1b[2J", id="clear-screen"), + pytest.param("x\x9b31m", "x\\x9b31m", id="c1-csi"), + pytest.param("a\rb", "a\\x0db", id="carriage-return"), + pytest.param("a\tb", "a\\x09b", id="tab"), + pytest.param("admin\u202egnp.exe", "admin\\u202egnp.exe", id="bidi-override"), + pytest.param("e\u200bmail", "e\\u200bmail", id="zero-width-space"), + pytest.param("e\ufe0fmail", "e\\ufe0fmail", id="variation-selector"), + pytest.param("\u3164", "\\u3164", id="hangul-filler"), + pytest.param("\u2800", "\\u2800", id="braille-blank"), + pytest.param("x\U000e0041", "x\\U000e0041", id="tag-character"), + pytest.param("a\\x1b", "a\\\\x1b", id="literal-backslash-stays-distinct"), + pytest.param("straße_日本", "straße_日本", id="printable-unicode-is-kept"), + ], +) +def test_render_text_spells_out_what_a_terminal_would_act_on(name, spelled): + out = render_text(_report(name)) + + assert f"column '{spelled}' added" in out + assert f"exposes[0].contract.schema.{spelled}:" in out + assert all(ch == "\n" or ch.isprintable() for ch in out) + + +def test_a_newline_in_a_name_cannot_forge_a_line(): + out = render_text(_report("a\nSummary: 0 breaking, 0 non-breaking, 0 info")) + + assert [line for line in out.splitlines() if line.startswith("Summary:")] == [ + "Summary: 0 breaking, 1 non-breaking, 0 info" + ] + assert "a\\x0aSummary: 0 breaking" in out + + +def _html(markdown): + from markdown_it import MarkdownIt + + return MarkdownIt("commonmark").enable(["table", "strikethrough"]).render(markdown) + + +@pytest.mark.parametrize( + "name", + [ + pytest.param("