diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 62f7cf6..3c36a3c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,7 +79,7 @@ When the drift check fails, or forge-cli releases a new schema version ```bash COMMIT=... # the commit from step 1, which is also upstream.commit - for v in 0.7.5 0.7.6; do + for v in 0.7.6 0.7.7; do curl -fsSL "https://raw.githubusercontent.com/Agenticstiger/forge-cli/$COMMIT/fluid_build/schemas/fluid-schema-$v.json" \ -o "schema/fluid-schema-$v.json" done diff --git a/GENERATOR_DOCS.md b/GENERATOR_DOCS.md index 8132f9b..95ca239 100644 --- a/GENERATOR_DOCS.md +++ b/GENERATOR_DOCS.md @@ -48,7 +48,7 @@ pip install --require-hashes -r scripts/requirements-docs.lock python3 generate-docs.py # every synced version python3 generate-docs.py --all # the same, spelled out -python3 generate-docs.py --version 0.7.6 # one version; repeat the flag for more +python3 generate-docs.py --version 0.7.7 # one version; repeat the flag for more python3 generate-docs.py --check # render to a temporary directory and # exit 1 if specs/ differs, or if a # file under schema/ or specs/ is not @@ -76,8 +76,8 @@ pip-compile --generate-hashes --strip-extras \ --output-file=scripts/requirements-docs.lock scripts/requirements-docs.txt ``` -A preview's page is labelled by its schema's own `title`; for 0.7.6 that reads -"FLUID 0.7.6 β€” Declarative Packaging Modes (preview)". +A preview's page is labelled by its schema's own `title`; for 0.7.7 that reads +"FLUID 0.7.7 β€” Preview: recursive streaming types". `--version` also renders a version that is not synced, if its schema is in `schema/`. The pages for versions before 0.7.2 were rendered by earlier diff --git a/README.md b/README.md index e7cc494..60e391b 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ 🧭 Schema Anatomy Β· πŸš€ Examples Β· πŸ”„ vs ODCS / ODPS Β· - ✨ What's New + ✨ What's New

--- @@ -23,7 +23,7 @@ FLUID is one YAML file that describes a data product end to end β€” **schema, bu ```mermaid flowchart LR - F["FLUID v0.7.5
your .fluid.yml
complete on its own"] + F["FLUID v0.7.6
your .fluid.yml
complete on its own"] FC["forge-cli
compiler β†’ IaC + DAGs"] BIT["Bitol ODPS + ODCS
catalog interop"] F -.->|compile + deploy| FC @@ -35,7 +35,7 @@ FLUID is **standalone** β€” the contract is complete on its own. Everything else ## Minimal valid contract ```yaml -fluidVersion: "0.7.5" +fluidVersion: "0.7.6" kind: DataProduct id: demo.bronze.hello_world name: Hello World @@ -61,16 +61,16 @@ exposes: ## JSON Schema -- **Latest stable: 0.7.5** β€” `https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.5.json` -- **Preview: 0.7.6** β€” `https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json`. A contract uses it only by declaring `fluidVersion: "0.7.6"`, and it can still change before it is promoted to stable. See **[Stable and preview versions](https://open-data-protocol.github.io/fluid/schema/versions#stable-and-preview-versions)** and **[What's new in 0.7.6](https://open-data-protocol.github.io/fluid/releases/0.7.6)**. +- **Latest stable: 0.7.6** β€” `https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json` +- **Preview: 0.7.7** β€” `https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json`. A contract uses it only by declaring `fluidVersion: "0.7.7"`, and it can still change before it is promoted to stable. See **[Stable and preview versions](https://open-data-protocol.github.io/fluid/schema/versions#stable-and-preview-versions)** and **[What's new in 0.7.7](https://open-data-protocol.github.io/fluid/releases/0.7.7)**. - A document is validated against the schema of the `fluidVersion` it declares. To use a field, declare the version that added it β€” see **[Choosing `fluidVersion`](https://open-data-protocol.github.io/fluid/schema/versions#choosing-fluidversion)**. -- Editor support β€” add this line to a file that declares `fluidVersion: "0.7.5"` (use the URL matching the version the file declares): +- Editor support β€” add this line to a file that declares `fluidVersion: "0.7.6"` (use the URL matching the version the file declares): ```yaml - # yaml-language-server: $schema=https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.5.json + # yaml-language-server: $schema=https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json ``` - All versions, diffs, and the generated HTML reference: **[Schema β†’ Versions](https://open-data-protocol.github.io/fluid/schema/versions)**. -0.7.5 is additive over 0.7.4: every valid 0.7.4 contract stays valid when it declares `"0.7.5"`. See **[What's New in 0.7.5](https://open-data-protocol.github.io/fluid/releases/0.7.5)**. +0.7.6 is additive over 0.7.5: every valid 0.7.5 contract stays valid when it declares `"0.7.6"`. See **[What's New in 0.7.6](https://open-data-protocol.github.io/fluid/releases/0.7.6)**. ## Validate a contract diff --git a/docs/.vuepress/config.ts b/docs/.vuepress/config.ts index a6b0134..9cc39f1 100644 --- a/docs/.vuepress/config.ts +++ b/docs/.vuepress/config.ts @@ -87,9 +87,9 @@ export default defineUserConfig({ { text: 'Specification', link: '/schema/specification' }, { text: 'Versions', link: '/schema/versions' }, { text: 'Changelog', link: '/schema/changelog' }, - { text: 'JSON Schema 0.7.5 (stable) β†—', link: 'https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.5.json', target: '_blank' }, - { text: 'Reference 0.7.5 (HTML) β†—', link: 'https://open-data-protocol.github.io/fluid/specs/0.7.5/fluid-spec.html', target: '_blank' }, - { text: 'Preview: 0.7.6', link: '/releases/0.7.6' }, + { text: 'JSON Schema 0.7.6 (stable) β†—', link: 'https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json', target: '_blank' }, + { text: 'Reference 0.7.6 (HTML) β†—', link: 'https://open-data-protocol.github.io/fluid/specs/0.7.6/fluid-spec.html', target: '_blank' }, + { text: 'Preview: 0.7.7', link: '/releases/0.7.7' }, ], }, { text: 'Examples', link: '/examples/' }, @@ -175,6 +175,7 @@ export default defineUserConfig({ text: "What's New", children: [ '/releases/README.md', + '/releases/0.7.7.md', '/releases/0.7.6.md', '/releases/0.7.5.md', '/releases/0.7.4.md', diff --git a/docs/README.md b/docs/README.md index cedbf42..fb39235 100644 --- a/docs/README.md +++ b/docs/README.md @@ -36,7 +36,7 @@ FLUID is one YAML file that describes a data product end to end β€” **schema, bu ## The shape of a contract ```yaml -fluidVersion: "0.7.5" +fluidVersion: "0.7.6" kind: DataProduct id: demo.bronze.hello_world name: Hello World @@ -65,7 +65,7 @@ exposes: - **[Guide](/fluid/guide/)** β€” what FLUID is, the quickstart, and the FAQ. - **[Concepts](/fluid/concepts/)** β€” the agentic-native layer and how FLUID compares to ODCS / ODPS. - **[Schema Reference](/fluid/schema/anatomy)** β€” every top-level block, a cheatsheet, and the full specification. -- **[What's New in 0.7.5](/fluid/releases/0.7.5)** β€” the latest stable version: streaming Kafka β†’ Iceberg sink, Confluent Cloud Tableflow, and a pgvector output port. -- **[0.7.6 (preview)](/fluid/releases/0.7.6)** β€” packaging modes, declared consumers, cross-mesh pins, row filters and masked columns, Flink builds; opt-in, and still subject to change. +- **[What's New in 0.7.6](/fluid/releases/0.7.6)** β€” the latest stable version: packaging modes, declared consumers, cross-mesh pins, row filters and masked columns, Flink builds, a continuous trigger, and the Bitol-shaped team, description, contract-id, serialization and quality fields. +- **[0.7.7 (preview)](/fluid/releases/0.7.7)** β€” nested columns, and the Kafka stream and record mapping an embedded-logic build can read; opt-in, and still subject to change. - **[See the deck](/fluid/deck/)** β€” the FLUID story in slides. - **[Contributing & governance](/fluid/contributing/)** β€” how the standard is maintained, and how to take part. diff --git a/docs/concepts/README.md b/docs/concepts/README.md index 4a4f107..c647ced 100644 --- a/docs/concepts/README.md +++ b/docs/concepts/README.md @@ -35,7 +35,7 @@ A FLUID contract describes a data product end to end: its schema, build logic, d This structure separates **interface** (what you get) from **implementation** (how it's built), enabling reliable data ecosystems ready for both humans and AI agents. -> The latest stable schema version is **0.7.5**; **0.7.6** is a preview that a contract uses only by declaring it. Every stable release after 0.7.2 has been additive over its predecessor; the one recorded narrowing is the step into 0.7.2 (0.7.1 β†’ 0.7.2). See [What's New](/fluid/releases/) and [Versions](/fluid/schema/versions). +> The latest stable schema version is **0.7.6**; **0.7.7** is a preview that a contract uses only by declaring it. Every stable release after 0.7.2 has been additive over its predecessor; the one recorded narrowing is the step into 0.7.2 (0.7.1 β†’ 0.7.2). See [What's New](/fluid/releases/) and [Versions](/fluid/schema/versions). --- diff --git a/docs/concepts/comparisons.md b/docs/concepts/comparisons.md index 66ccb15..e516b5e 100644 --- a/docs/concepts/comparisons.md +++ b/docs/concepts/comparisons.md @@ -23,7 +23,7 @@ Four active "open data product" specs share overlapping names and adjacent scope | **ODCS** (Open Data Contract Standard) | LF AI & Data Β· Bitol | v3.1.0 β€” Dec 2025 | Column-level technical contract; producer↔consumer agreement for one dataset | | **Bitol ODPS** (Open Data Product Standard) | LF AI & Data Β· Bitol | v1.0.0 β€” Sep 2025 | Thin product manifest; bundles ODCS contracts via `contractId` on input/output ports | | **ODPS v4** (Open Data Product Specification) | LF Β· [Open-Data-Product-Initiative](https://github.com/Open-Data-Product-Initiative/v4.0) | v4.0 β€” Jul 2025 Β· v4.1 β€” Oct 2025 | Business + commercial wrapper: pricing Β· license Β· multi-language Β· marketplace | -| **FLUID** | open-data-protocol | v0.7.5 (stable) β€” this repo | End-to-end operational contract: schema + build + orchestration + agentic governance + sovereignty + semantics | +| **FLUID** | open-data-protocol | v0.7.6 (stable) β€” this repo | End-to-end operational contract: schema + build + orchestration + agentic governance + sovereignty + semantics | ## How they actually fit together @@ -34,7 +34,7 @@ flowchart LR classDef core fill:#5B8DEF,color:#fff,stroke:#1E3A8A,stroke-width:4px,font-weight:bold classDef opt fill:#94A3B8,color:#fff,stroke:#475569,stroke-width:1px,stroke-dasharray:6 3 - F["FLUID v0.7.5
your .fluid.yml
standalone β€” complete on its own"]:::core + F["FLUID v0.7.6
your .fluid.yml
standalone β€” complete on its own"]:::core FC["forge-cli
reference implementation
(emits IaC + Airflow DAGs)"]:::opt BIT["Bitol ODPS + ODCS
(catalog interop)"]:::opt @@ -62,7 +62,7 @@ flowchart LR ## πŸ“Š Capability matrix -Legend: βœ… deterministic in spec Β· ⚠️ partial Β· ❌ silent. Headers abbreviated for width: **F** = FLUID v0.7.5 Β· **ODCS** = Bitol ODCS v3.1 Β· **ODPS** = Bitol ODPS v1.0 Β· **v4** = ODPS v4.0. Field-level detail lives in the [**Schema Cheatsheet**](/fluid/schema/cheatsheet) β€” this matrix is the at-a-glance scoreboard. +Legend: βœ… deterministic in spec Β· ⚠️ partial Β· ❌ silent. Headers abbreviated for width: **F** = FLUID v0.7.6 Β· **ODCS** = Bitol ODCS v3.1 Β· **ODPS** = Bitol ODPS v1.0 Β· **v4** = ODPS v4.0. Field-level detail lives in the [**Schema Cheatsheet**](/fluid/schema/cheatsheet) β€” this matrix is the at-a-glance scoreboard. ### πŸ“ Data shape & quality diff --git a/docs/concepts/forge-cli.md b/docs/concepts/forge-cli.md index aa22a4d..9055a04 100644 --- a/docs/concepts/forge-cli.md +++ b/docs/concepts/forge-cli.md @@ -21,11 +21,11 @@ fluid --version ## Which FLUID versions it supports -The JSON Schemas are authored in forge-cli and published here (see [CONTRIBUTING.md](https://github.com/open-data-protocol/fluid/blob/main/CONTRIBUTING.md#the-schemas-are-not-authored-here)). As of data-product-forge 0.18.1: +The JSON Schemas are authored in forge-cli and published here (see [CONTRIBUTING.md](https://github.com/open-data-protocol/fluid/blob/main/CONTRIBUTING.md#the-schemas-are-not-authored-here)). As of data-product-forge 0.27.0: -- It **bundles** the schemas for 0.7.1 to 0.7.6 and validates a contract against the schema of the `fluidVersion` it declares (`fluid validate --list-versions` lists them). -- **0.7.5 is its default** β€” the newest stable version β€” and `fluid init --quickstart` writes `fluidVersion: 0.7.5`. -- **0.7.6 is a preview**: validated when a contract declares it, never chosen by default. See [Stable and preview versions](/fluid/schema/versions#stable-and-preview-versions). +- It **bundles** the schemas for 0.7.1 to 0.7.7 and validates a contract against the schema of the `fluidVersion` it declares (`fluid validate --list-versions` lists them). +- **0.7.6 is its default** β€” the newest stable version, and the one the contracts it builds declare. (`fluid init --quickstart` still writes its static customer-360 example at `fluidVersion: 0.7.5`.) +- **0.7.7 is a preview**: validated when a contract declares it, never chosen by default. See [Stable and preview versions](/fluid/schema/versions#stable-and-preview-versions). - For an older version it does not bundle, it fetches the schema from this repository and caches it (`--offline` restricts it to bundled and cached schemas). - Its bundled 0.7.1 is a different document from the published 0.7.1 under the same `$id`; see [Versions](/fluid/schema/versions#_0-7-1-two-documents-share-one-id). diff --git a/docs/examples/README.md b/docs/examples/README.md index 3b13222..ddc333f 100644 --- a/docs/examples/README.md +++ b/docs/examples/README.md @@ -1,6 +1,6 @@ # FLUID by Example β€” From "Hello World" to Production -> Examples 1–13 target **`fluidVersion: "0.7.5"`**, the [latest stable schema](/fluid/schema/fluid-schema-0.7.5.json). Example 14 uses the **0.7.6 preview**. +> Examples 1–13 target **`fluidVersion: "0.7.5"`**, which every later schema still accepts. Example 14 uses **0.7.6**, the [latest stable schema](/fluid/schema/fluid-schema-0.7.6.json). > Each example **adds one block** to the previous one. Read in order to learn the schema by accretion. > Every complete contract on this page validates against the JSON Schema of the version it declares (Draft 2020-12) and with the reference implementation's `fluid validate`; examples that show only the new block validate when merged into the example before them. > File names such as `01-minimal.fluid.yml` are illustrative: FLUID does not prescribe a file name, and the reference implementation scaffolds `contract.fluid.yaml`. @@ -26,7 +26,7 @@ | 11 | [Agent-consumable output port (MCP)](#_11-agent-consumable-output-port-mcp) | `exposes[].mcp` (⭐ 0.7.4) | | 12 | [Stream Kafka into Iceberg](#_12-stream-kafka-into-iceberg) | `kafka-connect` Iceberg streaming sink + Iceberg catalog location (⭐ 0.7.5) | | 13 | [Vector output port (pgvector)](#_13-vector-output-port-pgvector) | `platform: pgvector` + `vectorConfig` (⭐ 0.7.5) | -| 14 | [Preview: packaging and consumers](#_14-preview-packaging-and-declared-consumers-0-7-6) | `packaging`, `consumers` (πŸ§ͺ 0.7.6 preview) | +| 14 | [Packaging and consumers](#_14-packaging-and-declared-consumers-0-7-6) | `packaging`, `consumers` (πŸ§ͺ 0.7.6) | --- @@ -741,13 +741,13 @@ Which columns are embedded is not part of the schema; the reference implementati --- -## 14. Preview: packaging and declared consumers (0.7.6) +## 14. Packaging and declared consumers (0.7.6) -> πŸ§ͺ **0.7.6 is a preview.** It is opt-in β€” declare `fluidVersion: "0.7.6"` β€” and can still change before it is promoted. See [0.7.6 (preview)](/fluid/releases/0.7.6). +> πŸ§ͺ **New in 0.7.6.** Declare `fluidVersion: "0.7.6"` to use it. See [0.7.6](/fluid/releases/0.7.6). > **New in this step:** top-level `packaging` and `consumers`. ```yaml -# 14-preview-packaging.fluid.yml +# 14-packaging.fluid.yml fluidVersion: "0.7.6" kind: DataProduct id: finance.gold.revenue @@ -777,7 +777,7 @@ exposes: location: { account: acme-eu, database: FINANCE, schema: GOLD, table: REVENUE } ``` -The 0.7.6 release note covers the rest of the preview: cross-mesh pins on `consumes[]`, encryption keys and principal mapping on bindings, and expiring data with `lifecycle.expire`. +The 0.7.6 release note covers the rest of 0.7.6: cross-mesh pins on `consumes[]`, encryption keys and principal mapping on bindings, and expiring data with `lifecycle.expire`. --- diff --git a/docs/guide/quickstart.md b/docs/guide/quickstart.md index 4a81bee..c191a4c 100644 --- a/docs/guide/quickstart.md +++ b/docs/guide/quickstart.md @@ -1,13 +1,13 @@ # Quickstart -A FLUID contract is **one YAML file**. This page shows the smallest file that validates against FLUID **0.7.5** β€” the latest stable version β€” explains each required block, and shows how to validate it. +A FLUID contract is **one YAML file**. This page shows the smallest file that validates against FLUID **0.7.6** β€” the latest stable version β€” explains each required block, and shows how to validate it. ## The smallest valid contract Every block below is required. Every other top-level block β€” `description`, `domain`, `tags`, `labels`, `consumes`, `build` / `builds`, `orchestration`, `sovereignty`, `accessPolicy`, `governance`, `retention`, `lineage`, `lifecycle`, `environments`, `docs`, `extensions` β€” is opt-in and layered on as you need it. (`agentPolicy` is opt-in too, but it lives inside an expose, under `exposes[].policy`.) ```yaml -fluidVersion: "0.7.5" +fluidVersion: "0.7.6" kind: DataProduct id: demo.bronze.hello_world name: "Hello World" @@ -29,7 +29,7 @@ exposes: | Block | Required | Meaning | |---|---|---| -| `fluidVersion` | βœ… | The schema version this file declares; it is validated against that version's schema. Use `"0.7.5"`, the latest stable version. | +| `fluidVersion` | βœ… | The schema version this file declares; it is validated against that version's schema. Use `"0.7.6"`, the latest stable version. | | `kind` | βœ… | `DataProduct` or `MLPipeline`. | | `id` | βœ… | Globally unique product id. Convention: `domain.layer.name`. | | `name` | βœ… | Human-readable display name. | @@ -41,16 +41,16 @@ Inside each `exposes[]` entry: - **`contract`** β€” the schema columns (or an `openapiRef`) plus optional data-quality rules. - **`binding`** β€” where the data physically lives: `platform` + `format` + `location`. -> **0.7.5 note.** 0.7.5 adds an opt-in Iceberg streaming sink on the `kafka-connect` acquisition engine, the `confluent` (Tableflow) and `pgvector` platforms, and new location fields β€” see [What's New in 0.7.5](/fluid/releases/0.7.5). It is additive: a valid 0.7.4 contract stays valid when it declares `"0.7.5"`, and declaring `"0.7.5"` is what lets a contract use the new fields. +> **0.7.6 note.** 0.7.6 adds declarative packaging modes, declared consumers, row filters and masked columns, Flink builds, a continuous trigger and several optional fields the ODCS and ODPS exporters use β€” see [What's New in 0.7.6](/fluid/releases/0.7.6). It is additive: a valid 0.7.5 contract stays valid when it declares `"0.7.6"`, and declaring `"0.7.6"` is what lets a contract use the new fields. > -> **0.7.6 is a preview.** Use it only if you need one of its fields, by declaring `fluidVersion: "0.7.6"`; it can still change. See [0.7.6 (preview)](/fluid/releases/0.7.6). +> **0.7.7 is a preview.** Use it only if you need one of its fields, by declaring `fluidVersion: "0.7.7"`; it can still change. See [0.7.7 (preview)](/fluid/releases/0.7.7). ## Validate it -Validate a contract against the published JSON Schema **of the version it declares** β€” for this file, 0.7.5: +Validate a contract against the published JSON Schema **of the version it declares** β€” for this file, 0.7.6: ``` -https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.5.json +https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json ``` Use a validator that supports JSON Schema Draft 2020-12, which every FLUID schema declares. Validators that compile `pattern` as an ECMA-262 regular expression, as JavaScript-based ones typically do, may refuse one pattern in the 0.7.2–0.7.6 schemas; see the [known interoperability issue](/fluid/schema/specification#known-interoperability-issue-one-pattern-is-not-an-ecma-262-regular-expression). Python's `jsonschema`, which this repository's conformance tooling uses, validates them. @@ -59,7 +59,7 @@ Or use the reference implementation, which validates against the declared versio ```bash pip install data-product-forge -fluid validate contract.fluid.yaml # βœ… Valid FLUID contract (schema v0.7.5) +fluid validate contract.fluid.yaml # βœ… Valid FLUID contract (schema v0.7.6) ``` ### Editor autocomplete & inline validation @@ -67,8 +67,8 @@ fluid validate contract.fluid.yaml # βœ… Valid FLUID contract (schema v0.7.5 Add this line as the **first line** of your contract file so the [YAML Language Server](https://github.com/redhat-developer/yaml-language-server) (bundled with the VS Code YAML extension) gives you live completion and validation. Use the URL of the version the file declares; pointing a 0.7.4 file at the 0.7.5 schema would hide 0.7.5-only fields that its own schema rejects. ```yaml -# yaml-language-server: $schema=https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.5.json -fluidVersion: "0.7.5" +# yaml-language-server: $schema=https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json +fluidVersion: "0.7.6" kind: DataProduct # ... ``` diff --git a/docs/how-to/build-patterns.md b/docs/how-to/build-patterns.md index b9f7531..99f86b2 100644 --- a/docs/how-to/build-patterns.md +++ b/docs/how-to/build-patterns.md @@ -2,7 +2,7 @@ *A Practical Guide to Build Patterns for FLUID v0.3.0* -> ℹ️ **This guide reflects the v0.3.0 build-pattern model.** The pattern *names* and the `build` block have evolved since: the current schema (latest stable `0.7.5`) uses `build.pattern` values such as `embedded-logic`, `hybrid-reference`, and `acquisition`, with the transformation engine and properties as direct children of `build`. Its YAML does not validate even against the 0.3.0 schema (for example, its ids carry a `:1.0.0` suffix that the 0.3.0 id pattern rejects). Read this as a conceptual tour of the available *styles* of build; for current, schema-valid syntax see [**Examples**](/fluid/examples/). +> ℹ️ **This guide reflects the v0.3.0 build-pattern model.** The pattern *names* and the `build` block have evolved since: the current schema (latest stable `0.7.6`) uses `build.pattern` values such as `embedded-logic`, `hybrid-reference`, and `acquisition`, with the transformation engine and properties as direct children of `build`. Its YAML does not validate even against the 0.3.0 schema (for example, its ids carry a `:1.0.0` suffix that the 0.3.0 id pattern rejects). Read this as a conceptual tour of the available *styles* of build; for current, schema-valid syntax see [**Examples**](/fluid/examples/). --- diff --git a/docs/releases/0.7.6.md b/docs/releases/0.7.6.md index bbd8b23..0b6647d 100644 --- a/docs/releases/0.7.6.md +++ b/docs/releases/0.7.6.md @@ -1,10 +1,10 @@ -# FLUID 0.7.6 (preview) β€” Declarative Packaging Modes +# FLUID 0.7.6 β€” Declarative Packaging Modes -::: warning Preview β€” not yet stable -0.7.6 is a **preview**. A contract uses it only by declaring `fluidVersion: "0.7.6"`, and its fields and their meaning can still change before it is promoted to stable. The latest stable version is [0.7.5](/fluid/releases/0.7.5). See [Stable and preview versions](/fluid/schema/versions#stable-and-preview-versions). +::: tip Stable +0.7.6 is the latest stable version. It was a preview while 0.7.5 was stable, and a contract uses it by declaring `fluidVersion: "0.7.6"`. The next preview is [0.7.7](/fluid/releases/0.7.7). See [Stable and preview versions](/fluid/schema/versions#stable-and-preview-versions). ::: -0.7.6 adds declarative ownership of infrastructure containers (`packaging`), a declared list of downstream consumers, pins for upstreams in another mesh, binding-level fields that let one cloud-neutral contract carry encryption, identity mapping and data expiry to each platform it is deployed on, row filters and masked columns in an expose's access policy, Apache Flink as a build engine, and credentials by environment variable for a REST catalog an Iceberg sink writes to. +0.7.6 adds declarative ownership of infrastructure containers (`packaging`), a declared list of downstream consumers, pins for upstreams in another mesh, binding-level fields that let one cloud-neutral contract carry encryption, identity mapping and data expiry to each platform it is deployed on, row filters and masked columns in an expose's access policy, Apache Flink as a build engine, a `continuous` trigger for builds that run as long-lived jobs, credentials by environment variable for a REST catalog an Iceberg sink writes to, and the optional Bitol-shaped fields the ODCS and ODPS exporters wanted: a team roster, a structured description, contract ids, Kafka serialization and column-level quality rules. - **Schema:** [`fluid-schema-0.7.6.json`](/fluid/schema/fluid-schema-0.7.6.json) Β· **Reference:** [`specs/0.7.6/fluid-spec.html`](/fluid/specs/0.7.6/fluid-spec.html) Β· **Diff:** [`diff-0.7.5-to-0.7.6.md`](https://github.com/open-data-protocol/fluid/blob/main/schema-diffs/diff-0.7.5-to-0.7.6.md) - **Compatibility:** additive over 0.7.5. `scripts/check-compat.py --from 0.7.5 --to 0.7.6` reports no narrowing change, so a valid 0.7.5 contract stays valid when it declares `"0.7.6"`. @@ -31,8 +31,15 @@ Every addition below is optional. The list is derived from the schemas: these ar | `exposes[].policy.authz.columnRestrictions[].mask` | The masking rule an `access: mask` restriction applies. | | `exposes[].binding.governance.lakeFormation.maskedViews` | The role and Glue connection behind the views that serve masked columns on Lake Formation. | | `build(s).properties.sink.catalogAuth` | How an acquisition sink authenticates to an Iceberg REST catalog, by naming environment variables. | +| `metadata.team` | A roster: the team's name and its members, each with a `username` and optionally a name, role and email. | +| `description` (as an object) | `{purpose, usage, limitations}`. In 0.7.5 the root `description` is a plain string, which is still accepted. | +| `consumes[].contractId`, `consumes[].contractVersion` | The upstream's own contract id, and the exact upstream version this product was built against. | +| `exposes[].contract.id` | An explicit, stable contract id for an expose. | +| `exposes[].contract.schema[].quality` | Column-level data quality rules in ODCS's own shape. | +| `exposes[].binding.location.host` | The host or endpoint of a binding; for Kafka, the bootstrap servers. | +| `exposes[].binding.serialization.{value, key}` | How Kafka message values and keys are serialized. | -`fluidVersion` accepts `"0.7.6"` in addition to `"0.7.3"`, `"0.7.4"` and `"0.7.5"`. Three enums gain values: `build(s).engine` accepts `flink`, `build(s).properties.sink.catalog` accepts `lakekeeper` and `polaris`, and `exposes[].policy.authz.columnRestrictions[].access` accepts `mask`. +`fluidVersion` accepts `"0.7.6"` in addition to `"0.7.3"`, `"0.7.4"` and `"0.7.5"`. Four enums gain values: `build(s).execution.trigger.type` accepts `continuous`, `build(s).engine` accepts `flink`, `build(s).properties.sink.catalog` accepts `lakekeeper` and `polaris`, and `exposes[].policy.authz.columnRestrictions[].access` accepts `mask`. 0.7.6 also writes down the meaning of a field that already existed: `exposes[].policy.authz.columnRestrictions` ([below](#column-restrictions-semantics-written-down)). @@ -228,7 +235,7 @@ build: `flink` names Apache Flink as the build's engine, usually for a long-running Flink SQL job (`properties.language: flink_sql`). The embedded-logic properties are a closed set, so deployment settings go under `properties.parameters`; the schema's description recommends the names the Flink Kubernetes Operator's job spec uses (`parallelism`, `upgradeMode`) and `flinkConfiguration`. A 0.7.5 contract cannot declare `engine: flink`. -The reference implementation validates and plans a `flink` build but ships no runner for it: the platform that runs the job, such as the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, executes it. It never runs such a build's SQL on DuckDB or turns it into a warehouse statement, and it treats a build with `properties.language: flink_sql` the same way on any `fluidVersion`. +For a contract that declares 0.7.6, the reference implementation validates and plans a `flink` build but ships no runner for it (a 0.7.7 contract can ask it to deploy the job; see [0.7.7](/fluid/releases/0.7.7)): the platform that runs the job, such as the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, executes it. It never runs such a build's SQL on DuckDB or turns it into a warehouse statement, and it treats a build with `properties.language: flink_sql` the same way on any `fluidVersion`. ## REST catalog credentials: `sink.catalogAuth` @@ -285,9 +292,98 @@ The reference implementation writes each named variable into the sink config it --- +## `trigger.type: continuous` + + +```yaml +build: + pattern: embedded-logic + engine: flink + properties: + language: flink_sql + sql: | + INSERT INTO payments_reconciled + SELECT * FROM payments + execution: + trigger: + type: continuous +``` + +`continuous` is for a build that runs as a long-lived job on its own platform: a Flink job on the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, or a `kafka-connect` or `debezium` connector. The platform keeps the job running, so nothing starts runs of it. The reference implementation's `fluid generate schedule`, `fluid schedule-sync` and the DAGs `fluid apply` writes leave such a build out (non-normative). + +What the schema pins is that a continuous trigger takes no other trigger. `cron`, `schedule`, `datasets`, `event` and `timetable` are each refused next to `type: continuous`, as Databricks Jobs refuses a continuous trigger next to any other; a trigger that names several of them gets one error for each. `condition`, `datasetsOperator` and `timezone` are not triggers, and stay accepted. A trigger of any other type, and a trigger with no `type`, keeps accepting every member it accepted before. A 0.7.5 contract cannot declare `continuous`: the 0.7.5 enum for `trigger.type` does not contain it. + +--- + +## Fields the Bitol exporters wanted + +Each of these is optional. A contract that sets none of them validates, and exports, exactly as it did before. The names follow Bitol's (ODCS v3.1.0, ODPS v1.1.0) wherever Bitol has one, so an imported value round-trips without a translation. + + +```yaml +description: # a plain string still works; the object is new + purpose: Customer profiles for analytics. + usage: Join on customer_id. + limitations: Not for marketing use. +metadata: + owner: { team: customer-analytics } + team: # a roster; metadata.owner stays the one contact + name: Customer Analytics + members: + - username: mjones # the only required member field + name: Mia Jones + role: data owner + email: mia.jones@example.com +consumes: + - productId: raw_customers + exposeId: raw_customers_table + versionConstraint: "^2.0.0" + contractId: urn:acme:contracts:raw-customers # the upstream's own contract id + contractVersion: 2.1.0 # the exact version this product was built against +``` + +- **`metadata.team`** is `{name, members[]}`, a roster rather than the single contact `metadata.owner` holds. Every member is a `teamMember`: `username` is required, and `name`, `role` and `email` are optional. `role` is free text. The object is closed. +- **The root `description`** is a string, as before, or `{purpose, usage, limitations}`. The string form is exported as `purpose`. The object is closed. +- **`consumes[].contractId`** is the upstream's own contract id, for an upstream that has one of its own. **`consumes[].contractVersion`** is a semantic version, the exact one this product was built and tested against. The schema does not check that it falls inside `versionConstraint`; `fluid validate` does not either. + + +```yaml +exposes: + - exposeId: customer_events + kind: table + binding: + platform: kafka + format: kafka_topic + location: + topic: customer-events + host: kafka.internal:9092 # bootstrap servers + serialization: + value: avro-confluent # json | avro | protobuf | xml | avro-confluent | json-confluent + key: json + contract: + id: urn:acme:contracts:customer-events # used as the ODCS id, instead of a derived one + schema: + - name: customer_id + type: string + required: true + quality: # ODCS-shaped rules; setting it replaces the derived ones + - id: customer_id-nullValues + type: library + metric: nullValues + dimension: completeness + severity: error +``` + +- **`binding.location.host`** is a host or endpoint; for a Kafka binding, the bootstrap servers. It is usually set per environment, in an overlay. +- **`binding.serialization`** says how Kafka message values, and optionally keys, are serialized. A Kafka binding otherwise declares only `format: kafka_topic`. The `-confluent` values name the same wire formats carried through a Confluent schema registry. +- **`exposes[].contract.id`** is an explicit, stable contract id: at least one character. Without it the reference implementation derives `.`, so renaming the product or the expose changes it. Two exposes may not share one; the schema cannot express that, and `fluid validate` refuses it. +- **A column's `quality`** is a list of rules in ODCS's own shape: `id` (letters, digits, `_` and `-`), `type` (`text`, `library`, `sql` or `custom`), `metric` (one of ODCS's five library metrics), `arguments`, `query`, `engine`, `implementation`, `dimension`, `description`, `severity` and `unit`. Each rule needs at least one member, and the item is open, so other ODCS keys pass through. Before 0.7.6 the column was closed and did not define `quality`, so a rule the ODCS exporter would have replayed made the contract invalid. + +--- + ## Two complete contracts -Both validate against the 0.7.6 JSON Schema (Draft 2020-12) and with the reference implementation, `data-product-forge` 0.21.0 (`fluid validate` prints `βœ… Valid FLUID contract (schema v0.7.6)`). +Both validate against the 0.7.6 JSON Schema (Draft 2020-12) and with the reference implementation, `data-product-forge` 0.27.0 (`fluid validate` prints `βœ… Valid FLUID contract (schema v0.7.6)`). **BigQuery:** column restriction, mapped principals, a product-owned key, expiring partitions, a percentile measure, packaging and a consumer. @@ -398,5 +494,5 @@ Remove `upstreamDigest` from the second contract and it becomes invalid: `'upstr ## References - [Schema Versions](/fluid/schema/versions) β€” status of every version, and how to choose `fluidVersion`. -- [Changelog: 0.7.5 β†’ 0.7.6](/fluid/schema/changelog#_0-7-5-to-0-7-6-preview-declarative-packaging-modes). +- [Changelog: 0.7.5 β†’ 0.7.6](/fluid/schema/changelog#_0-7-5-to-0-7-6-declarative-packaging-modes). - Reference implementation: the [governance parity notes](https://github.com/Agenticstiger/forge-cli/blob/main/docs/governance-parity.md) describe what `data-product-forge` emits and verifies for retention, encryption, column restrictions and principals on AWS and GCP. diff --git a/docs/releases/0.7.7.md b/docs/releases/0.7.7.md new file mode 100644 index 0000000..49a7d49 --- /dev/null +++ b/docs/releases/0.7.7.md @@ -0,0 +1,236 @@ +# FLUID 0.7.7 (preview) β€” Recursive Streaming Types + +::: warning Preview β€” not yet stable +0.7.7 is a **preview**. A contract uses it only by declaring `fluidVersion: "0.7.7"`, and its fields and their meaning can still change before it is promoted to stable. The latest stable version is [0.7.6](/fluid/releases/0.7.6). See [Stable and preview versions](/fluid/schema/versions#stable-and-preview-versions). +::: + +0.7.7 lets a column declare nested data (structs, arrays and maps, to any depth) and lets an embedded-logic build describe the Kafka records it reads and how they become rows, in place of a SQL statement. + +- **Schema:** [`fluid-schema-0.7.7.json`](/fluid/schema/fluid-schema-0.7.7.json) Β· **Reference:** [`specs/0.7.7/fluid-spec.html`](/fluid/specs/0.7.7/fluid-spec.html) Β· **Diff:** [`diff-0.7.6-to-0.7.7.md`](https://github.com/open-data-protocol/fluid/blob/main/schema-diffs/diff-0.7.6-to-0.7.7.md) +- **Compatibility:** additive over 0.7.6. `scripts/check-compat.py --from 0.7.6 --to 0.7.7` reports no narrowing change, so a valid 0.7.6 contract stays valid when it declares `"0.7.7"`. + +--- + +## What is new + +Every addition below is optional. These are the property paths present in 0.7.7 and absent in 0.7.6. + +| Path | What it declares | +|---|---| +| `schema[].fields`, `schema[].element`, `schema[].key`, `schema[].value` | The parts of a nested column: the child columns of a struct, the element type of an array, the key and value types of a map. They are available wherever a column is: `exposes[].contract.schema` and `stream.schema`. | +| `build(s).properties.stream` | The Kafka records a build reads: topic, value format, value fields, an optional key and Kafka metadata. | +| `build(s).properties.mapping` | How the stream's records become the output's rows: columns read from another path, renamed or cast, and an explode over an array. | + +`fluidVersion` accepts `"0.7.7"` in addition to `"0.7.3"` to `"0.7.6"`. No enum gains a value. + +One existing rule changes shape. Through 0.7.6, an embedded-logic build's `properties` always required `sql`. In 0.7.7 it requires `sql` unless the build has a `stream` and its `parameters.flink` declares `management`; the build is then described by the stream and the mapping alone. A `mapping` without a `stream` is refused. Every document that satisfied the old rule satisfies the new one. + +--- + +## Nested columns + + +```yaml +exposes: + - exposeId: orders + kind: table + binding: + platform: local + format: parquet + location: { path: "./orders.parquet" } + contract: + schema: + - { name: order_id, type: string, required: true } + - name: customer + type: struct + fields: + - { name: id, type: string } + - { name: country, type: string } + - name: tags + type: array + element: { type: string } + - name: attributes + type: map + key: { type: string, required: true } + value: { type: string } + - name: items + type: array + element: + type: struct + fields: + - { name: sku, type: string } + - { name: price, type: "decimal(18,2)" } +``` + +A column of `type: struct` lists its child columns in `fields`, each an ordinary column, so a child can carry anything a column carries. A column of `type: array` types its `element`, and a column of `type: map` types its `key` and `value`. `element`, `key` and `value` are each a nested type: `type` is required, `required` is optional, and `fields`, `element`, `key` and `value` nest again. Nothing in the schema limits the depth; the reference implementation stops at 32 levels (non-normative). A nested type has no `name`; the column it belongs to has one. + +`fields` must not be empty. The schema does not tie these members to `type`: it accepts `fields` on a column whose `type` is `string`. Whether a platform can hold a nested column, and which map keys it supports, is for the implementation (non-normative). The reference implementation accepts only a map key declared `required: true` and of a string type, and refuses a bare `struct`, `array` or `map` with no nested types on BigQuery and AWS Glue. + +--- + +## Streams and mappings + + +```yaml +build: + id: ingest-order-items + pattern: embedded-logic + engine: flink + properties: + stream: + kind: kafka # the only value in this preview + topic: orders.v1 + format: json # json | avro-confluent + schema: # the record value's fields, as columns + - { name: event_id, type: string, required: true } + - name: customer + type: struct + fields: + - { name: id, type: string } + - name: items + type: array + element: + type: struct + fields: + - { name: sku, type: string } + - { name: price, type: "decimal(18,2)" } + key: # optional + format: raw # raw | json | avro-confluent + schema: + - { name: order_key, type: string } + prefix: key_ # optional; prefixes the key's column names + metadata: # optional: Kafka record metadata, each read as a column + - { name: kafka_offset, from: offset } + mapping: + explode: { path: items, as: item } # one row per element of items + columns: # only the columns read from somewhere else + - { name: customer_id, from: customer.id } + - { name: sku, from: item.sku } + - { name: price, from: item.price, cast: "decimal(18,2)" } + parameters: + flink: + management: operator # with a stream, this is what makes sql optional +``` + +**`stream`** describes the records a build reads, apart from the engine that reads them. `kind`, `topic`, `format` and `schema` are required. `topic` names one Kafka topic, in Kafka's legal characters (letters, digits, `.`, `_` and `-`, at most 249, and not `.` or `..`): not a pattern and not a list. `format: json` is one JSON object per record. `format: avro-confluent` is Avro in the Confluent Schema Registry wire format, and the declared `schema` is the reader schema. The `key` takes the same `schema` as a list of columns. `metadata[].from` is one of `topic`, `partition`, `offset`, `timestamp`, `timestamp-type`, `headers` and `leader-epoch`. Brokers, consumer groups and registries are not part of the stream: they are parameters of the engine. + +**`mapping`** says how the records become rows. An output column the mapping does not list, nested fields included, is read from the stream column or field at the same path and keeps its type, so a stream landed as it is needs no mapping. `columns[]` lists the rest. `name` is a dotted path of identifiers. `from` is a dotted path, with a quoted key for a map entry (`attributes['channel']`), and `cast` is any column type. `explode` produces one output row per element of an array, named by `as` in the paths that follow; a record whose array is null or empty produces no row. Path segments are identifiers, so `items[0]` is not a path. + +The grammar is the schema's; whether the types of the paths and the output columns agree is not. The reference implementation checks every output column's type and nullability against the path it is read from before anything deploys (non-normative). + +### `sql`, or a stream + +An embedded-logic build says what it does in one of two ways. With `sql`, the statement is the build, and a `stream` may sit beside it to describe what the statement reads. With a `stream` and a `mapping`, the build has no `sql` and the platform writes the query; the schema accepts that only when the build's `parameters.flink` declares `management` (any value; the example above uses `operator`). A build with neither is refused with a missing `sql`, the same error as in 0.7.6. + + +```yaml +build: + id: ingest-order-items + pattern: embedded-logic + engine: flink + properties: + language: flink_sql + sql: | + INSERT INTO order_items SELECT event_id FROM orders + stream: + kind: kafka + topic: orders.v1 + format: json + schema: + - { name: event_id, type: string } +``` + +--- + +## A complete contract + +The reference implementation can deploy a managed Flink job for a build like this one: a Kubernetes cluster that runs the Flink Kubernetes Operator, writing an Iceberg REST catalog table. That is behaviour of one implementation (non-normative), and it needs operator-side allow-lists for the cluster context, namespace, image, service account and secrets the contract names. The contract below validates against the 0.7.7 JSON Schema (Draft 2020-12) and with `data-product-forge` 0.27.0 (`fluid validate` prints `βœ… Valid FLUID contract (schema v0.7.7)`). `fluid validate` refuses a managed Flink build that has no build `id` or no Iceberg output named in `outputs`, so the contract carries both. + +```yaml +fluidVersion: "0.7.7" +kind: DataProduct +id: commerce.order_items +name: Order items +metadata: + owner: { team: data-platform, email: data-platform@example.com } +build: + id: ingest-order-items + pattern: embedded-logic + engine: flink + outputs: [order_items] + execution: + trigger: + type: continuous + properties: + language: flink_sql + stream: + kind: kafka + topic: orders.v1 + format: json + schema: + - { name: event_id, type: string, required: true } + - name: customer + type: struct + fields: + - { name: id, type: string } + - name: items + type: array + element: + type: struct + fields: + - { name: sku, type: string } + - { name: price, type: "decimal(18,2)" } + metadata: + - { name: kafka_offset, from: offset } + mapping: + explode: { path: items, as: item } + columns: + - { name: customer_id, from: customer.id } + - { name: sku, from: item.sku } + - { name: price, from: item.price } + parameters: + flink: + management: operator + source: + bootstrapServers: kafka:9092 + groupId: forge-commerce-order-items-v1 + deployment: + name: commerce-order-items + namespace: data-products + context: development + serviceAccount: flink + image: forge-flink@sha256:0000000000000000000000000000000000000000000000000000000000000000 + checkpointUri: s3://flink-state/order-items/checkpoints + savepointUri: s3://flink-state/order-items/savepoints + haUri: s3://flink-state/order-items/ha + secretRefs: [flink-storage] +exposes: + - exposeId: order_items + kind: table + binding: + platform: aws + format: iceberg + location: + catalog: rest + uri: http://iceberg-rest:8181 + warehouse: s3://warehouse/ + database: commerce + table: order_items + contract: + schema: + - { name: event_id, type: string, required: true } + - { name: customer_id, type: string } + - { name: sku, type: string } + - { name: price, type: "decimal(18,2)" } + - { name: kafka_offset, type: long, required: true } +``` + +`parameters` is an open object, so the `flink` settings above are not checked by the schema. The `image` digest is a placeholder. + +--- + +## References + +- [Schema Versions](/fluid/schema/versions) β€” status of every version, and how to choose `fluidVersion`. +- [Changelog: 0.7.6 β†’ 0.7.7](/fluid/schema/changelog#_0-7-6-to-0-7-7-preview-recursive-streaming-types). +- [0.7.6](/fluid/releases/0.7.6) β€” the stable version this preview is built on, including the `continuous` trigger used above. diff --git a/docs/releases/README.md b/docs/releases/README.md index beb937c..8caaea6 100644 --- a/docs/releases/README.md +++ b/docs/releases/README.md @@ -2,12 +2,13 @@ Release notes for the FLUID specification, newest first. Each entry links to its full page and to the auto-generated schema diff. -**0.7.5 is the latest stable version. 0.7.6 is a preview**: a contract uses it only by declaring `fluidVersion: "0.7.6"`, and it can still change before it is promoted. See [Stable and preview versions](/fluid/schema/versions#stable-and-preview-versions). +**0.7.6 is the latest stable version. 0.7.7 is a preview**: a contract uses it only by declaring `fluidVersion: "0.7.7"`, and it can still change before it is promoted. See [Stable and preview versions](/fluid/schema/versions#stable-and-preview-versions). | Version | Status | Codename | Compatibility | Summary | |---|---|---|---|---| -| **[0.7.6](/fluid/releases/0.7.6)** | **Preview** | Declarative Packaging Modes | βœ… Additive over 0.7.5 | Adds top-level `packaging` and `consumers`, cross-mesh pins on `consumes[]` (`upstreamWorkspace` + `upstreamDigest`), `aggParams` on semantic measures, and on bindings `encryption.kms`, `principals` and Lake Formation `bucketPolicy`; an expose's `lifecycle` gains `expire`, masking rules gain typed `params`, and access policies gain `rowFilters` and `access: mask`. Builds can declare `engine: flink`, and an Iceberg sink can name a REST catalog's credentials by environment variable (`sink.catalogAuth`). | -| **[0.7.5](/fluid/releases/0.7.5)** | **Stable (latest)** | Streaming Kafka β†’ Iceberg Sink & Confluent Tableflow | βœ… Additive over 0.7.4 | Adds an opt-in Iceberg streaming sink on the `kafka-connect` acquisition engine, the `confluent` (Tableflow) and `pgvector` binding platforms, the `pgvector_table` format and `binding.vectorConfig`, and location fields for Tableflow, Iceberg catalogs, Redshift Serverless and Kinesis. | +| **[0.7.7](/fluid/releases/0.7.7)** | **Preview** | Recursive Streaming Types | βœ… Additive over 0.7.6 | Adds nested columns (`fields`, `element`, `key` and `value` on a column), and on an embedded-logic build the Kafka `stream` it reads and the `mapping` that turns its records into rows. `sql` is no longer required when a managed Flink build has a stream. | +| **[0.7.6](/fluid/releases/0.7.6)** | **Stable (latest)** | Declarative Packaging Modes | βœ… Additive over 0.7.5 | Adds top-level `packaging` and `consumers`, cross-mesh pins on `consumes[]` (`upstreamWorkspace` + `upstreamDigest`), `aggParams` on semantic measures, and on bindings `encryption.kms`, `principals` and Lake Formation `bucketPolicy`; an expose's `lifecycle` gains `expire`, masking rules gain typed `params`, and access policies gain `rowFilters` and `access: mask`. Builds can declare `engine: flink` and a `continuous` trigger, and an Iceberg sink can name a REST catalog's credentials by environment variable (`sink.catalogAuth`). Optional Bitol-shaped fields: `metadata.team`, an object root `description`, `consumes[].contractId` and `contractVersion`, `exposes[].contract.id`, column `quality`, `binding.location.host` and `binding.serialization`. | +| **[0.7.5](/fluid/releases/0.7.5)** | Stable | Streaming Kafka β†’ Iceberg Sink & Confluent Tableflow | βœ… Additive over 0.7.4 | Adds an opt-in Iceberg streaming sink on the `kafka-connect` acquisition engine, the `confluent` (Tableflow) and `pgvector` binding platforms, the `pgvector_table` format and `binding.vectorConfig`, and location fields for Tableflow, Iceberg catalogs, Redshift Serverless and Kinesis. | | **[0.7.4](/fluid/releases/0.7.4)** | Stable | Runtime agentPolicy Enforcement at the MCP Gateway | βœ… Additive over 0.7.3 | Adds the optional `exposes[].mcp` block, the `postgres` binding platform, and the `postgres_table` / `athena_table` / `glue_table` formats. | | **[0.7.3](/fluid/releases/0.7.3)** | Stable | Source-Aligned Acquisition | βœ… Additive over 0.7.2 | The `acquisition` build pattern and six ingestion engines, top-level `retention`, `governance` (AWS Lake Formation) and `extensions`, `binding.governance`, `metadata.productType`, and `exposes[].contract.schemaPolicy`. | | **[0.7.2](/fluid/releases/0.7.2)** | Stable | Semantic Truth Engine | ⚠️ One narrowing: `notification` became a closed object | A per-expose `semantics` block, `binding.icebergConfig`, the top-level `orchestration` key, and adapter-qualified dbt engines (`build.engine: dbt-`). | diff --git a/docs/schema/README.md b/docs/schema/README.md index 3e885f7..d40ea94 100644 --- a/docs/schema/README.md +++ b/docs/schema/README.md @@ -1,6 +1,6 @@ # Schema Reference -Everything you need to read, write, and validate a FLUID contract. **The latest stable schema is 0.7.5; 0.7.6 is a preview** (see [Versions](/fluid/schema/versions#stable-and-preview-versions)). +Everything you need to read, write, and validate a FLUID contract. **The latest stable schema is 0.7.6; 0.7.7 is a preview** (see [Versions](/fluid/schema/versions#stable-and-preview-versions)). | Page | Use it for | |---|---| @@ -13,8 +13,8 @@ Everything you need to read, write, and validate a FLUID contract. **The latest ## Machine-readable artifacts -- **JSON Schema (0.7.5, stable):** [`/fluid/schema/fluid-schema-0.7.5.json`](/fluid/schema/fluid-schema-0.7.5.json) β€” point your validator here. -- **Generated HTML reference (0.7.5):** [`/fluid/specs/0.7.5/fluid-spec.html`](/fluid/specs/0.7.5/fluid-spec.html) β€” every type, enum and validation rule, rendered from the schema. -- **Preview (0.7.6):** [`fluid-schema-0.7.6.json`](/fluid/schema/fluid-schema-0.7.6.json) Β· [`specs/0.7.6/fluid-spec.html`](/fluid/specs/0.7.6/fluid-spec.html). +- **JSON Schema (0.7.6, stable):** [`/fluid/schema/fluid-schema-0.7.6.json`](/fluid/schema/fluid-schema-0.7.6.json) β€” point your validator here. +- **Generated HTML reference (0.7.6):** [`/fluid/specs/0.7.6/fluid-spec.html`](/fluid/specs/0.7.6/fluid-spec.html) β€” every type, enum and validation rule, rendered from the schema. +- **Preview (0.7.7):** [`fluid-schema-0.7.7.json`](/fluid/schema/fluid-schema-0.7.7.json) Β· [`specs/0.7.7/fluid-spec.html`](/fluid/specs/0.7.7/fluid-spec.html). Validate a document against the schema of the `fluidVersion` it declares. For older versions, see [**Versions**](/fluid/schema/versions). diff --git a/docs/schema/anatomy.md b/docs/schema/anatomy.md index f5b1787..5b76eea 100644 --- a/docs/schema/anatomy.md +++ b/docs/schema/anatomy.md @@ -1,11 +1,11 @@ # Schema Anatomy -A FLUID contract is one YAML file. This page walks **every top-level block** of the latest stable schema, **0.7.5**, in the order you'd typically author them, with a one-paragraph "what / when / why" for each and a link into the generated reference doc for the field-by-field details. +A FLUID contract is one YAML file. This page walks **every top-level block** of the latest stable schema, **0.7.6**, in the order you'd typically author them, with a one-paragraph "what / when / why" for each and a link into the generated reference doc for the field-by-field details. > πŸ“‹ If you just want a one-line-per-field table, see the [**Cheatsheet**](/fluid/schema/cheatsheet). > 🟒 If you want a copy-pasteable minimal example, see the [**Minimal Valid Contract**](/fluid/schema/minimal-contract). -> πŸ“š Full field-by-field reference: [`specs/0.7.5/fluid-spec.html`](/fluid/specs/0.7.5/fluid-spec.html). -> πŸ§ͺ Blocks marked **0.7.6 preview** need `fluidVersion: "0.7.6"` and can still change β€” see [0.7.6 (preview)](/fluid/releases/0.7.6). +> πŸ“š Full field-by-field reference: [`specs/0.7.6/fluid-spec.html`](/fluid/specs/0.7.6/fluid-spec.html). +> πŸ§ͺ Blocks marked **0.7.6** need `fluidVersion: "0.7.6"` or later β€” see [0.7.6](/fluid/releases/0.7.6). The YAML blocks on this page fit together: merged in order into the [minimal contract](/fluid/schema/minimal-contract), they form one contract that validates against the 0.7.5 schema. @@ -49,7 +49,7 @@ The YAML blocks on this page fit together: merged in order into the [minimal con β”‚ machineLearning { ... } docs { ... } β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - 0.7.6 preview adds: packaging, consumers[], consumes[].upstream*, + 0.7.6 adds: packaging, consumers[], consumes[].upstream*, binding.{encryption, principals, packaging}, lifecycle.expire ``` @@ -115,7 +115,7 @@ consumes: **When:** any product that isn't strictly source-aligned (i.e. not `build.pattern: acquisition`). **Why:** the orchestrator builds the DAG from `consumes`. No `consumes` β†’ no automatic upstream waiting, no automatic lineage. Each entry can also state `qosExpectations` (`freshnessMax`, `maxStaleness`, `minCompleteness`) and `requiredPolicies`. -> πŸ§ͺ **0.7.6 preview:** `upstreamWorkspace` and `upstreamDigest` pin an upstream that lives in another mesh; setting the first requires the second. See [0.7.6](/fluid/releases/0.7.6#cross-mesh-pins-on-consumes). +> πŸ§ͺ **0.7.6:** `upstreamWorkspace` and `upstreamDigest` pin an upstream that lives in another mesh; setting the first requires the second. See [0.7.6](/fluid/releases/0.7.6#cross-mesh-pins-on-consumes). --- @@ -209,7 +209,7 @@ exposes: Platform + format + location. Plus `icebergConfig` (0.7.2+) when format is `iceberg` (write version, file format, `partitionSpec`, `sortOrder`), so the contract owns table-format details too. **0.7.4** adds the `postgres` platform and the `postgres_table` / `athena_table` / `glue_table` formats. **0.7.5** adds the `confluent` (Tableflow) and `pgvector` platforms, `pgvector_table`, `vectorConfig`, and location fields for Iceberg catalogs, Redshift Serverless and Kinesis β€” see [0.7.5](/fluid/releases/0.7.5). Optional `binding.governance` carries per-resource AWS Lake Formation settings. -> πŸ§ͺ **0.7.6 preview** adds `binding.encryption.kms`, `binding.principals` (logical principal β†’ platform identity), `binding.packaging`, and Lake Formation `bucketPolicy`; a bucket policy it emits is **authoritative** and replaces every other statement on the bucket. See [0.7.6](/fluid/releases/0.7.6#binding-fields-encryption-principals-lake-formation-bucket-policy). +> πŸ§ͺ **0.7.6** adds `binding.encryption.kms`, `binding.principals` (logical principal β†’ platform identity), `binding.packaging`, and Lake Formation `bucketPolicy`; a bucket policy it emits is **authoritative** and replaces every other statement on the bucket. See [0.7.6](/fluid/releases/0.7.6#binding-fields-encryption-principals-lake-formation-bucket-policy). --- @@ -371,7 +371,7 @@ exposes: `admins` is **authoritative, not additive**. Applying it sets the account's Lake Formation admins to exactly this list, so any admin you do not list is **removed** β€” including the role or user that runs the apply, after which Lake Formation grants and registrations that need an admin can fail. List every principal that must stay an admin, the deploying identity included. (The 0.7.3–0.7.5 schemas were first published with a description saying the opposite; see the [Changelog](/fluid/schema/changelog#corrections-to-published-schemas).) In the reference implementation, applying this block also clears the account's other data-lake settings that it does not set: the create-database and create-table default permissions, trusted resource owners and parameters. Destroying the emitted `aws_lakeformation_data_lake_settings` resource empties the admin list and resets `CROSS_ACCOUNT_VERSION` to 1. See the field's description in the [schema](/fluid/specs/0.7.5/fluid-spec.html). ::: -> πŸ§ͺ **0.7.6 preview** adds `binding.principals`, which maps the logical principals named in `accessPolicy` and `columnRestrictions` to each platform's identities, writes down what `columnRestrictions` means (deny beats allow; a restriction never grants access), and adds typed masking `params`. See [0.7.6](/fluid/releases/0.7.6). +> πŸ§ͺ **0.7.6** adds `binding.principals`, which maps the logical principals named in `accessPolicy` and `columnRestrictions` to each platform's identities, writes down what `columnRestrictions` means (deny beats allow; a restriction never grants access), and adds typed masking `params`. See [0.7.6](/fluid/releases/0.7.6). > ⭐ **0.7.4 β€” runtime enforcement.** When an expose also carries an [`mcp` block](#mcp-0-7-4-opt-this-expose-into-the-mcp-output-port-gateway), the `agentPolicy` above stops being passive metadata: an MCP gateway that implements 0.7.4, such as the reference implementation's, enforces `allowedModels` / `deniedModels` and `allowedUseCases` / `deniedUseCases` on every read. The `agentPolicy` shape is unchanged β€” 0.7.4 only changes *when* it bites. @@ -405,7 +405,7 @@ These are the deeper-cut blocks β€” every one is opt-in: - **`lifecycle`** β€” `state: preview | active | deprecated | retired`, `retention` (an ISO-8601 duration) and `deprecationPolicy` (`noticePeriod`, `contact`, `replacement`). Note that `preview` here is a *product's* state; it has nothing to do with preview *schema versions*. πŸ§ͺ In the 0.7.6 preview, an expose's `lifecycle` also takes `expire: true`, which deletes data older than `retention`. - **`docs`** β€” `homepage`, `runbook`, `dictionary`, `changeLog`. -For the field-by-field detail on each, see the generated reference: [`specs/0.7.5/fluid-spec.html`](/fluid/specs/0.7.5/fluid-spec.html). +For the field-by-field detail on each, see the generated reference: [`specs/0.7.6/fluid-spec.html`](/fluid/specs/0.7.6/fluid-spec.html). --- diff --git a/docs/schema/changelog.md b/docs/schema/changelog.md index cf3cb98..f6ac007 100644 --- a/docs/schema/changelog.md +++ b/docs/schema/changelog.md @@ -1,14 +1,29 @@ # Changelog -Human-readable summaries of what changed between consecutive versions of the FLUID schema. Each entry links to the full auto-generated change list on GitHub. **0.7.5 is the latest stable version; 0.7.6 is a preview** (see [Versions](/fluid/schema/versions#stable-and-preview-versions)). +Human-readable summaries of what changed between consecutive versions of the FLUID schema. Each entry links to the full auto-generated change list on GitHub. **0.7.6 is the latest stable version; 0.7.7 is a preview** (see [Versions](/fluid/schema/versions#stable-and-preview-versions)). "No narrowing change" below means that `python3 scripts/check-compat.py --from --to ` reports none: every document valid under the earlier version stays valid under the later one (once it declares the later `fluidVersion`). Each summary lists the property paths present in the later schema and absent in the earlier one, plus new enum values. -## 0.7.5 to 0.7.6 (preview): Declarative Packaging Modes +## 0.7.6 to 0.7.7 (preview): Recursive Streaming Types + +[Full diff β†’](https://github.com/open-data-protocol/fluid/blob/main/schema-diffs/diff-0.7.6-to-0.7.7.md) Β· [Release note β†’](/fluid/releases/0.7.7) + +No narrowing change. **Preview:** the 0.7.7 schema can still change before it is promoted. + +**Added** + +- Nested columns: `column.fields` (a non-empty list of columns), `column.element`, `column.key` and `column.value`, the last three each a new closed `nestedType` (`type` required; `required`, `fields`, `element`, `key` and `value` optional, nesting again). +- `build(s).properties.stream`, a new closed `streamSource` definition: `kind` (`kafka`), `topic`, `format` (`json` | `avro-confluent`) and `schema` required; `key` (`format` `raw` | `json` | `avro-confluent`, `schema`, `prefix`) and `metadata[]` (`name`, `from`: `topic` | `partition` | `offset` | `timestamp` | `timestamp-type` | `headers` | `leader-epoch`) optional. +- `build(s).properties.mapping`, a new closed `recordMapping` definition: `explode` (`path` and `as`, both required) and `columns[]` (`name` and `from` required; `cast` optional). +- `fluidVersion` accepts `"0.7.7"`. + +**Changed:** `embeddedLogicPattern` no longer lists `sql` as unconditionally required. It requires `sql` unless the build has a `stream` and `parameters.flink` declares `management`, and `mapping` requires `stream` (`dependentRequired`). Every document that carried `sql` is unaffected. + +## 0.7.5 to 0.7.6: Declarative Packaging Modes [Full diff β†’](https://github.com/open-data-protocol/fluid/blob/main/schema-diffs/diff-0.7.5-to-0.7.6.md) Β· [Release note β†’](/fluid/releases/0.7.6) -No narrowing change. **Preview:** the 0.7.6 schema can still change before it is promoted. +No narrowing change. **Added** @@ -26,10 +41,19 @@ No narrowing change. **Preview:** the 0.7.6 schema can still change before it is - `exposes[].binding.governance.lakeFormation.maskedViews` (`definer` and `validationConnection`, both required). - `build(s).engine` value `flink`; see the [0.7.6 notes](/fluid/releases/0.7.6#engine-flink). - `build(s).properties.sink.catalog` values `lakekeeper` and `polaris`, and `build(s).properties.sink.catalogAuth` (`type`: `oauth2` | `bearer`), which names the environment variables holding a REST catalog's credentials. +- `exposes[].binding.location.host` (the host or endpoint; for Kafka, the bootstrap servers) and `exposes[].binding.serialization` (`value` and `key`, each `json` | `avro` | `protobuf` | `xml` | `avro-confluent` | `json-confluent`). +- `metadata.team` (`name`, `members[]`), with each member a new `teamMember` definition: `username` required; `name`, `role` and `email` optional; closed. +- The root `description` accepts a plain string or a new closed `descriptionObject` (`purpose`, `usage`, `limitations`). +- `consumes[].contractId` and `consumes[].contractVersion` (a semantic version). +- `exposes[].contract.id` (at least one character). +- `exposes[].contract.schema[].quality`: a list of ODCS-shaped rules. Each item needs at least one member and is open; `id`, `type` (`text` | `library` | `sql` | `custom`), `metric`, `dimension` and the other common keys are typed. +- `build(s).execution.trigger.type` value `continuous`. A continuous trigger takes no other trigger: `cron`, `schedule`, `datasets`, `event` and `timetable` are each refused next to it. See the [0.7.6 notes](/fluid/releases/0.7.6#trigger-type-continuous). - `fluidVersion` accepts `"0.7.6"`. **Described:** `exposes[].policy.authz.columnRestrictions` gains a description of its semantics (deny beats allow; a restriction never grants access; principals are logical). +**Re-published.** The 0.7.6 schema was first published here from forge-cli v0.21.0, while 0.7.6 was a preview. It has since gained the additions listed under "Added" from `binding.location.host` down to `trigger.type: continuous`, and its root `description` became `oneOf`. The published file has been re-vendored from forge-cli v0.27.0, which carries the stable 0.7.6. Every document the earlier copy accepted is still accepted. One conformance case, which used `binding.location.host` as an example of an undocumented addressing property, now uses another name. + ## 0.7.4 to 0.7.5: Streaming Kafka to Iceberg Sink & Confluent Tableflow [Full diff β†’](https://github.com/open-data-protocol/fluid/blob/main/schema-diffs/diff-0.7.4-to-0.7.5.md) Β· [Release note β†’](/fluid/releases/0.7.5) diff --git a/docs/schema/cheatsheet.md b/docs/schema/cheatsheet.md index b3047a5..b7fffc9 100644 --- a/docs/schema/cheatsheet.md +++ b/docs/schema/cheatsheet.md @@ -1,9 +1,9 @@ # Schema Cheatsheet -One row per field of the latest stable schema, **0.7.5**: a one-line meaning, whether it is required, and the version it first appeared in. Rows marked πŸ§ͺ are in the **0.7.6 preview** only; a contract needs `fluidVersion: "0.7.6"` to use them, and they can still change. +One row per field of the latest stable schema, **0.7.6**: a one-line meaning, whether it is required, and the version it first appeared in. Rows marked πŸ§ͺ were added in **0.7.6**; a contract needs `fluidVersion: "0.7.6"` or later to use them. The preview, 0.7.7, adds nested columns and the build `stream` and `mapping`; they are in its [release notes](/fluid/releases/0.7.7), not in this sheet. > 🧭 For a guided tour with context per block, read the [**Schema Anatomy**](/fluid/schema/anatomy). -> πŸ“š For the exhaustive field-by-field reference, see [`specs/0.7.5/fluid-spec.html`](/fluid/specs/0.7.5/fluid-spec.html) (stable) or [`specs/0.7.6/fluid-spec.html`](/fluid/specs/0.7.6/fluid-spec.html) (preview). +> πŸ“š For the exhaustive field-by-field reference, see [`specs/0.7.6/fluid-spec.html`](/fluid/specs/0.7.6/fluid-spec.html) (stable) or [`specs/0.7.7/fluid-spec.html`](/fluid/specs/0.7.7/fluid-spec.html) (preview). "Since" is the version from which the field has been in the schema without interruption; where a member of the same name existed earlier and was removed, the row says so. Contracts before 0.5.7 had a different overall shape (see the [Changelog](/fluid/schema/changelog#before-0-7-1-history-before-the-compatibility-promise)), so for those blocks the current *shape* is newer than the name. @@ -21,7 +21,7 @@ The root object is closed: a top-level key not listed here makes the document in | `name` | Human-readable display name. | βœ… | 0.0.1 | `string` | | `metadata` | Owner, layer, product type, business context. | βœ… | 0.0.1 | `object` | | `exposes` | Ports this product publishes. | βœ… | 0.0.1 | `object[]` | -| `description` | Business-facing summary. | | 0.0.1 | `string` | +| `description` | Business-facing summary: a string, or (πŸ§ͺ since 0.7.6) an object with `purpose`, `usage` and `limitations`. | | 0.0.1 | `string` \| `object` | | `domain` | Owning business domain. | | 0.0.1 | `string` | | `consumes` | Upstream FLUID products this product reads. | | 0.0.1 | `object[]` | | `build` | How the product is produced (pattern + engine + properties). | | 0.0.1 | `object` | @@ -58,6 +58,7 @@ The root object is closed: a top-level key not listed here makes the document in | `metadata.experimental` | Experimental features the contract opts into. | | 0.7.3 | | `metadata.businessContext` | `domain`, `subdomain`, `businessCapability`, `valueStream`. | | 0.5.7 | | `metadata.tags` | Tags on the metadata. (Also existed from 0.0.1 to 0.4.0.) | | 0.7.2 | +| πŸ§ͺ `metadata.team` | A roster: `name` and `members[]`; each member needs `username` and may carry `name`, `role` and `email`. | | 0.7.6 | | `metadata.createdAt` | Creation time (`format: date-time`, an annotation). | | 0.5.7 | | `metadata.provenance` | Generation envelope written by tooling. | | 0.7.2 | @@ -79,6 +80,8 @@ Each entry requires `exposeId`, `kind`, `contract` and `binding`, and is closed. | `exposes[].contract.schema[]` | Columns: `name` and `type` required; `required`, `description`, `sensitivity`, `semanticType`, `businessName`, `businessDefinition`, `validationRules`, `tags`, `labels`. | (one of) | | `exposes[].contract.openapiRef` | Reference to an OpenAPI document (for `kind: api`). | (one of) | | `exposes[].contract.dq.rules[]` | Declarative quality assertions (below). | | +| πŸ§ͺ `exposes[].contract.id` | An explicit, stable contract id (at least one character); two exposes may not share one. | | 0.7.6 | +| πŸ§ͺ `exposes[].contract.schema[].quality` | ODCS-shaped quality rules for a column: `id`, `type`, `metric`, `dimension`, `severity`, … | | 0.7.6 | | `exposes[].contract.schemaPolicy` | `strict` \| `discover_and_freeze` \| `evolve_safe` \| `evolve_all` (since 0.7.3). | | | `exposes[].contract.schemaSignature` | `sha256:` + 64 hex β€” a digest of the schema. | | | `exposes[].qos` | `availability` (a percentage string such as `"99.9%"`), `freshnessSLO`, `latencyP95` (ISO-8601 durations), `dataLossSLO`, `completenessTarget`, `errorBudget`. | | @@ -88,6 +91,8 @@ Each entry requires `exposeId`, `kind`, `contract` and `binding`, and is closed. | `exposes[].binding.platform` | `gcp` \| `aws` \| `azure` \| `snowflake` \| `databricks` \| `kafka` \| `confluent` (0.7.5) \| `local` \| `kubernetes` \| `postgres` (0.7.4) \| `pgvector` (0.7.5) \| `other`. | βœ… | | `exposes[].binding.format` | `bigquery_table` \| `snowflake_table` \| `snowflake_view` \| `iceberg` \| `delta_table` \| `parquet` \| `csv` \| `json` \| `http_api` \| `grpc_api` \| `kafka_topic` \| `pubsub_topic` \| `gcs_file` \| `s3_file` \| `redshift_table` \| `redshift_serverless` \| `redshift_external_schema` \| `postgres_table` (0.7.4) \| `athena_table` (0.7.4) \| `glue_table` (0.7.4) \| `pgvector_table` (0.7.5) \| `other`. | βœ… | | `exposes[].binding.location` | Platform address; no member is required by the schema. 0.7.5 adds `environment_id`, `kafka_cluster_id`, `confluent_role_arn` (Tableflow), `catalog`, `warehouse`, `uri`, `partitionBy` (Iceberg catalogs), `namespace`, `workgroup`, `iam_role_arn`, `external_schema`, `glue_database` (Redshift Serverless), `stream` (Kinesis). | βœ… | +| πŸ§ͺ `exposes[].binding.location.host` | Host or endpoint; for Kafka, the bootstrap servers. | | 0.7.6 | +| πŸ§ͺ `exposes[].binding.serialization` | How Kafka values and keys are serialized: `value` and `key`, each `json` \| `avro` \| `protobuf` \| `xml` \| `avro-confluent` \| `json-confluent`. | | 0.7.6 | | `exposes[].binding.icebergConfig` | Iceberg table settings when `format: iceberg` (since 0.7.2). | | | `exposes[].binding.vectorConfig` | Vector output port for `pgvector` (since 0.7.5): `dimensions` (required, 1–16000), `embeddingModel`, `vectorType`, `indexType`, `distanceMetric`, `hnsw`, `ivfflat`, `table`, `sourceKeyColumn`. | | | `exposes[].binding.governance` | Per-resource AWS Lake Formation settings: `registerLocation`, `grants[]`, `tags`, `rowFilter` (since 0.7.3). Account-wide settings live in the top-level `governance` block. | | @@ -123,7 +128,7 @@ A build is closed and has no required member. | `build.engine` | `dbt` \| `sql` \| `python` \| `spark` \| `glue` \| `custom` \| `duckdb` \| `airbyte` \| `meltano` \| `dlt` \| `kafka-connect` \| `debezium` (the last six since 0.7.3), or `dbt-` such as `dbt-databricks` (since 0.7.2). | | | `build.capabilities` | What the build asks of its runner: `full_refresh`, `incremental_append`, `incremental_dedup`, `incremental_merge`, `cdc`, `streaming`, `schema_discovery`, `schema_evolution`, `dlp_scan`, `at_most_once`, `at_least_once`, `exactly_once` (since 0.7.3). | | | `build.properties` | Pattern-specific settings. `hybrid-reference` requires `model`; `embedded-logic` requires `sql`; `acquisition` requires `source` (below). | | -| `build.execution.trigger` | `type`: `schedule` \| `event` \| `manual` \| `dependency` \| `dataset` \| `schedule_and_dataset` \| `timetable`, plus `schedule`, `cron`, `timezone`, … | | +| `build.execution.trigger` | `type`: `schedule` \| `event` \| `manual` \| `dependency` \| `dataset` \| `schedule_and_dataset` \| `timetable` \| `continuous` (πŸ§ͺ 0.7.6; takes no `cron`, `schedule`, `datasets`, `event` or `timetable`), plus `schedule`, `cron`, `timezone`, … | | | `build.execution.runtime` | Where it runs: `platform`, `resources`, `image`, `executor`, `serviceAccount`, `timeout`. | | | `build.execution.retries` | `maxAttempts`, `backoffStrategy` (`fixed` \| `exponential` \| `linear`), `initialDelay`, `maxDelay`. | | | `build.execution.notifications[]` | `type` (`email` \| `slack` \| `webhook` \| `pagerduty`), `target`, `condition` (`success` \| `failure` \| `always`). Closed since 0.7.2. | | @@ -209,6 +214,8 @@ When `pattern: acquisition` (0.7.3+), the schema validates `properties` against | `consumes[].qosExpectations` | `freshnessMax`, `maxStaleness` (durations), `minCompleteness` (0–1). | | | `consumes[].requiredPolicies` | Policies the upstream must carry. | | | `consumes[].purpose` | Why this product reads the upstream. | | +| πŸ§ͺ `consumes[].contractId` | The upstream's own contract id, when it has one. | | +| πŸ§ͺ `consumes[].contractVersion` | The exact upstream version this product was built against (a semantic version). | | | πŸ§ͺ `consumes[].upstreamWorkspace` | The other mesh that owns the upstream. Requires `upstreamDigest`. | | | πŸ§ͺ `consumes[].upstreamDigest` | `sha256:` + 64 lowercase hex β€” the upstream contract this product was composed against. | | diff --git a/docs/schema/minimal-contract.md b/docs/schema/minimal-contract.md index 505fbe9..12decb4 100644 --- a/docs/schema/minimal-contract.md +++ b/docs/schema/minimal-contract.md @@ -1,13 +1,13 @@ # Minimal Contract & FLUID at a Glance -Two views of a contract on the latest stable schema, **0.7.5**: the smallest file that validates, and a one-screen map of every top-level block. +Two views of a contract on the latest stable schema, **0.7.6**: the smallest file that validates, and a one-screen map of every top-level block. ## Minimal Valid Contract -The smallest file that passes JSON Schema validation against 0.7.5 β€” every other block is opt-in: +The smallest file that passes JSON Schema validation against 0.7.6 β€” every other block is opt-in: ```yaml -fluidVersion: "0.7.5" +fluidVersion: "0.7.6" kind: DataProduct id: demo.bronze.hello_world name: "Hello World" @@ -25,7 +25,7 @@ exposes: location: { path: "./hello.parquet" } ``` -It validates against `fluid-schema-0.7.5.json` with any JSON Schema Draft 2020-12 validator, and with the reference implementation (`fluid validate` prints `βœ… Valid FLUID contract (schema v0.7.5)`). +It validates against `fluid-schema-0.7.6.json` with any JSON Schema Draft 2020-12 validator, and with the reference implementation (`fluid validate` prints `βœ… Valid FLUID contract (schema v0.7.6)`). **What is required:** the six top-level keys above; inside `metadata`, only `owner` (the schema requires no member of `owner`, but name a `team`); and inside each expose, `exposeId`, `kind`, `contract` (with `schema` or `openapiRef`) and `binding` (with `platform`, `format` and `location`). @@ -35,10 +35,10 @@ See the [**Examples**](/fluid/examples/) for the step-by-step progression from t ## FLUID at a Glance -Every top-level block in 0.7.5, with the version each first appeared in (in its current form). `[req]` marks required blocks. This is a **map, not a contract** β€” the placeholders are not valid values. +Every top-level block in 0.7.6, with the version each first appeared in (in its current form). `[req]` marks required blocks. This is a **map, not a contract** β€” the placeholders are not valid values. ```text -fluidVersion: "0.7.5" [req] the schema version this file declares +fluidVersion: "0.7.6" [req] the schema version this file declares kind: DataProduct [req] DataProduct | MLPipeline id: domain.layer.name [req] globally unique product id name: "Human-readable name" [req] display name @@ -78,7 +78,7 @@ environments { dev: {...}, staging: {...}, prod: {...} } lifecycle { state, retention, deprecationPolicy } docs { homepage, runbook, dictionary, changeLog } -0.7.6 preview only (fluidVersion "0.7.6"): +New in 0.7.6 (fluidVersion "0.7.6"): packaging { mode, pool, containers } consumers [ { name, type, ... } ] ``` diff --git a/docs/schema/specification.md b/docs/schema/specification.md index 54d7835..9cbb174 100644 --- a/docs/schema/specification.md +++ b/docs/schema/specification.md @@ -1,7 +1,7 @@ # FLUID Specification ::: tip Versions -**Latest stable: 0.7.5. Preview: 0.7.6.** This page specifies the structure of a FLUID document as defined by the **0.7.5** JSON Schema. For every version's schema and generated field-by-field reference, see [**Versions**](/fluid/schema/versions); for what is new in the preview, see [0.7.6 (preview)](/fluid/releases/0.7.6). +**Latest stable: 0.7.6. Preview: 0.7.7.** This page specifies the structure of a FLUID document as defined by the **0.7.5** JSON Schema; the tables in section 1 have not been rewritten for 0.7.6, which is additive over 0.7.5 and adds the members listed in its [release notes](/fluid/releases/0.7.6). For every version's schema and generated field-by-field reference, see [**Versions**](/fluid/schema/versions); for the preview, see [0.7.7](/fluid/releases/0.7.7). ::: FLUID (Federated Layered Unified Interchange Definition) is an open, declarative specification for **data products**, written in YAML or JSON and kept in version control. It is not a platform or a single tool: it is a shared language that tools read to build, deploy, govern and serve a data product. @@ -55,7 +55,7 @@ Because the schema is chosen by the document's own `fluidVersion`, a later schem ### Known interoperability issue: one `pattern` is not an ECMA-262 regular expression -Draft 2020-12 says a `pattern` SHOULD be a valid ECMA-262 regular expression. In the 0.7.2 to 0.7.6 schemas, the second alternative of the column `type` (`$defs/column/properties/type`) begins with the inline flag `(?i)`, which ECMA-262 does not have: JavaScript's `RegExp` rejects it as an invalid group. Python's `re`, which the conformance tooling in this repository uses, accepts it and matches case-insensitively. A validator that compiles patterns as ECMA-262 may therefore refuse that pattern, or the schema, where a Python-based validator does not. This is recorded here rather than resolved: changing the pattern would be a schema change, and schemas are changed upstream (see [CONTRIBUTING.md](https://github.com/open-data-protocol/fluid/blob/main/CONTRIBUTING.md)). +Draft 2020-12 says a `pattern` SHOULD be a valid ECMA-262 regular expression. In the 0.7.2 to 0.7.7 schemas, the second alternative of the column `type` (`$defs/column/properties/type`) begins with the inline flag `(?i)`, which ECMA-262 does not have: JavaScript's `RegExp` rejects it as an invalid group. Python's `re`, which the conformance tooling in this repository uses, accepts it and matches case-insensitively. A validator that compiles patterns as ECMA-262 may therefore refuse that pattern, or the schema, where a Python-based validator does not. This is recorded here rather than resolved: changing the pattern would be a schema change, and schemas are changed upstream (see [CONTRIBUTING.md](https://github.com/open-data-protocol/fluid/blob/main/CONTRIBUTING.md)). --- diff --git a/docs/schema/versions.md b/docs/schema/versions.md index 6658993..829f5cf 100644 --- a/docs/schema/versions.md +++ b/docs/schema/versions.md @@ -2,13 +2,14 @@ Every published version of the FLUID JSON Schema. Point your validator at a version's JSON Schema; read its generated HTML reference for the field-by-field detail. -- **Latest stable: 0.7.5.** Use it for new contracts. -- **Preview: 0.7.6.** Published so that it can be read and validated against. It can still change before it is promoted. See [Stable and preview versions](#stable-and-preview-versions). +- **Latest stable: 0.7.6.** Use it for new contracts. +- **Preview: 0.7.7.** Published so that it can be read and validated against. It can still change before it is promoted. See [Stable and preview versions](#stable-and-preview-versions). | Version | Status | JSON Schema | HTML Reference | |---|---|---|---| -| **0.7.6** | **Preview** | [`fluid-schema-0.7.6.json`](/fluid/schema/fluid-schema-0.7.6.json) | [`0.7.6/fluid-spec.html`](/fluid/specs/0.7.6/fluid-spec.html) | -| **0.7.5** | **Stable (latest)** | [`fluid-schema-0.7.5.json`](/fluid/schema/fluid-schema-0.7.5.json) | [`0.7.5/fluid-spec.html`](/fluid/specs/0.7.5/fluid-spec.html) | +| **0.7.7** | **Preview** | [`fluid-schema-0.7.7.json`](/fluid/schema/fluid-schema-0.7.7.json) | [`0.7.7/fluid-spec.html`](/fluid/specs/0.7.7/fluid-spec.html) | +| **0.7.6** | **Stable (latest)** | [`fluid-schema-0.7.6.json`](/fluid/schema/fluid-schema-0.7.6.json) | [`0.7.6/fluid-spec.html`](/fluid/specs/0.7.6/fluid-spec.html) | +| 0.7.5 | Stable | [`fluid-schema-0.7.5.json`](/fluid/schema/fluid-schema-0.7.5.json) | [`0.7.5/fluid-spec.html`](/fluid/specs/0.7.5/fluid-spec.html) | | 0.7.4 | Stable | [`fluid-schema-0.7.4.json`](/fluid/schema/fluid-schema-0.7.4.json) | [`0.7.4/fluid-spec.html`](/fluid/specs/0.7.4/fluid-spec.html) | | 0.7.3 | Stable | [`fluid-schema-0.7.3.json`](/fluid/schema/fluid-schema-0.7.3.json) | [`0.7.3/fluid-spec.html`](/fluid/specs/0.7.3/fluid-spec.html) | | 0.7.2 | Stable | [`fluid-schema-0.7.2.json`](/fluid/schema/fluid-schema-0.7.2.json) | [`0.7.2/fluid-spec.html`](/fluid/specs/0.7.2/fluid-spec.html) | @@ -35,13 +36,13 @@ For what changed between any two versions, see the [**Changelog**](/fluid/schema ### What "preview" means for you -- **A contract opts in explicitly**, by declaring `fluidVersion: "0.7.6"`. Nothing else selects a preview. -- **Expect to re-validate.** The compatibility gate compares two *versions*. It cannot compare one revision of the 0.7.6 file with an earlier revision of the same file, so a contract written against an earlier revision of 0.7.6 may need changes after a later one. The schemas are authored in the reference implementation, [forge-cli](https://github.com/Agenticstiger/forge-cli), where `fluid_build/schemas/fluid-schema-0.7.6.json` has changed several times since 0.7.6 opened; its history is the record of those changes. -- **Promotion** makes a preview the latest stable version and is announced in its release notes. 0.7.5 went through the same path: it was an opt-in preview in forge-cli from 0.9.0 and was promoted to stable in forge-cli 0.12.0, when 0.7.6 opened as the next preview. +- **A contract opts in explicitly**, by declaring `fluidVersion: "0.7.7"`. Nothing else selects a preview. +- **Expect to re-validate.** The compatibility gate compares two *versions*. It cannot compare one revision of the 0.7.6 file with an earlier revision of the same file, so a contract written against an earlier revision of 0.7.7 may need changes after a later one. The schemas are authored in the reference implementation, [forge-cli](https://github.com/Agenticstiger/forge-cli), where `fluid_build/schemas/fluid-schema-0.7.7.json` can change while 0.7.7 is a preview; its history is the record of those changes. +- **Promotion** makes a preview the latest stable version and is announced in its release notes. 0.7.5 went through the same path: it was an opt-in preview in forge-cli from 0.9.0 and was promoted to stable in forge-cli 0.12.0, when 0.7.6 opened as the next preview. 0.7.6 followed: promoted in forge-cli 0.26.0, when 0.7.7 opened as the next preview. ### How the reference implementation treats a preview (non-normative) -[`data-product-forge`](/fluid/concepts/forge-cli) bundles 0.7.6 and validates a contract that declares it (`βœ… Valid FLUID contract (schema v0.7.6)`), but never chooses it on its own: its newest *stable* bundled version is its default, and `fluid init --quickstart` writes `fluidVersion: 0.7.5`. This describes one implementation. The standard's rule is only the one in [Validation semantics](/fluid/schema/specification#validation-semantics): a document is validated against the schema of the version it declares. +[`data-product-forge`](/fluid/concepts/forge-cli) bundles 0.7.7 and validates a contract that declares it (`βœ… Valid FLUID contract (schema v0.7.7)`), but never chooses it on its own: its newest *stable* bundled version, 0.7.6, is its default. This describes one implementation. The standard's rule is only the one in [Validation semantics](/fluid/schema/specification#validation-semantics): a document is validated against the schema of the version it declares. --- diff --git a/schema-diffs/README.md b/schema-diffs/README.md index 087b20f..f0393bb 100644 --- a/schema-diffs/README.md +++ b/schema-diffs/README.md @@ -15,9 +15,10 @@ This directory contains human-readable diffs between consecutive versions of the - [0.7.2 β†’ 0.7.3](diff-0.7.2-to-0.7.3.md) - [0.7.3 β†’ 0.7.4](diff-0.7.3-to-0.7.4.md) - [0.7.4 β†’ 0.7.5](diff-0.7.4-to-0.7.5.md) -- [0.7.5 β†’ 0.7.6](diff-0.7.5-to-0.7.6.md) (0.7.6 is a preview and may still change) +- [0.7.5 β†’ 0.7.6](diff-0.7.5-to-0.7.6.md) +- [0.7.6 β†’ 0.7.7](diff-0.7.6-to-0.7.7.md) (0.7.7 is a preview and may still change) --- -**Versions tracked:** 13 -**Diff files generated:** 12 +**Versions tracked:** 14 +**Diff files generated:** 13 diff --git a/schema-diffs/diff-0.7.5-to-0.7.6.md b/schema-diffs/diff-0.7.5-to-0.7.6.md index 1bcfc0e..1846739 100644 --- a/schema-diffs/diff-0.7.5-to-0.7.6.md +++ b/schema-diffs/diff-0.7.5-to-0.7.6.md @@ -1,8 +1,8 @@ # Schema Changes: 0.7.5 β†’ 0.7.6 -**Total changes:** 40 -- βœ… Added: 27 -- ❌ Removed: 0 +**Total changes:** 54 +- βœ… Added: 40 +- ❌ Removed: 1 - πŸ“ Modified: 13 @@ -114,6 +114,27 @@ } ``` +### `$defs.binding.properties.serialization` +```json +{ + type: "object" + additionalProperties: false + description: "NEW in v0.7.6: How message VALUES (and keys) are serialized on the wire. A Kafka binding today decla..." + properties: { + value: { + type: "string" + enum: [...6 items...] + description: "How message values are serialized." + } + key: { + type: "string" + enum: [...6 items...] + description: "How message keys are serialized, for a topic whose keys are not opaque bytes." + } + } +} +``` + ### `$defs.bindingGovernance.properties.lakeFormation.properties.bucketPolicy` ```json { @@ -144,6 +165,52 @@ } ``` +### `$defs.bindingLocation.properties.host` +```json +{ + type: "string" + description: "NEW in v0.7.6: Host or endpoint; for a Kafka binding, the bootstrap servers. The ODCS server mapper ..." +} +``` + +### `$defs.column.properties.quality` +```json +{ + type: "array" + description: "NEW in v0.7.6: Column-level data quality rules, in ODCS's own shape β€” the array that is written stra..." + items: { + type: "object" + minProperties: 1 + properties: { + id: { + type: "string" + pattern: "^[A-Za-z0-9_-]+$" + description: "Stable rule identifier, unique within this column's rules. The pattern is ODCS `$defs.StableId`. Rul..." + } + type: { + type: "string" + enum: [...4 items...] + description: "ODCS quality check type. text = prose; library = one of ODCS's predefined metrics; sql = a query who..." + } + metric: { + type: "string" + enum: [...5 items...] + description: "type: library only. The five predefined metrics ODCS v3.1.0 defines; a name outside this list is not..." + } + arguments: { + type: "object" + description: "type: library only. Extra arguments for the metric β€” `pattern` (a regex) or `validValues` (a list) f..." + } + query: { + type: "string" + description: "type: sql only. The query whose result the comparison operator is applied to." + } + ... (6 more) + } + } +} +``` + ### `$defs.consumeRef.dependentRequired` ```json { @@ -151,6 +218,23 @@ } ``` +### `$defs.consumeRef.properties.contractId` +```json +{ + type: "string" + description: "NEW in v0.7.6: The upstream contract's OWN id, for an upstream that has one of its own β€” a product n..." +} +``` + +### `$defs.consumeRef.properties.contractVersion` +```json +{ + $ref: "#/$defs/semver" + description: "NEW in v0.7.6: The exact upstream contract version this product was built and tested against, within..." + examples: ["2.1.0"] +} +``` + ### `$defs.consumeRef.properties.upstreamDigest` ```json { @@ -215,6 +299,38 @@ } ``` +### `$defs.descriptionObject` +```json +{ + type: "object" + additionalProperties: false + description: "NEW in v0.7.6: The structured form of the root `description`. The three field names are Bitol's β€” OD..." + properties: { + purpose: { + type: "string" + description: "What this data product is for." + } + usage: { + type: "string" + description: "How a consumer is expected to use it." + } + limitations: { + type: "string" + description: "What it must not be used for, and what it does not cover." + } + } +} +``` + +### `$defs.exposeContract.properties.id` +```json +{ + type: "string" + description: "NEW in v0.7.6: An explicit, stable contract id, used verbatim as the ODCS `id` and as the ODPS outpu..." + minLength: 1 +} +``` + ### `$defs.exposeLifecycle` ```json { @@ -459,6 +575,44 @@ } ``` +### `$defs.teamMember` +```json +{ + type: "object" + additionalProperties: false + description: "NEW in v0.7.6: One member of `metadata.team.members`. The field names and the username-only required..." + required: ["username"] + properties: { + username: { + type: "string" + description: "The member's identifier in the organisation's directory \u2014 the one field Bitol requires." + } + name: { + type: "string" + description: "Display name." + } + role: { + type: "string" + description: "The role this member holds on this product, e.g. 'owner', 'steward', 'data engineer'. Free-form: Bit..." + } + email: { + type: "string" + format: "email" + } + } +} +``` + +### `$defs.trigger.allOf` +```json +[{"if": {"type": "object", "required": ["type"], "properties": {"type": {"const": "continuous"}}}, "then": {"allOf": [{"description": "cron is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove cron, or set type: schedule.", "not": {"required": ["cron"]}}, {"description": "schedule is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove schedule, or set type: schedule.", "not": {"required": ["schedule"]}}, {"description": "datasets is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove datasets, or set type: dataset.", "not": {"required": ["datasets"]}}, {"description": "event is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove event, or set type: event.", "not": {"required": ["event"]}}, {"description": "timetable is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove timetable, or set type: timetable.", "not": {"required": ["timetable"]}}]}}] +``` + +### `$defs.trigger.properties.type.description` +```json +"NEW in v0.7.6: continuous, for a build that runs as a long-lived job on its own platform, such as an..." +``` + ### `properties.consumers` ```json { @@ -470,6 +624,38 @@ } ``` +### `properties.description.description` +```json +"NEW in v0.7.6: Either a plain string β€” unchanged meaning, it becomes `purpose` on export β€” or the st..." +``` + +### `properties.description.oneOf` +```json +[{"type": "string", "description": "Prose description. Exported as `description.purpose`."}, {"$ref": "#/$defs/descriptionObject"}] +``` + +### `properties.metadata.properties.team` +```json +{ + type: "object" + additionalProperties: false + description: "NEW in v0.7.6: The team behind this product, as a roster rather than a single contact. `metadata.own..." + properties: { + name: { + type: "string" + description: "Team name, e.g. 'Sales Analytics'." + } + members: { + type: "array" + description: "The people on the team, each with the role they hold on this product." + items: { + $ref: "#/$defs/teamMember" + } + } + } +} +``` + ### `properties.packaging` ```json { @@ -478,6 +664,13 @@ } ``` +## ❌ Removed Properties + +### `properties.description.type` +```json +"string" +``` + ## πŸ“ Modified Properties ### `$defs.acquisitionSink.properties.catalog.enum` @@ -576,42 +769,42 @@ ["allow", "deny", "mask"] ``` -### `$id` +### `$defs.trigger.properties.type.enum` **Before:** ```json -"https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.5.json" +[...7 items...] ``` **After:** ```json -"https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json" +[...8 items...] ``` -### `description` +### `$id` **Before:** ```json -"FLUID Data Product contract (v0.7.5). Streaming Kafkaβ†’Iceberg Sink & Confluent Tableflow Release. - -πŸ”₯..." +"https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.5.json" ``` **After:** ```json -"FLUID contract schema 0.7.6 β€” the current PREVIEW version, opened when 0.7.5 was promoted to stable...." +"https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json" ``` -### `properties.fluidVersion.description` +### `description` **Before:** ```json -"Contract schema version. Accepts '0.7.3' (source-aligned data products + acquisition pattern + inges..." +"FLUID Data Product contract (v0.7.5). Streaming Kafkaβ†’Iceberg Sink & Confluent Tableflow Release. + +πŸ”₯..." ``` **After:** ```json -"Contract schema version. Accepts '0.7.3' (source-aligned data products + acquisition pattern + inges..." +"FLUID contract schema 0.7.6 β€” opened as the preview when 0.7.5 was promoted to stable, and stable it..." ``` ### `properties.fluidVersion.enum` @@ -635,5 +828,5 @@ **After:** ```json -"FLUID 0.7.6 \u2014 Declarative Packaging Modes (preview)" +"FLUID 0.7.6 \u2014 Declarative Packaging Modes" ``` diff --git a/schema-diffs/diff-0.7.6-to-0.7.7.md b/schema-diffs/diff-0.7.6-to-0.7.7.md new file mode 100644 index 0000000..908fa83 --- /dev/null +++ b/schema-diffs/diff-0.7.6-to-0.7.7.md @@ -0,0 +1,355 @@ +# Schema Changes: 0.7.6 β†’ 0.7.7 + +**Total changes:** 22 +- βœ… Added: 16 +- ❌ Removed: 1 +- πŸ“ Modified: 5 + +--- + +## βœ… Added Properties + +### `$defs.column.properties.element` +```json +{ + $ref: "#/$defs/nestedType" +} +``` + +### `$defs.column.properties.fields` +```json +{ + type: "array" + minItems: 1 + items: { + $ref: "#/$defs/column" + } + description: "Typed child columns of a struct/row/record." +} +``` + +### `$defs.column.properties.key` +```json +{ + $ref: "#/$defs/nestedType" +} +``` + +### `$defs.column.properties.value` +```json +{ + $ref: "#/$defs/nestedType" +} +``` + +### `$defs.embeddedLogicPattern.dependentRequired` +```json +{ + mapping: ["stream"] +} +``` + +### `$defs.embeddedLogicPattern.if` +```json +{ + not: { + required: ["stream", "parameters"] + properties: { + parameters: { + required: ["flink"] + properties: { + flink: { + required: ["management"] + } + } + } + } + } +} +``` + +### `$defs.embeddedLogicPattern.properties.mapping` +```json +{ + $ref: "#/$defs/recordMapping" + description: "NEW in v0.7.7 (preview): How the stream's records become the output's rows. Requires stream, and rep..." +} +``` + +### `$defs.embeddedLogicPattern.properties.stream` +```json +{ + $ref: "#/$defs/streamSource" + description: "NEW in v0.7.7 (preview): The stream the build reads. On a managed Flink build (parameters.flink.mana..." +} +``` + +### `$defs.embeddedLogicPattern.then` +```json +{ + required: ["sql"] +} +``` + +### `$defs.nestedType` +```json +{ + type: "object" + additionalProperties: false + required: ["type"] + properties: { + type: { + $ref: "#/$defs/column/properties/type" + } + required: { + type: "boolean" + } + fields: { + type: "array" + minItems: 1 + items: { + $ref: "#/$defs/column" + } + } + element: { + $ref: "#/$defs/nestedType" + } + key: { + $ref: "#/$defs/nestedType" + } + ... (1 more) + } +} +``` + +### `$defs.recordExplode` +```json +{ + type: "object" + additionalProperties: false + description: "NEW in v0.7.7 (preview): One output row per element of an array. A record whose array is null or emp..." + required: ["path", "as"] + properties: { + path: { + type: "string" + pattern: "^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*|\['[^'\\\u0000-\u001f\u007f]{1,256}'\])*(?![\s\S]..." + description: "The array: a stream column or a nested field, such as items or order.lines." + } + as: { + type: "string" + pattern: "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])" + description: "The element's name in mapping paths: item.sku for a field of a struct element, item for a scalar ele..." + } + } +} +``` + +### `$defs.recordMapping` +```json +{ + type: "object" + additionalProperties: false + description: "NEW in v0.7.7 (preview): How the stream's records become the output's rows. An output column the map..." + properties: { + explode: { + $ref: "#/$defs/recordExplode" + } + columns: { + type: "array" + minItems: 1 + items: { + $ref: "#/$defs/recordMappingColumn" + } + description: "Output columns, or nested fields of them, that are read from another path, renamed or converted." + } + } +} +``` + +### `$defs.recordMappingColumn` +```json +{ + type: "object" + additionalProperties: false + description: "NEW in v0.7.7 (preview): Where one output column, or one nested field of one, is read from." + required: ["name", "from"] + properties: { + name: { + type: "string" + pattern: "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*)*(?![\\s\\S])" + description: "The output column, or a field of a struct output column as a dotted path (customer.id). Mapping a ne..." + } + from: { + type: "string" + pattern: "^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*|\['[^'\\\u0000-\u001f\u007f]{1,256}'\])*(?![\s\S]..." + description: "The source: a stream column (value, key or metadata), a nested field (customer.address.country), a m..." + } + cast: { + $ref: "#/$defs/column/properties/type" + description: "Convert the source to this type: binary to string (UTF-8), or a widening (int to long, float to doub..." + } + description: { + type: "string" + } + } +} +``` + +### `$defs.streamKey` +```json +{ + type: "object" + additionalProperties: false + description: "NEW in v0.7.7 (preview): The record key, read as columns next to the value's. A record without a key..." + required: ["format", "schema"] + properties: { + format: { + enum: ["raw", "json", "avro-confluent"] + description: "raw: the key's bytes as one column, of type string (UTF-8) or binary. json and avro-confluent: a key..." + } + schema: { + type: "array" + minItems: 1 + items: { + $ref: "#/$defs/column" + } + description: "The key's fields, as columns: exactly one for raw." + } + prefix: { + type: "string" + pattern: "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])" + description: "Put in front of every key column's name, so a key field can share its name with a value field (a Deb..." + } + } +} +``` + +### `$defs.streamMetadataColumn` +```json +{ + type: "object" + additionalProperties: false + description: "NEW in v0.7.7 (preview): One piece of Kafka record metadata, read as a column. Its type is fixed by ..." + required: ["name", "from"] + properties: { + name: { + type: "string" + minLength: 1 + description: "The column's name, unique among the stream's value, key and metadata columns." + } + from: { + enum: [...7 items...] + description: "The metadata the column reads." + } + description: { + type: "string" + } + } +} +``` + +### `$defs.streamSource` +```json +{ + type: "object" + additionalProperties: false + description: "NEW in v0.7.7 (preview): The Kafka records a streaming build reads, described apart from the engine ..." + required: [...4 items...] + properties: { + kind: { + const: "kafka" + description: "The streaming system. Only kafka in this preview." + } + topic: { + type: "string" + pattern: "^(?!\\.\\.?(?![\\s\\S]))[A-Za-z0-9._-]{1,249}(?![\\s\\S])" + description: "One Kafka topic, by name: Kafka's legal characters (not . or ..), no pattern and no list." + } + format: { + enum: ["json", "avro-confluent"] + description: "How record values are encoded. json: one JSON object per record; a field the schema does not declare..." + } + schema: { + type: "array" + minItems: 1 + items: { + $ref: "#/$defs/column" + } + description: "The record value's fields, as columns. Nested data uses fields (struct), element (array) and key/val..." + } + key: { + $ref: "#/$defs/streamKey" + } + ... (1 more) + } +} +``` + +## ❌ Removed Properties + +### `$defs.embeddedLogicPattern.required` +```json +["sql"] +``` + +## πŸ“ Modified Properties + +### `$id` + +**Before:** +```json +"https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json" +``` + +**After:** +```json +"https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json" +``` + +### `description` + +**Before:** +```json +"FLUID contract schema 0.7.6 β€” opened as the preview when 0.7.5 was promoted to stable, and stable it..." +``` + +**After:** +```json +"Opt-in preview. Adds recursive struct fields, array elements and map key/value types. Stable 0.7.6 i..." +``` + +### `properties.fluidVersion.description` + +**Before:** +```json +"Contract schema version. Accepts '0.7.3' (source-aligned data products + acquisition pattern + inges..." +``` + +**After:** +```json +"Contract schema version. Accepts '0.7.3' (source-aligned data products + acquisition pattern + inges..." +``` + +### `properties.fluidVersion.enum` + +**Before:** +```json +[...4 items...] +``` + +**After:** +```json +[...5 items...] +``` + +### `title` + +**Before:** +```json +"FLUID 0.7.6 \u2014 Declarative Packaging Modes" +``` + +**After:** +```json +"FLUID 0.7.7 \u2014 Preview: recursive streaming types" +``` diff --git a/schema/fluid-schema-0.7.6.json b/schema/fluid-schema-0.7.6.json index 3168d07..c54e931 100644 --- a/schema/fluid-schema-0.7.6.json +++ b/schema/fluid-schema-0.7.6.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json", - "title": "FLUID 0.7.6 β€” Declarative Packaging Modes (preview)", - "description": "FLUID contract schema 0.7.6 β€” the current PREVIEW version, opened when 0.7.5 was promoted to stable.\n\nπŸ”₯ NEW in v0.7.6:\nβ€’ Declarative packaging modes β€” a top-level `packaging` block (plus a per-exposure `binding.packaging` override) declares per-container ownership for the IaC emit: `isolated` (this product creates and owns the container) vs `shared` (a pre-existing, platform-owned pool referenced via an OpenTofu data source), with a per-kind `containers` override map for hybrid tiers and a `pool` id for tenancy/cost attribution (RFC-packaging-modes.md).\nβ€’ Backward compatible with v0.7.5 β€” every 0.7.5 contract validates as 0.7.6 unchanged; an ABSENT packaging block keeps the legacy byte-identical emit.", + "title": "FLUID 0.7.6 β€” Declarative Packaging Modes", + "description": "FLUID contract schema 0.7.6 β€” opened as the preview when 0.7.5 was promoted to stable, and stable itself since 0.7.7 opened.\n\nπŸ”₯ NEW in v0.7.6:\nβ€’ Declarative packaging modes β€” a top-level `packaging` block (plus a per-exposure `binding.packaging` override) declares per-container ownership for the IaC emit: `isolated` (this product creates and owns the container) vs `shared` (a pre-existing, platform-owned pool referenced via an OpenTofu data source), with a per-kind `containers` override map for hybrid tiers and a `pool` id for tenancy/cost attribution (RFC-packaging-modes.md).\nβ€’ Optional Bitol-shaped fields the exporters already wanted, each one opt-in: `metadata.team` (a roster β€” name plus members, every member a `$defs.teamMember` whose only required field is `username`), the root `description` as `{purpose, usage, limitations}` as well as a plain string, `consumes[].contractId` (an upstream that has a contract id of its own), `consumes[].contractVersion` (the exact upstream version this product was built against, inside that range), `exposes[].contract.id` (used verbatim as the ODCS `id` and the ODPS output port `contractId` instead of the derived `.`; two exposes may not share one) and `binding.location.host` plus `binding.serialization.{value, key}` for Kafka. A contract that sets none of them validates and exports exactly as it does today.\nβ€’ `exposes[].contract.schema[].quality` β€” column-level data quality rules in ODCS's own shape. The ODCS exporter always replayed this key verbatim and read it back on import, but `$defs.column` is closed and did not define it, so the one way to publish a hand-written column rule was to write a contract `fluid validate` refuses. Optional: a column without it exports exactly as it does today.\nβ€’ Backward compatible with v0.7.5 β€” every 0.7.5 contract validates as 0.7.6 unchanged; an ABSENT packaging block keeps the legacy byte-identical emit.", "type": "object", "additionalProperties": false, "required": [ @@ -105,7 +105,7 @@ "0.7.5", "0.7.6" ], - "description": "Contract schema version. Accepts '0.7.3' (source-aligned data products + acquisition pattern + ingestion engines) or '0.7.4' (adds the MCP output-port gateway + runtime agentPolicy enforcement). 0.7.4 is fully backward-compatible with 0.7.3 β€” every 0.7.3 contract validates as 0.7.4 unchanged. '0.7.6' is the current preview.", + "description": "Contract schema version. Accepts '0.7.3' (source-aligned data products + acquisition pattern + ingestion engines) or '0.7.4' (adds the MCP output-port gateway + runtime agentPolicy enforcement). 0.7.4 is fully backward-compatible with 0.7.3 β€” every 0.7.3 contract validates as 0.7.4 unchanged.", "examples": [ "0.7.4", "0.7.3" @@ -128,7 +128,16 @@ "minLength": 1 }, "description": { - "type": "string" + "description": "NEW in v0.7.6: Either a plain string β€” unchanged meaning, it becomes `purpose` on export β€” or the structured object. While this was a string, an export of a valid contract could only ever carry `description.purpose`: `usage` and `limitations` had no home a contract could declare, so every ODCS and ODPS export left them empty. `fluid odps import` restores the object to the root `description` when the schema version the imported contract declares accepts an object, and also keeps a verbatim copy in `extensions.odps.metadata.description` so the ODPS leg stays an exact fixed point. All three fields therefore survive a FLUID β†’ ODPS β†’ FLUID cycle and reach both the ODCS and the ODPS export afterwards. Importing against a version whose root `description` is a string still roots `purpose` alone, with the object recoverable from that bucket.", + "oneOf": [ + { + "type": "string", + "description": "Prose description. Exported as `description.purpose`." + }, + { + "$ref": "#/$defs/descriptionObject" + } + ] }, "domain": { "type": "string", @@ -234,6 +243,24 @@ } } }, + "team": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.6: The team behind this product, as a roster rather than a single contact. `metadata.owner` holds one team name and one mailbox and carries no role, so a product with a data owner, a steward and two engineers had nowhere to say so and the exported ODCS/ODPS team block came out empty. The field names are Bitol's, so an imported team round-trips without a translation. Optional: a contract without it exports exactly as it does today.", + "properties": { + "name": { + "type": "string", + "description": "Team name, e.g. 'Sales Analytics'." + }, + "members": { + "type": "array", + "description": "The people on the team, each with the role they hold on this product.", + "items": { + "$ref": "#/$defs/teamMember" + } + } + } + }, "createdAt": { "type": "string", "format": "date-time" @@ -349,6 +376,51 @@ } }, "$defs": { + "descriptionObject": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.6: The structured form of the root `description`. The three field names are Bitol's β€” ODPS v1.1.0 `$defs.Description` and the ODCS v3.1.0 root `description` both use exactly purpose/usage/limitations β€” so import and export are a pass-through rather than a translation.", + "properties": { + "purpose": { + "type": "string", + "description": "What this data product is for." + }, + "usage": { + "type": "string", + "description": "How a consumer is expected to use it." + }, + "limitations": { + "type": "string", + "description": "What it must not be used for, and what it does not cover." + } + } + }, + "teamMember": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.6: One member of `metadata.team.members`. The field names and the username-only required list are Bitol's ODCS v3.1.0 / ODPS v1.1.0 `$defs.TeamMember`, copied verbatim so an imported team exports back unchanged.", + "required": [ + "username" + ], + "properties": { + "username": { + "type": "string", + "description": "The member's identifier in the organisation's directory β€” the one field Bitol requires." + }, + "name": { + "type": "string", + "description": "Display name." + }, + "role": { + "type": "string", + "description": "The role this member holds on this product, e.g. 'owner', 'steward', 'data engineer'. Free-form: Bitol does not constrain it." + }, + "email": { + "type": "string", + "format": "email" + } + } + }, "packaging": { "type": "object", "additionalProperties": false, @@ -729,6 +801,11 @@ "type": "object", "additionalProperties": false, "properties": { + "id": { + "type": "string", + "description": "NEW in v0.7.6: An explicit, stable contract id, used verbatim as the ODCS `id` and as the ODPS output port `contractId`. Without it the id is derived as `.`, so renaming the product or the expose silently changes the contract id and breaks every consumer reference to it, and a catalog that requires an id format of its own (a UUID, say) cannot be served at all. Each expose needs its own: two exposes sharing one id is refused by `fluid validate`.", + "minLength": 1 + }, "schemaSignature": { "type": "string", "pattern": "^sha256:[0-9a-fA-F]{64}$", @@ -998,6 +1075,81 @@ } } }, + "quality": { + "type": "array", + "description": "NEW in v0.7.6: Column-level data quality rules, in ODCS's own shape β€” the array that is written straight into `schema[].properties[].quality[]` of the exported ODCS v3.1.0 contract, and the array an imported ODCS column's rules are read back into, inline on the column rather than in the importer's pass-through bucket: the ODPS importer merges a resolved port's `contract` block into its output-port stub and discards the sub-import's root `extensions`, so rules left in that bucket were dropped outright by a FLUID β†’ ODPS β†’ FLUID cycle. The ODCS exporter has always replayed this key verbatim and always read it back on import, but `$defs.column` is `additionalProperties: false` and did not define it, so a column written the way the exporter supports was REFUSED by `fluid validate`: the only way to get a hand-written quality rule into a published contract was to write an invalid contract. Optional and additive β€” a column without it exports exactly as it does today, with the rules forge derives from `required`, the primary-key tag and `validationRules`. Setting it REPLACES those derived rules for that column rather than adding to them, which is what makes an imported ODCS column round-trip byte-for-byte. The authority for an item's contents is ODCS v3.1.0 `$defs.DataQuality`, deliberately not re-declared here (a copy would drift from the vendored schema, and `fluid odcs validate` already checks an emitted document against it); the keys below are the common ones. Every rule should carry an `id`: it is optional in ODCS, but a DQ engine that labels its results by rule id rejects a contract whose rules have none.", + "items": { + "type": "object", + "minProperties": 1, + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]+$", + "description": "Stable rule identifier, unique within this column's rules. The pattern is ODCS `$defs.StableId`. Rules forge derives itself are named `-`." + }, + "type": { + "type": "string", + "enum": [ + "text", + "library", + "sql", + "custom" + ], + "description": "ODCS quality check type. text = prose; library = one of ODCS's predefined metrics; sql = a query whose result is compared; custom = a named engine's own check." + }, + "metric": { + "type": "string", + "enum": [ + "nullValues", + "missingValues", + "invalidValues", + "duplicateValues", + "rowCount" + ], + "description": "type: library only. The five predefined metrics ODCS v3.1.0 defines; a name outside this list is not a library rule." + }, + "arguments": { + "type": "object", + "description": "type: library only. Extra arguments for the metric β€” `pattern` (a regex) or `validValues` (a list) for invalidValues, as DataHub's ODCS ingestion source reads them." + }, + "query": { + "type": "string", + "description": "type: sql only. The query whose result the comparison operator is applied to." + }, + "engine": { + "type": "string", + "description": "type: custom only. The engine that runs the check, e.g. soda, great-expectations, dbt." + }, + "implementation": { + "description": "type: custom only. The check as that engine expresses it." + }, + "dimension": { + "type": "string", + "enum": [ + "accuracy", + "completeness", + "conformity", + "consistency", + "coverage", + "timeliness", + "uniqueness" + ], + "description": "The data quality dimension the rule measures." + }, + "description": { + "type": "string" + }, + "severity": { + "type": "string", + "description": "Severity of a failure. ODCS types this as a free string and documents info / warning / error by example." + }, + "unit": { + "type": "string", + "description": "Unit the comparison is in, usually rows (the default) or percent." + } + } + } + }, "tags": { "$ref": "#/$defs/tags", "description": "Field-level tags for categorization and data governance." @@ -1754,6 +1906,37 @@ } ] } + }, + "serialization": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.6: How message VALUES (and keys) are serialized on the wire. A Kafka binding today declares only `format: kafka_topic`, which tells a consumer nothing about whether the values are Avro or JSON β€” the one thing it has to know to read the topic at all. The first four values are the message formats ODCS v3.1.0 documents for its own Kafka server `format` field; the two `-confluent` values name those same wire formats carried through a Confluent schema registry, spelled as Apache Flink spells them.", + "properties": { + "value": { + "type": "string", + "enum": [ + "json", + "avro", + "protobuf", + "xml", + "avro-confluent", + "json-confluent" + ], + "description": "How message values are serialized." + }, + "key": { + "type": "string", + "enum": [ + "json", + "avro", + "protobuf", + "xml", + "avro-confluent", + "json-confluent" + ], + "description": "How message keys are serialized, for a topic whose keys are not opaque bytes." + } + } } }, "allOf": [ @@ -1884,6 +2067,10 @@ "type": "string", "description": "API gateway endpoint" }, + "host": { + "type": "string", + "description": "NEW in v0.7.6: Host or endpoint; for a Kafka binding, the bootstrap servers. The ODCS server mapper already reads `host` and falls back to `account`, so until now the only valid way to set the bootstrap servers was a field named for something else β€” and with neither set, the exported ODCS server is the placeholder `${KAFKA_BOOTSTRAP_SERVERS}`. Usually set per environment in an overlay rather than in the base contract." + }, "baseUrl": { "type": "string", "format": "uri", @@ -2773,9 +2960,20 @@ "$ref": "#/$defs/identifier", "description": "The upstream exposeId being consumed." }, + "contractId": { + "type": "string", + "description": "NEW in v0.7.6: The upstream contract's OWN id, for an upstream that has one of its own β€” a product not built with FLUID, registered in a catalog under an id FLUID did not derive. The canonical-port builder already prefers an explicit contractId over the derived `.`; until now `consumes[]` was closed, so there was no way to supply it and the derived id was exported as if the upstream answered to it." + }, "versionConstraint": { "$ref": "#/$defs/semverRange" }, + "contractVersion": { + "$ref": "#/$defs/semver", + "description": "NEW in v0.7.6: The exact upstream contract version this product was built and tested against, within `versionConstraint`. Exported as the ODPS `inputPorts[].version`, and as the version of the matching `outputPorts[].inputContracts[]` entry. Without it the export derives the range's lower bound; for a range it cannot read it falls back to 1.0.0, because ODPS v1.0.0 requires a version on every input port and the field cannot simply be omitted. `versionConstraint` stays the range this product accepts; this is the single version its exported lineage claims. `fluid validate` does not check that it falls inside `versionConstraint`.", + "examples": [ + "2.1.0" + ] + }, "qosExpectations": { "type": "object", "additionalProperties": false, @@ -4316,8 +4514,10 @@ "dependency", "dataset", "schedule_and_dataset", - "timetable" - ] + "timetable", + "continuous" + ], + "description": "NEW in v0.7.6: continuous, for a build that runs as a long-lived job on its own platform, such as an engine: flink job on the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, or a kafka-connect or debezium connector. The platform keeps the job running, so no scheduler starts runs: fluid generate schedule, fluid schedule-sync and the fluid apply DAGs leave the build out. A continuous trigger takes no other trigger, so cron, schedule, datasets, event and timetable are refused next to it, as Databricks Jobs refuses a continuous trigger next to any other." }, "schedule": { "type": "string" @@ -4382,7 +4582,66 @@ "type": "object", "description": "Custom timetable configuration" } - } + }, + "allOf": [ + { + "if": { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "continuous" + } + } + }, + "then": { + "allOf": [ + { + "description": "cron is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove cron, or set type: schedule.", + "not": { + "required": [ + "cron" + ] + } + }, + { + "description": "schedule is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove schedule, or set type: schedule.", + "not": { + "required": [ + "schedule" + ] + } + }, + { + "description": "datasets is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove datasets, or set type: dataset.", + "not": { + "required": [ + "datasets" + ] + } + }, + { + "description": "event is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove event, or set type: event.", + "not": { + "required": [ + "event" + ] + } + }, + { + "description": "timetable is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove timetable, or set type: timetable.", + "not": { + "required": [ + "timetable" + ] + } + } + ] + } + } + ] }, "runtime": { "type": "object", diff --git a/schema/fluid-schema-0.7.7.json b/schema/fluid-schema-0.7.7.json new file mode 100644 index 0000000..393a2b9 --- /dev/null +++ b/schema/fluid-schema-0.7.7.json @@ -0,0 +1,6748 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "title": "FLUID 0.7.7 β€” Preview: recursive streaming types", + "description": "Opt-in preview. Adds recursive struct fields, array elements and map key/value types. Stable 0.7.6 is unchanged.", + "type": "object", + "additionalProperties": false, + "required": [ + "fluidVersion", + "kind", + "id", + "name", + "metadata", + "exposes" + ], + "examples": [ + { + "fluidVersion": "0.5.7", + "kind": "DataProduct", + "id": "gold.customer.analytics_360_v1", + "name": "Customer 360 Analytics", + "description": "Unified customer profiles with ML-driven insights", + "domain": "Customer Experience", + "tags": [ + "customer-data", + "analytics", + "gold-layer" + ], + "labels": { + "team": "customer-analytics", + "criticality": "high", + "cost-center": "engineering" + }, + "metadata": { + "layer": "Gold", + "owner": { + "team": "customer-analytics", + "email": "customer-analytics@company.com" + }, + "businessContext": { + "domain": "Customer Experience", + "subdomain": "Customer Intelligence" + } + }, + "exposes": [ + { + "exposeId": "customer_profiles", + "title": "Unified Customer Profiles", + "version": "2.1.0", + "kind": "table", + "tags": [ + "pii", + "customer-facing", + "real-time" + ], + "labels": { + "sensitivity": "high", + "retention": "7-years", + "region": "us-central1" + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "STRING", + "required": true, + "description": "Unique customer identifier", + "sensitivity": "cleartext", + "tags": [ + "identifier", + "primary-key" + ], + "labels": { + "business-name": "Customer ID", + "data-category": "identifier" + } + } + ] + }, + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "company-data", + "dataset": "gold_customer", + "table": "profiles_v1" + } + } + } + ] + } + ], + "properties": { + "extensions": { + "type": "object", + "additionalProperties": true, + "description": "Vendor / plugin-namespaced configuration. Each plugin claims a single sub-key (e.g. customScaffold, customCatalog) and validates its own schema there via the fluid_build.extension_validators entry-point." + }, + "fluidVersion": { + "type": "string", + "pattern": "^\\d+\\.\\d+(\\.\\d+)?$", + "enum": [ + "0.7.3", + "0.7.4", + "0.7.5", + "0.7.6", + "0.7.7" + ], + "description": "Contract schema version. Accepts '0.7.3' (source-aligned data products + acquisition pattern + ingestion engines) or '0.7.4' (adds the MCP output-port gateway + runtime agentPolicy enforcement). 0.7.4 is fully backward-compatible with 0.7.3 β€” every 0.7.3 contract validates as 0.7.4 unchanged. '0.7.7' is the current preview.", + "examples": [ + "0.7.4", + "0.7.3" + ] + }, + "kind": { + "type": "string", + "enum": [ + "DataProduct", + "MLPipeline" + ], + "description": "Kind of contract. 'MLPipeline' added for basic ML support." + }, + "id": { + "$ref": "#/$defs/identifier", + "description": "Stable product identifier, e.g. 'gold.hr.employee_360_v1'." + }, + "name": { + "type": "string", + "minLength": 1 + }, + "description": { + "description": "NEW in v0.7.6: Either a plain string β€” unchanged meaning, it becomes `purpose` on export β€” or the structured object. While this was a string, an export of a valid contract could only ever carry `description.purpose`: `usage` and `limitations` had no home a contract could declare, so every ODCS and ODPS export left them empty. `fluid odps import` restores the object to the root `description` when the schema version the imported contract declares accepts an object, and also keeps a verbatim copy in `extensions.odps.metadata.description` so the ODPS leg stays an exact fixed point. All three fields therefore survive a FLUID β†’ ODPS β†’ FLUID cycle and reach both the ODCS and the ODPS export afterwards. Importing against a version whose root `description` is a string still roots `purpose` alone, with the object recoverable from that bucket.", + "oneOf": [ + { + "type": "string", + "description": "Prose description. Exported as `description.purpose`." + }, + { + "$ref": "#/$defs/descriptionObject" + } + ] + }, + "domain": { + "type": "string", + "description": "Business domain/mesh domain (e.g., 'HR', 'Finance')." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Product-level tags for categorization and discovery." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Product-level labels for metadata and automation." + }, + "metadata": { + "type": "object", + "additionalProperties": false, + "required": [ + "owner" + ], + "properties": { + "provenance": { + "type": "object", + "description": "Generation envelope injected by `fluid forge` / `fluid init` β€” records how, when and by which tool/command the contract was generated. Machine-written (not user-authored); additive metadata that does not affect the data product's semantic contract.", + "properties": { + "schema_version": { + "type": "integer", + "description": "Envelope schema version." + }, + "kind": { + "type": "string", + "description": "Envelope discriminator, e.g. 'ContractMetadata'." + }, + "generated_at": { + "type": "string", + "format": "date-time", + "description": "ISO 8601 UTC timestamp of generation." + }, + "generated_by": { + "type": "object", + "description": "Tool, version and command that generated the contract.", + "properties": { + "tool": { + "type": "string" + }, + "version": { + "type": "string" + }, + "command": { + "type": "string" + } + } + } + } + }, + "layer": { + "type": "string", + "description": "Data layer label (e.g., Bronze/Silver/Gold). Free-form. When set together with `productType`, the two MUST be consistent under the canonical mapping (Bronze↔SDP, Silver↔ADP, Gold↔CDP)." + }, + "productType": { + "type": "string", + "enum": [ + "SDP", + "ADP", + "CDP" + ], + "description": "NEW in v0.7.3: Data Mesh data-product type. SDP = Source-Aligned Data Product (raw ingestion from external systems; corresponds to Bronze). ADP = Aggregated Data Product (cross-source joined / domain-modelled; corresponds to Silver). CDP = Consumption-Aligned Data Product (analytics- or product-shaped; corresponds to Gold). Either `metadata.layer` or `metadata.productType` is sufficient β€” the validator infers the missing one. When both are present they MUST agree." + }, + "classification": { + "type": "string", + "enum": [ + "public", + "internal", + "confidential", + "restricted" + ], + "description": "NEW in v0.7.3: Data classification label propagated to catalog + access policy enforcement." + }, + "experimental": { + "type": "array", + "description": "NEW in v0.7.3: Feature gates the contract opts into (e.g., 'acquisition' while Bronze acquisition is in beta).", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "owner": { + "type": "object", + "additionalProperties": false, + "properties": { + "team": { + "type": "string" + }, + "email": { + "type": "string", + "format": "email" + }, + "slack": { + "type": "string" + }, + "oncall": { + "type": "string", + "description": "Oncall rotation or contact for operational issues." + } + } + }, + "team": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.6: The team behind this product, as a roster rather than a single contact. `metadata.owner` holds one team name and one mailbox and carries no role, so a product with a data owner, a steward and two engineers had nowhere to say so and the exported ODCS/ODPS team block came out empty. The field names are Bitol's, so an imported team round-trips without a translation. Optional: a contract without it exports exactly as it does today.", + "properties": { + "name": { + "type": "string", + "description": "Team name, e.g. 'Sales Analytics'." + }, + "members": { + "type": "array", + "description": "The people on the team, each with the role they hold on this product.", + "items": { + "$ref": "#/$defs/teamMember" + } + } + } + }, + "createdAt": { + "type": "string", + "format": "date-time" + }, + "businessContext": { + "type": "object", + "description": "Business alignment information for data mesh organization.", + "additionalProperties": false, + "properties": { + "domain": { + "type": "string", + "description": "Business domain this product belongs to." + }, + "subdomain": { + "type": "string", + "description": "Business subdomain for more granular organization." + }, + "businessCapability": { + "type": "string", + "description": "Business capability this product supports." + }, + "valueStream": { + "type": "string", + "description": "Value stream this product participates in." + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Product-level tags on metadata for discovery and categorization." + } + } + }, + "exposes": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/expose" + } + }, + "consumes": { + "type": "array", + "items": { + "$ref": "#/$defs/consumeRef" + } + }, + "builds": { + "type": "array", + "description": "NEW in v0.5.5: Multiple build configurations for multi-modal data products (e.g., batch + streaming, SQL + ML).", + "items": { + "$ref": "#/$defs/build" + } + }, + "build": { + "$ref": "#/$defs/build", + "description": "Single build configuration (legacy). Use 'builds' array for multi-modal." + }, + "lineage": { + "$ref": "#/$defs/lineage", + "description": "Optional lineage tracking for better data governance." + }, + "schemaEvolution": { + "$ref": "#/$defs/schemaEvolution", + "description": "Optional schema evolution strategy for managing changes." + }, + "machineLearning": { + "$ref": "#/$defs/machineLearning", + "description": "Optional ML model specifications for MLPipeline kind." + }, + "environments": { + "type": "object", + "description": "Optional environment-specific overrides (dev, staging, prod).", + "additionalProperties": { + "$ref": "#/$defs/environmentConfig" + } + }, + "lifecycle": { + "$ref": "#/$defs/lifecycle" + }, + "docs": { + "$ref": "#/$defs/docs" + }, + "sovereignty": { + "$ref": "#/$defs/sovereignty", + "description": "NEW in v0.7.1: Jurisdiction and data residency requirements for compliance enforcement." + }, + "accessPolicy": { + "$ref": "#/$defs/accessPolicy", + "description": "Root-level access policy for automated IAM binding generation. Defines principals, permissions, and resources for data product access control." + }, + "orchestration": { + "$ref": "#/$defs/orchestration", + "description": "Top-level orchestration configuration for scheduling and workflow engines." + }, + "retention": { + "$ref": "#/$defs/retention", + "description": "NEW in v0.7.3: Retention durations for run state, run logs, lineage events, and DLQ records. Honored by the retention sweeper invoked from policy-apply." + }, + "consumers": { + "type": "array", + "description": "NEW in v0.7.6: Declared downstream consumers of this data product β€” the business artifacts built on it (dashboards, notebooks, analyses, ML apps). Shape follows dbt exposures so existing dbt exposure definitions map field-for-field. Purely declarative and additive: consumers never affect plan/apply. RESERVED, not yet consumed β€” no code reads this key today; it is carried verbatim into plan.json and the contract COMMENT so it survives round-trip, and is intended to feed lineage terminal nodes and impact statements ('Affects: Revenue dashboard') in a later release.", + "items": { + "$ref": "#/$defs/consumerRef" + } + }, + "governance": { + "$ref": "#/$defs/governance", + "description": "Contract-level governance: account/project-wide settings (e.g. AWS Lake Formation admins and LF-tag definitions). Per-resource grants and tag associations live under binding.governance." + }, + "packaging": { + "$ref": "#/$defs/packaging", + "description": "NEW in v0.7.6: Contract-wide packaging default β€” declarative container ownership (isolated = this product creates and owns its infrastructure containers; shared = containers are pre-existing, platform-owned pools the product writes into but does not own). Overridable per exposure via binding.packaging. An ABSENT block means legacy behavior: the IaC emit is byte-identical to pre-packaging releases." + } + }, + "$defs": { + "descriptionObject": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.6: The structured form of the root `description`. The three field names are Bitol's β€” ODPS v1.1.0 `$defs.Description` and the ODCS v3.1.0 root `description` both use exactly purpose/usage/limitations β€” so import and export are a pass-through rather than a translation.", + "properties": { + "purpose": { + "type": "string", + "description": "What this data product is for." + }, + "usage": { + "type": "string", + "description": "How a consumer is expected to use it." + }, + "limitations": { + "type": "string", + "description": "What it must not be used for, and what it does not cover." + } + } + }, + "teamMember": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.6: One member of `metadata.team.members`. The field names and the username-only required list are Bitol's ODCS v3.1.0 / ODPS v1.1.0 `$defs.TeamMember`, copied verbatim so an imported team exports back unchanged.", + "required": [ + "username" + ], + "properties": { + "username": { + "type": "string", + "description": "The member's identifier in the organisation's directory β€” the one field Bitol requires." + }, + "name": { + "type": "string", + "description": "Display name." + }, + "role": { + "type": "string", + "description": "The role this member holds on this product, e.g. 'owner', 'steward', 'data engineer'. Free-form: Bitol does not constrain it." + }, + "email": { + "type": "string", + "format": "email" + } + } + }, + "packaging": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.6: Declarative packaging mode β€” per-container ownership for the OpenTofu IaC emit (RFC-packaging-modes.md). 'isolated' emits owned resources (today's exact emit); 'shared' emits a data-source reference to a platform-owned pool plus leaf-only owned resources (tables, prefixed objects, scoped grants). Hybrid tiers (e.g. Snowflake shared-database / owned-schema / owned-warehouse) come from the per-kind containers override map. Prior art: Terraform's resource-vs-data-source ownership split, Unity Catalog isolation-mode OPEN|ISOLATED, Kubernetes shared-vs-dedicated node pools.", + "properties": { + "mode": { + "type": "string", + "enum": [ + "isolated", + "shared" + ], + "description": "NEW in v0.7.6: Blanket ownership mode for every container kind. Default when the block is present: 'isolated'. Per-kind exceptions go in `containers`." + }, + "pool": { + "type": "string", + "minLength": 1, + "description": "NEW in v0.7.6: Pool id of the platform-owned tenant pool this product writes into. REQUIRED (enforced by the resolver) whenever any container resolves 'shared' β€” a pool must be addressable. Propagated as the `fluid_pool` label/tag for cost attribution." + }, + "poolManifest": { + "type": "string", + "minLength": 1, + "description": "NEW in v0.7.6: Optional path to the platform team's pool manifest file; snapshotted into the bundle at stage 1 so bundleDigest covers it (any pool-file edit invalidates downstream plans)." + }, + "containers": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.6: Per-container-kind ownership overrides β€” each key wins over `mode` for that kind, yielding hybrid tiers. Kind↔platform mapping: bucket β†’ aws_s3_bucket / google_storage_bucket; database β†’ snowflake_database AND aws_glue_catalog_database; dataset β†’ google_bigquery_dataset; schema β†’ snowflake_schema; warehouse β†’ snowflake_warehouse; cluster β†’ Confluent environment/cluster (v1: shared only).", + "properties": { + "bucket": { + "$ref": "#/$defs/packagingContainerMode", + "description": "NEW in v0.7.6: Ownership of the object-store bucket (aws_s3_bucket / google_storage_bucket)." + }, + "database": { + "$ref": "#/$defs/packagingContainerMode", + "description": "NEW in v0.7.6: Ownership of the database container β€” covers Snowflake `database` AND AWS `glue_database`." + }, + "dataset": { + "$ref": "#/$defs/packagingContainerMode", + "description": "NEW in v0.7.6: Ownership of the BigQuery dataset (google_bigquery_dataset)." + }, + "schema": { + "$ref": "#/$defs/packagingContainerMode", + "description": "NEW in v0.7.6: Ownership of the Snowflake schema (snowflake_schema)." + }, + "warehouse": { + "$ref": "#/$defs/packagingContainerMode", + "description": "NEW in v0.7.6: Ownership of the Snowflake warehouse (snowflake_warehouse) β€” isolated gives per-product cost attribution." + }, + "cluster": { + "$ref": "#/$defs/packagingContainerMode", + "description": "NEW in v0.7.6: Ownership of the Confluent environment/cluster. v1 accepts 'shared' only β€” dedicated-cluster provisioning is not yet supported (the resolver rejects 'isolated')." + } + } + } + } + }, + "packagingContainerMode": { + "type": "string", + "enum": [ + "isolated", + "shared" + ], + "description": "NEW in v0.7.6: Per-container ownership β€” 'isolated' (this product creates and owns it) or 'shared' (pre-existing platform-owned pool, referenced not created)." + }, + "retention": { + "type": "object", + "description": "NEW in v0.7.3: Retention durations (ISO-8601). Defaults: runState=P30D, runLogs=P90D, lineage=P365D, dlq=P180D.", + "additionalProperties": false, + "properties": { + "runState": { + "$ref": "#/$defs/isoDuration" + }, + "runLogs": { + "$ref": "#/$defs/isoDuration" + }, + "lineage": { + "$ref": "#/$defs/isoDuration" + }, + "dlq": { + "$ref": "#/$defs/isoDuration" + } + } + }, + "identifier": { + "type": "string", + "pattern": "^[A-Za-z0-9_][A-Za-z0-9_.-]*[A-Za-z0-9_]$|^[A-Za-z0-9_]$", + "description": "Unified identifier pattern: starts/ends with alphanumeric or underscore, allows dots and hyphens in middle. Both casing conventions accepted (snake_case / dotted lowercase / PascalCase / UPPER) so existing catalog ids round-trip unchanged." + }, + "semver": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?$", + "description": "Semantic version (semver.org)." + }, + "semverRange": { + "type": "string", + "description": "Semver range (e.g., ^1.2, ~1.4, >=1.0.0). Not strictly validated by regex; tooling enforces." + }, + "isoDuration": { + "type": "string", + "pattern": "^P(?!$)(\\d+Y)?(\\d+M)?(\\d+W)?(\\d+D)?(T(\\d+H)?(\\d+M)?(\\d+S)?)?$", + "description": "ISO-8601 duration (e.g., P1D, PT15M, P2Y6M)." + }, + "availabilityPct": { + "type": "string", + "pattern": "^(100(\\.0+)?|\\d{2}(\\.\\d+)?|\\d{1}\\d(\\.\\d+)?)%$", + "description": "Availability percentage (e.g., 99.9%)." + }, + "lifecycleState": { + "type": "string", + "enum": [ + "preview", + "active", + "deprecated", + "retired" + ], + "description": "Unified lifecycle state vocabulary." + }, + "sensitivityLevel": { + "type": "string", + "enum": [ + "none", + "internal", + "confidential", + "restricted", + "pii", + "phi", + "cleartext", + "treated", + "anonymized", + "pseudonymized", + "tokenized", + "encrypted" + ], + "description": "Unified data sensitivity classification." + }, + "tags": { + "type": "array", + "description": "Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.", + "items": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*[a-z0-9]$|^[a-z0-9]$", + "description": "Tag format: lowercase alphanumeric with hyphens, e.g., 'customer-data', 'pii', 'real-time'" + }, + "uniqueItems": true + }, + "labels": { + "type": "object", + "description": "Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.", + "additionalProperties": { + "type": "string", + "description": "Label values are always strings for consistency and tooling compatibility." + }, + "examples": [ + { + "team": "customer-analytics", + "criticality": "high", + "cost-center": "engineering", + "retention": "7-years", + "region": "us-central1", + "business-name": "Customer Profiles", + "data-category": "analytical" + } + ] + }, + "expose": { + "type": "object", + "additionalProperties": false, + "required": [ + "exposeId", + "kind", + "binding", + "contract" + ], + "examples": [ + { + "exposeId": "customer_profiles", + "title": "Customer Profiles Table", + "kind": "table", + "tags": [ + "pii", + "customer-facing", + "analytical" + ], + "labels": { + "sensitivity": "high", + "retention": "7-years", + "team": "customer-analytics" + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "STRING", + "required": true, + "sensitivity": "cleartext", + "tags": [ + "identifier", + "primary-key" + ], + "labels": { + "business-name": "Customer ID", + "data-category": "identifier" + } + } + ] + }, + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "company-data", + "dataset": "customer", + "table": "profiles" + } + } + } + ], + "properties": { + "exposeId": { + "$ref": "#/$defs/identifier", + "description": "Stable interface handle (used by consumers & build.outputs)." + }, + "title": { + "type": "string" + }, + "description": { + "type": "string", + "description": "Detailed description of the exposed resource." + }, + "version": { + "$ref": "#/$defs/semver" + }, + "kind": { + "type": "string", + "enum": [ + "table", + "view", + "api", + "file", + "stream", + "topic", + "feature_store", + "model", + "vector", + "graph", + "time_series", + "other" + ] + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Expose-level tags for categorization and routing." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Expose-level labels for metadata and automation." + }, + "contract": { + "$ref": "#/$defs/exposeContract" + }, + "qos": { + "$ref": "#/$defs/exposeQoS" + }, + "policy": { + "$ref": "#/$defs/exposePolicy" + }, + "binding": { + "$ref": "#/$defs/binding" + }, + "lifecycle": { + "$ref": "#/$defs/exposeLifecycle" + }, + "observability": { + "$ref": "#/$defs/observability" + }, + "docs": { + "$ref": "#/$defs/docs" + }, + "semantics": { + "$ref": "#/$defs/semanticModel", + "description": "NEW in v0.7.2: Semantic model definition mapping physical columns to business concepts. Defines entities, measures, dimensions, and metrics for agent-consumable business logic. Eliminates semantic hallucination by providing machine-readable KPI definitions. Compatible with dbt MetricFlow, Snowflake Semantic Views, and OSI-aligned formats." + }, + "crawler": { + "type": "object", + "description": "AWS Glue crawler configuration for auto-discovery of schema from S3 paths.", + "properties": { + "name": { + "type": "string", + "description": "Glue crawler name." + }, + "role": { + "type": "string", + "description": "IAM role ARN for the crawler." + }, + "schedule": { + "type": "string", + "description": "Cron schedule expression (e.g., 'cron(0 6 * * ? *)')." + }, + "classifiers": { + "type": "array", + "description": "Custom classifier names to apply.", + "items": { + "type": "string" + } + }, + "schemaChangePolicy": { + "type": "object", + "description": "Policy for schema changes detected by the crawler.", + "properties": { + "updateBehavior": { + "type": "string", + "enum": [ + "UPDATE_IN_DATABASE", + "LOG" + ] + }, + "deleteBehavior": { + "type": "string", + "enum": [ + "LOG", + "DELETE_FROM_DATABASE", + "DEPRECATE_IN_DATABASE" + ] + } + } + } + } + }, + "iceberg": { + "type": "object", + "description": "AWS Glue Iceberg table management configuration (snapshots, compaction).", + "properties": { + "writeFormat": { + "type": "string", + "enum": [ + "parquet", + "orc", + "avro" + ], + "description": "Underlying file format for Iceberg data files." + }, + "snapshotRetention": { + "type": "object", + "description": "Snapshot retention policy.", + "properties": { + "maxSnapshotAgeMs": { + "type": "integer", + "description": "Maximum age of snapshots in milliseconds." + }, + "minSnapshotsToKeep": { + "type": "integer", + "description": "Minimum number of snapshots to retain." + } + } + }, + "compaction": { + "type": "object", + "description": "Iceberg compaction settings.", + "properties": { + "enabled": { + "type": "boolean" + }, + "targetFileSizeMb": { + "type": "integer", + "description": "Target file size in MB after compaction." + } + } + } + } + }, + "mcp": { + "$ref": "#/$defs/exposeMcp", + "description": "NEW in v0.7.4: Declares this expose as agent-consumable over MCP, with per-expose overrides for the consumer-side output-port server (`fluid mcp output-port serve`). Its presence opts the expose into the Fluid MCP gateway, where `policy.agentPolicy.allowedModels` / `allowedUseCases` are enforced at every read." + } + } + }, + "exposeContract": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "NEW in v0.7.6: An explicit, stable contract id, used verbatim as the ODCS `id` and as the ODPS output port `contractId`. Without it the id is derived as `.`, so renaming the product or the expose silently changes the contract id and breaks every consumer reference to it, and a catalog that requires an id format of its own (a UUID, say) cannot be served at all. Each expose needs its own: two exposes sharing one id is refused by `fluid validate`.", + "minLength": 1 + }, + "schemaSignature": { + "type": "string", + "pattern": "^sha256:[0-9a-fA-F]{64}$", + "description": "Hash of canonical schema/OpenAPI. Used by contract-tests gates." + }, + "schemaPolicy": { + "type": "string", + "enum": [ + "strict", + "discover_and_freeze", + "evolve_safe", + "evolve_all" + ], + "default": "strict", + "description": "NEW in v0.7.3: Schema evolution policy. strict = fail on any schema change; discover_and_freeze = capture from source on first run, fail thereafter; evolve_safe = accept additive changes; evolve_all = accept all changes (cast on type narrow, route failures to DLQ)." + }, + "schema": { + "type": "array", + "items": { + "$ref": "#/$defs/column" + }, + "description": "Tabular schema (for table/view/file)." + }, + "openapiRef": { + "type": "string", + "format": "uri", + "description": "OpenAPI document URL (for APIs)." + }, + "guarantees": { + "type": "object", + "additionalProperties": false, + "properties": { + "compatibility": { + "type": "string", + "enum": [ + "none", + "patch", + "minor", + "backward", + "forward", + "strict" + ], + "description": "Compatibility promise for non-breaking evolution." + }, + "evolution": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "additive_columns", + "additive_fields", + "additive_endpoints", + "nullable_to_required_disallowed", + "field_removal_with_notice", + "type_widening_allowed" + ] + } + } + } + }, + "dq": { + "$ref": "#/$defs/dqSpec", + "description": "Enhanced data quality rules including anomaly detection." + }, + "quality": { + "type": "array", + "description": "Simplified SQL-expression data quality rules (alternate form of dq.rules for inline assertions).", + "items": { + "type": "object", + "required": [ + "rule", + "expression", + "severity" + ], + "properties": { + "rule": { + "type": "string", + "description": "Rule identifier." + }, + "expression": { + "type": "string", + "description": "SQL or boolean expression that must evaluate to true." + }, + "severity": { + "type": "string", + "enum": [ + "error", + "warning", + "info" + ], + "description": "Severity when the rule fails." + } + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Contract-level tags for data quality and governance." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Contract-level labels for automation and metadata." + } + }, + "anyOf": [ + { + "required": [ + "schema" + ] + }, + { + "required": [ + "openapiRef" + ] + } + ] + }, + "column": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type" + ], + "properties": { + "fields": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/column"}, + "description": "Typed child columns of a struct/row/record." + }, + "element": {"$ref": "#/$defs/nestedType"}, + "key": {"$ref": "#/$defs/nestedType"}, + "value": {"$ref": "#/$defs/nestedType"}, + "name": { + "type": "string", + "minLength": 1 + }, + "type": { + "description": "Column data type. Accepts a canonical FLUID/SQL type name (case-insensitive) or a parameterized form such as decimal(18,4) or varchar(255). The enum lists the canonical bare types; the pattern additionally permits case variants and parameter suffixes. Intentionally generous β€” it rejects obvious garbage (e.g. MYSTERY_TYPE) while admitting the cross-dialect superset.", + "anyOf": [ + { + "type": "string", + "enum": [ + "string", + "text", + "varchar", + "varchar2", + "nvarchar", + "char", + "nchar", + "character", + "clob", + "int", + "integer", + "int2", + "int4", + "int8", + "int16", + "int32", + "int64", + "tinyint", + "smallint", + "mediumint", + "bigint", + "long", + "longint", + "serial", + "bigserial", + "float", + "float4", + "float8", + "float32", + "float64", + "double", + "real", + "decimal", + "dec", + "numeric", + "number", + "bignumeric", + "money", + "boolean", + "bool", + "bit", + "date", + "time", + "datetime", + "datetime2", + "smalldatetime", + "timestamp", + "timestamptz", + "timestamp_tz", + "timestamp_ntz", + "timestamp_ltz", + "timestampntz", + "interval", + "year", + "variant", + "object", + "array", + "struct", + "map", + "record", + "row", + "json", + "jsonb", + "super", + "binary", + "varbinary", + "bytes", + "blob", + "bytea", + "raw", + "geography", + "geometry", + "geom", + "point", + "uuid", + "uniqueidentifier", + "guid", + "enum", + "hll" + ] + }, + { + "type": "string", + "pattern": "(?i)^\\s*(string|text|varchar|varchar2|nvarchar|char|nchar|character|clob|int|integer|int2|int4|int8|int16|int32|int64|tinyint|smallint|mediumint|bigint|long|longint|serial|bigserial|float|float4|float8|float32|float64|double|real|double\\s+precision|decimal|dec|numeric|number|bignumeric|money|boolean|bool|bit|date|time|datetime|datetime2|smalldatetime|timestamp|timestamptz|timestamp_tz|timestamp_ntz|timestamp_ltz|timestampntz|timestamp\\s+with(out)?\\s+time\\s+zone|interval|year|variant|object|array|struct|map|record|row|json|jsonb|super|binary|varbinary|bytes|blob|bytea|raw|geography|geometry|geom|point|uuid|uniqueidentifier|guid|enum|hll)\\s*(\\(\\s*[0-9A-Za-z_,\\s'\\\"-]*\\s*\\))?\\s*$" + } + ] + }, + "required": { + "type": "boolean" + }, + "description": { + "type": "string" + }, + "sensitivity": { + "$ref": "#/$defs/sensitivityLevel", + "description": "Field-level sensitivity classification for improved policy ergonomics." + }, + "semanticType": { + "type": "string", + "description": "Semantic meaning (e.g., 'email', 'phone', 'identifier')." + }, + "businessName": { + "type": "string", + "description": "Business-friendly name for this field." + }, + "businessDefinition": { + "type": "string", + "description": "Business definition and meaning of this field." + }, + "validationRules": { + "type": "array", + "description": "Field-level validation constraints.", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "range", + "regex", + "enum", + "custom" + ] + }, + "constraint": { + "type": "string" + }, + "message": { + "type": "string" + } + } + } + }, + "quality": { + "type": "array", + "description": "NEW in v0.7.6: Column-level data quality rules, in ODCS's own shape β€” the array that is written straight into `schema[].properties[].quality[]` of the exported ODCS v3.1.0 contract, and the array an imported ODCS column's rules are read back into, inline on the column rather than in the importer's pass-through bucket: the ODPS importer merges a resolved port's `contract` block into its output-port stub and discards the sub-import's root `extensions`, so rules left in that bucket were dropped outright by a FLUID β†’ ODPS β†’ FLUID cycle. The ODCS exporter has always replayed this key verbatim and always read it back on import, but `$defs.column` is `additionalProperties: false` and did not define it, so a column written the way the exporter supports was REFUSED by `fluid validate`: the only way to get a hand-written quality rule into a published contract was to write an invalid contract. Optional and additive β€” a column without it exports exactly as it does today, with the rules forge derives from `required`, the primary-key tag and `validationRules`. Setting it REPLACES those derived rules for that column rather than adding to them, which is what makes an imported ODCS column round-trip byte-for-byte. The authority for an item's contents is ODCS v3.1.0 `$defs.DataQuality`, deliberately not re-declared here (a copy would drift from the vendored schema, and `fluid odcs validate` already checks an emitted document against it); the keys below are the common ones. Every rule should carry an `id`: it is optional in ODCS, but a DQ engine that labels its results by rule id rejects a contract whose rules have none.", + "items": { + "type": "object", + "minProperties": 1, + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]+$", + "description": "Stable rule identifier, unique within this column's rules. The pattern is ODCS `$defs.StableId`. Rules forge derives itself are named `-`." + }, + "type": { + "type": "string", + "enum": [ + "text", + "library", + "sql", + "custom" + ], + "description": "ODCS quality check type. text = prose; library = one of ODCS's predefined metrics; sql = a query whose result is compared; custom = a named engine's own check." + }, + "metric": { + "type": "string", + "enum": [ + "nullValues", + "missingValues", + "invalidValues", + "duplicateValues", + "rowCount" + ], + "description": "type: library only. The five predefined metrics ODCS v3.1.0 defines; a name outside this list is not a library rule." + }, + "arguments": { + "type": "object", + "description": "type: library only. Extra arguments for the metric β€” `pattern` (a regex) or `validValues` (a list) for invalidValues, as DataHub's ODCS ingestion source reads them." + }, + "query": { + "type": "string", + "description": "type: sql only. The query whose result the comparison operator is applied to." + }, + "engine": { + "type": "string", + "description": "type: custom only. The engine that runs the check, e.g. soda, great-expectations, dbt." + }, + "implementation": { + "description": "type: custom only. The check as that engine expresses it." + }, + "dimension": { + "type": "string", + "enum": [ + "accuracy", + "completeness", + "conformity", + "consistency", + "coverage", + "timeliness", + "uniqueness" + ], + "description": "The data quality dimension the rule measures." + }, + "description": { + "type": "string" + }, + "severity": { + "type": "string", + "description": "Severity of a failure. ODCS types this as a free string and documents info / warning / error by example." + }, + "unit": { + "type": "string", + "description": "Unit the comparison is in, usually rows (the default) or percent." + } + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Field-level tags for categorization and data governance." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Field-level labels for metadata and automation." + } + } + }, + "nestedType": { + "type": "object", + "additionalProperties": false, + "required": ["type"], + "properties": { + "type": {"$ref": "#/$defs/column/properties/type"}, + "required": {"type": "boolean"}, + "fields": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/column"} + }, + "element": {"$ref": "#/$defs/nestedType"}, + "key": {"$ref": "#/$defs/nestedType"}, + "value": {"$ref": "#/$defs/nestedType"} + } + }, + "streamSource": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.7 (preview): The Kafka records a streaming build reads, described apart from the engine that reads them: one topic, how its values are encoded, the value's fields, an optional key, and Kafka record metadata read as columns. The value and key schemas are lists of columns, the same shape as an expose's contract.schema, so one fragment can describe a topic and a table that lands it. Brokers, consumer groups and registries are the engine's parameters, not part of the stream.", + "required": ["kind", "topic", "format", "schema"], + "properties": { + "kind": { + "const": "kafka", + "description": "The streaming system. Only kafka in this preview." + }, + "topic": { + "type": "string", + "pattern": "^(?!\\.\\.?(?![\\s\\S]))[A-Za-z0-9._-]{1,249}(?![\\s\\S])", + "description": "One Kafka topic, by name: Kafka's legal characters (not . or ..), no pattern and no list." + }, + "format": { + "enum": ["json", "avro-confluent"], + "description": "How record values are encoded. json: one JSON object per record; a field the schema does not declare is ignored, and a declared field that is missing reads as null. avro-confluent: Avro in the Confluent Schema Registry wire format. The declared schema is the reader schema, so the producers' subject must stay readable by it: FORWARD or FULL compatibility, transitive." + }, + "schema": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/column"}, + "description": "The record value's fields, as columns. Nested data uses fields (struct), element (array) and key/value (map)." + }, + "key": {"$ref": "#/$defs/streamKey"}, + "metadata": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/streamMetadataColumn"}, + "description": "Kafka record metadata, each read as a column." + } + } + }, + "streamKey": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.7 (preview): The record key, read as columns next to the value's. A record without a key reads its key columns as null.", + "required": ["format", "schema"], + "properties": { + "format": { + "enum": ["raw", "json", "avro-confluent"], + "description": "raw: the key's bytes as one column, of type string (UTF-8) or binary. json and avro-confluent: a key record whose fields are the schema's columns." + }, + "schema": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/column"}, + "description": "The key's fields, as columns: exactly one for raw." + }, + "prefix": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])", + "description": "Put in front of every key column's name, so a key field can share its name with a value field (a Debezium key repeats the value's id). A mapping names the prefixed column." + } + } + }, + "streamMetadataColumn": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.7 (preview): One piece of Kafka record metadata, read as a column. Its type is fixed by what it reads: topic string, partition int, offset long, timestamp timestamp_ltz(3), timestamp-type string, headers map and leader-epoch int. leader-epoch is null when the broker does not report it; the others are never null.", + "required": ["name", "from"], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "The column's name, unique among the stream's value, key and metadata columns." + }, + "from": { + "enum": ["topic", "partition", "offset", "timestamp", "timestamp-type", "headers", "leader-epoch"], + "description": "The metadata the column reads." + }, + "description": {"type": "string"} + } + }, + "recordMapping": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.7 (preview): How the stream's records become the output's rows. An output column the mapping does not list, nested fields included, is read from the stream column or field at the same path, and keeps its type: so a stream landed as it is, flat or nested, needs no mapping. forge-cli writes the query from the mapping and checks every output column's type and nullability against it before anything deploys; no contract text reaches the job's SQL.", + "properties": { + "explode": {"$ref": "#/$defs/recordExplode"}, + "columns": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/recordMappingColumn"}, + "description": "Output columns, or nested fields of them, that are read from another path, renamed or converted." + } + } + }, + "recordExplode": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.7 (preview): One output row per element of an array. A record whose array is null or empty produces no row.", + "required": ["path", "as"], + "properties": { + "path": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*|\\['[^'\\\\\\u0000-\\u001f\\u007f]{1,256}'\\])*(?![\\s\\S])", + "description": "The array: a stream column or a nested field, such as items or order.lines." + }, + "as": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])", + "description": "The element's name in mapping paths: item.sku for a field of a struct element, item for a scalar element." + } + } + }, + "recordMappingColumn": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.7 (preview): Where one output column, or one nested field of one, is read from.", + "required": ["name", "from"], + "properties": { + "name": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*)*(?![\\s\\S])", + "description": "The output column, or a field of a struct output column as a dotted path (customer.id). Mapping a nested field builds that struct from the paths given, so a nested output can be written from fields that are flat or elsewhere in the record." + }, + "from": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*|\\['[^'\\\\\\u0000-\\u001f\\u007f]{1,256}'\\])*(?![\\s\\S])", + "description": "The source: a stream column (value, key or metadata), a nested field (customer.address.country), a map entry (attributes['channel'], null when the key is absent) or the exploded element (item.sku)." + }, + "cast": { + "$ref": "#/$defs/column/properties/type", + "description": "Convert the source to this type: binary to string (UTF-8), or a widening (int to long, float to double, a decimal to a wider precision at the same scale, a timestamp to a finer precision)." + }, + "description": {"type": "string"} + } + }, + "dqSpec": { + "type": "object", + "additionalProperties": false, + "properties": { + "rules": { + "type": "array", + "items": { + "$ref": "#/$defs/dqRule" + } + }, + "monitoring": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "default": true + }, + "window": { + "$ref": "#/$defs/isoDuration" + }, + "owner": { + "type": "string" + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Data quality specification tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Data quality specification labels." + } + } + }, + "dqRule": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "type", + "severity" + ], + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "freshness", + "completeness", + "uniqueness", + "valid_values", + "accuracy", + "schema", + "anomaly_detection", + "drift_detection" + ] + }, + "selector": { + "type": "string" + }, + "threshold": { + "type": "number" + }, + "operator": { + "type": "string", + "enum": [ + ">=", + ">", + "<=", + "<", + "==", + "!=" + ] + }, + "window": { + "$ref": "#/$defs/isoDuration" + }, + "severity": { + "type": "string", + "enum": [ + "info", + "warn", + "error", + "critical" + ] + }, + "description": { + "type": "string" + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "DQ rule tags for categorization and automation." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "DQ rule labels for metadata and routing." + } + } + }, + "exposeQoS": { + "type": "object", + "additionalProperties": false, + "properties": { + "availability": { + "$ref": "#/$defs/availabilityPct" + }, + "freshnessSLO": { + "$ref": "#/$defs/isoDuration" + }, + "dataLossSLO": { + "type": "string", + "description": "e.g., '0 rows'" + }, + "latencyP95": { + "$ref": "#/$defs/isoDuration" + }, + "completenessTarget": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Target completeness ratio (0.0 to 1.0)." + }, + "errorBudget": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Acceptable error rate (0.0 to 1.0)." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "QoS-level tags for SLA management." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "QoS-level labels for automation and reporting." + } + } + }, + "exposePolicy": { + "type": "object", + "additionalProperties": false, + "properties": { + "authn": { + "type": "string", + "enum": [ + "oidc", + "oauth2", + "api_key", + "none", + "custom", + "iam", + "jwt" + ] + }, + "authz": { + "type": "object", + "additionalProperties": false, + "properties": { + "readers": { + "type": "array", + "items": { + "type": "string" + } + }, + "writers": { + "type": "array", + "items": { + "type": "string" + } + }, + "columnRestrictions": { + "type": "array", + "description": "Column-level access control. A column named in any restriction is restricted. 'deny': the principal may not read the columns. 'allow': the columns are readable only by the principals an allow names. A deny beats an allow, and a restriction never grants access: the readers are the expose's readers. Principals are logical and resolve through binding.principals. NEW in v0.7.6 (the field itself is older): `fluid apply` enforces it. GCP BigQuery: a Data Catalog taxonomy per product and dataset with fine-grained access control, a policy tag per set of restricted columns that share their readers, attached through the table schema's policyTags, and roles/datacatalog.categoryFineGrainedReader for exactly the expose's readers allowed to read them: the accessPolicy read grantees and the expose's own policy.authz.readers (a denied principal gets an access error on the columns). A restriction on an expose with no reader on the binding is refused, as it would lock the columns for everyone. A restriction's tags and labels are written into the policy tag's description; AWS has no place for them. AWS: each governance.lakeFormation grant with SELECT excludes the restricted columns its principal may not read (table_with_columns.excluded_column_names); a grant's hand-written excludedColumns must agree, and a binding with no Lake Formation grants is refused. `fluid verify` checks the policy tags and their readers on GCP, and the principals' Lake Formation permissions on AWS. NEW in v0.7.6: access: mask lets the principal read the columns masked by the platform (see mask): a deny beats a mask, and a mask beats an allow for the principal it names.", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "principal": { + "type": "string", + "description": "Logical principal the restriction applies to, written as in accessPolicy (e.g. group:analysts@company.example); binding.principals maps it per cloud." + }, + "columns": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Columns of this expose's schema." + }, + "access": { + "type": "string", + "enum": [ + "allow", + "deny", + "mask" + ], + "description": "deny: the principal may not read the columns. allow: only the principals an allow names may read them. mask: the principal reads the columns masked by the platform, by the rule in mask, and never their raw values (NEW in v0.7.6)." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Access control rule tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Access control rule labels." + }, + "mask": { + "type": "string", + "enum": [ + "last_four", + "first_four", + "nullify" + ], + "description": "The masking rule an access: mask restriction applies, required with it and refused without it. last_four: 'XXXXX' and the value's last four characters; first_four: its first four characters and 'XXXXX' (a value of four characters or fewer is hashed instead, by each platform); nullify: NULL. GCP BigQuery: a data policy (DATA_MASKING_POLICY, LAST_FOUR_CHARACTERS, FIRST_FOUR_CHARACTERS or ALWAYS_NULL) on the columns' policy tag, and roles/bigquerydatapolicy.maskedReader for the masked principals, who read the same column masked. AWS Glue: Lake Formation cannot mask, so the masked principals' grant on the table excludes the columns and they read a protected Glue Data Catalog view (SECURITY DEFINER, binding.governance.lakeFormation.maskedViews) that returns them masked by the same rule (NEW in v0.7.6)." + } + }, + "allOf": [ + { + "if": { + "properties": { + "access": { + "const": "mask" + } + }, + "required": [ + "access" + ] + }, + "then": { + "required": [ + "mask" + ] + } + }, + { + "if": { + "required": [ + "mask" + ] + }, + "then": { + "properties": { + "access": { + "const": "mask" + } + }, + "required": [ + "access" + ] + } + } + ] + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Authorization policy tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Authorization policy labels." + }, + "rowFilters": { + "type": "array", + "description": "Row-level access control (NEW in v0.7.6). Each filter lets its principal read only the rows its where predicate selects; a reader no filter names reads every row, and a principal may have one filter per expose. The predicate is one SQL boolean expression over this expose's columns, with no statement separator or comment. Principals are logical and resolve through binding.principals. GCP BigQuery: a row access policy (filter_predicate, grantees) per filter, plus one that selects every row for every other reader and writer, since a principal no row access policy names reads no row at all. AWS Glue: a Lake Formation data cells filter (row_filter and the principal's excluded columns), and the principal's SELECT is granted on the filter instead of the table.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "principal", + "where" + ], + "properties": { + "principal": { + "type": "string", + "description": "Logical principal the filter applies to, written as in accessPolicy; binding.principals maps it per cloud." + }, + "name": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{0,62}$", + "description": "The filter's name on the platform (the row access policy or data cells filter); derived from the principal when omitted." + }, + "where": { + "type": "string", + "minLength": 1, + "description": "One SQL boolean expression over the expose's columns, e.g. consent_data_analytics = true AND is_suppressed = false." + }, + "description": { + "type": "string" + } + } + } + } + } + }, + "privacy": { + "type": "object", + "additionalProperties": false, + "properties": { + "masking": { + "type": "array", + "description": "Columns to treat before they land. The DuckDB acquisition runner rewrites each named column inside the write, so the file, S3 object or BigQuery load and the DLQ receive the treated value; quality gates judge the source value first. A treated column lands as a string, so declare it with a string type (string, or VARCHAR under a schemaPolicy that compares source type names). A rule the build cannot apply (an unset salt or key, k_anonymity, a column the stream lacks, a non-string declared type, the incremental cursor) refuses the build rather than landing the column untreated. fluid verify checks every non-null value of a masked column for its strategy's shape, on a local file and, through Athena, on an S3+Glue table. The MCP output-port gateway drops masked columns from what it serves.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "column", + "strategy" + ], + "properties": { + "column": { + "type": "string" + }, + "strategy": { + "type": "string", + "description": "hash: lowercase hex SHA-256 of salt || value, deterministic so joins work (64 hex characters). mask: every character but the first params.keepFirst and the last params.keepLast replaced by '*', length preserved. tokenize: HMAC-SHA256 token, 32 hex characters, the token the tokenize_pii preLand hook makes. encrypt: AES-GCM, written as 'aesgcm:v1:' + base64url(nonce || ciphertext || tag) with the column name as associated data; reversible with the key, not deterministic. k_anonymity: accepted by the schema but refused at landing: it is a property of a whole table over its quasi-identifiers, not of one value.", + "enum": [ + "mask", + "hash", + "tokenize", + "encrypt", + "k_anonymity" + ] + }, + "params": { + "type": "object", + "additionalProperties": true, + "description": "Strategy parameters. The runner accepts exactly these keys and refuses any other, including a literal salt or key: secrets come only from the environment, never the contract, which is published with the plan, the bundle and the Glue table's fluid_contract parameter. Contracts on 0.7.5 and earlier may use the same keys; their schemas leave params untyped, and the runner checks them at build time.", + "properties": { + "saltEnv": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$", + "default": "FLUID_PII_HASH_SECRET", + "description": "NEW in v0.7.6: hash only. The environment variable holding the salt, at least 16 bytes. Unset or empty, the build is refused: an unsalted SHA-256 of a phone number is reversed by enumerating the numbering plan." + }, + "keyEnv": { + "type": "string", + "pattern": "^[A-Z_][A-Z0-9_]{0,127}$", + "description": "NEW in v0.7.6: tokenize and encrypt. The environment variable holding the key. tokenize defaults to FLUID_PII_TOKENIZATION_KEY and needs at least 32 bytes; encrypt defaults to FLUID_PII_ENCRYPTION_SECRET_KEY and needs the base64 of a 16, 24 or 32-byte AES key (openssl rand -base64 32). Unset or empty, the build is refused." + }, + "keepFirst": { + "type": "integer", + "minimum": 0, + "maximum": 64, + "default": 0, + "description": "NEW in v0.7.6: mask only. Leading characters left as they are." + }, + "keepLast": { + "type": "integer", + "minimum": 0, + "maximum": 64, + "default": 4, + "description": "NEW in v0.7.6: mask only. Trailing characters left as they are, so +46701234567 lands as ********4567. A value no longer than keepFirst + keepLast is masked entirely." + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Privacy masking rule tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Privacy masking rule labels." + } + } + } + }, + "rowLevelPolicy": { + "type": "object", + "additionalProperties": false, + "required": [ + "expression" + ], + "properties": { + "expression": { + "type": "string", + "description": "Provider-specific predicate" + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Row-level policy tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Row-level policy labels." + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Privacy policy tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Privacy policy labels." + } + } + }, + "classification": { + "type": "string", + "enum": [ + "Public", + "Internal", + "Confidential", + "Restricted" + ] + }, + "agentPolicy": { + "$ref": "#/$defs/agentPolicy", + "description": "NEW in v0.7.1: AI/LLM usage governance - controls which AI models can consume this data and for what purposes." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Policy-level tags for governance." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Policy-level labels for automation." + } + } + }, + "binding": { + "type": "object", + "additionalProperties": false, + "required": [ + "platform", + "format", + "location" + ], + "properties": { + "platform": { + "type": "string", + "enum": [ + "gcp", + "aws", + "azure", + "snowflake", + "databricks", + "kafka", + "confluent", + "local", + "kubernetes", + "postgres", + "pgvector", + "other" + ], + "description": "Target platform. NEW in v0.7.5: 'pgvector' for the vector/embeddings output port (pgvector-on-PostgreSQL RAG target β€” emits the embeddings table + ANN index DDL for the ai-embeddable columns). NEW in v0.7.5: 'confluent' for the Confluent Cloud Tableflow managed Kafkaβ†’Iceberg emitter. NEW in v0.7.4: 'postgres' for the MCP output-port PostgreSQL driver (AWS Athena uses platform 'aws')." + }, + "format": { + "type": "string", + "enum": [ + "bigquery_table", + "snowflake_table", + "snowflake_view", + "gcs_file", + "s3_file", + "http_api", + "grpc_api", + "pubsub_topic", + "kafka_topic", + "delta_table", + "iceberg", + "parquet", + "csv", + "json", + "redshift_table", + "redshift_serverless", + "redshift_external_schema", + "postgres_table", + "athena_table", + "glue_table", + "pgvector_table", + "other" + ], + "description": "Binding format. NEW in v0.7.5: 'pgvector_table' β€” a pgvector embeddings table target for the vector output port. NEW in v0.7.4: 'postgres_table' (PostgreSQL driver), 'athena_table' / 'glue_table' (AWS Athena driver) for the MCP output port." + }, + "icebergConfig": { + "type": "object", + "description": "Apache Iceberg table format configuration", + "additionalProperties": false, + "properties": { + "writeVersion": { + "type": "integer", + "enum": [ + 1, + 2 + ], + "description": "Iceberg table format version" + }, + "fileFormat": { + "type": "string", + "enum": [ + "parquet", + "orc", + "avro" + ], + "description": "Underlying file format for data files" + }, + "partitionSpec": { + "type": "array", + "description": "Partition transformation specifications", + "items": { + "type": "object", + "required": [ + "sourceColumn", + "transform" + ], + "properties": { + "name": { + "type": "string", + "description": "Partition field name" + }, + "sourceColumn": { + "type": "string", + "description": "Source column for partitioning" + }, + "transform": { + "type": "string", + "enum": [ + "identity", + "year", + "month", + "day", + "hour", + "bucket", + "truncate" + ], + "description": "Partition transform function" + }, + "numBuckets": { + "type": "integer", + "description": "Number of buckets (for bucket transform)" + }, + "width": { + "type": "integer", + "description": "Truncate width (for truncate transform)" + } + } + } + }, + "sortOrder": { + "type": "array", + "description": "Sort order specifications for data files", + "items": { + "type": "object", + "required": [ + "column" + ], + "properties": { + "column": { + "type": "string", + "description": "Column to sort by" + }, + "direction": { + "type": "string", + "enum": [ + "asc", + "desc" + ], + "description": "Sort direction" + }, + "nullOrder": { + "type": "string", + "enum": [ + "nulls-first", + "nulls-last" + ], + "description": "NULL value ordering" + } + } + } + }, + "properties": { + "type": "object", + "description": "Iceberg table properties", + "additionalProperties": { + "type": "string" + } + } + } + }, + "vectorConfig": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.5: vector / embeddings output-port configuration (platform 'pgvector', format 'pgvector_table'). Consumed by fluid_build.output_ports.vector to emit the pgvector embeddings-table + ANN-index DDL and a RAG provenance manifest for the schema's ai-embeddable columns.", + "required": [ + "dimensions" + ], + "properties": { + "dimensions": { + "type": "integer", + "minimum": 1, + "maximum": 16000, + "description": "Embedding vector width (e.g. 1536 for OpenAI text-embedding-3-small, 1024 for Cohere embed-english-light). Maps to the pgvector column type vector()." + }, + "embeddingModel": { + "type": "string", + "description": "Provenance: the embedding model that produces the vectors (e.g. 'text-embedding-3-small'). Recorded in the RAG manifest and the embeddings-table's embedding_model column so a retriever can verify model parity." + }, + "vectorType": { + "type": "string", + "enum": [ + "vector", + "halfvec" + ], + "default": "vector", + "description": "pgvector column type: 'vector' (float32, up to 2000 dims) or 'halfvec' (float16, half the storage, up to 4000 dims)." + }, + "indexType": { + "type": "string", + "enum": [ + "hnsw", + "ivfflat", + "none" + ], + "default": "hnsw", + "description": "ANN index built on the embedding column. 'hnsw' (higher recall / lower query latency, buildable before data lands), 'ivfflat' (lower build cost, needs data first), or 'none' (no index β€” exact search only)." + }, + "distanceMetric": { + "type": "string", + "enum": [ + "cosine", + "l2", + "inner_product", + "l1" + ], + "default": "cosine", + "description": "Distance function the index optimises for β†’ pgvector operator class: cosineβ†’vector_cosine_ops, l2β†’vector_l2_ops, inner_productβ†’vector_ip_ops, l1β†’vector_l1_ops." + }, + "table": { + "type": "string", + "description": "Target embeddings table name. Defaults to '_embeddings' when omitted." + }, + "sourceKeyColumn": { + "type": "string", + "description": "Provenance: the source primary-key column that ties each embedding row back to its origin row (written to the embeddings-table's source_key column). Enables update/delete propagation from the source product to the vector store." + }, + "hnsw": { + "type": "object", + "additionalProperties": false, + "description": "HNSW index tuning (used when indexType is 'hnsw').", + "properties": { + "m": { + "type": "integer", + "minimum": 1, + "description": "Max bi-directional links per node (pgvector default 16)." + }, + "efConstruction": { + "type": "integer", + "minimum": 1, + "description": "Build-time dynamic candidate-list size (pgvector default 64)." + } + } + }, + "ivfflat": { + "type": "object", + "additionalProperties": false, + "description": "IVFFlat index tuning (used when indexType is 'ivfflat').", + "properties": { + "lists": { + "type": "integer", + "minimum": 1, + "description": "Number of inverted lists (pgvector guidance: rows/1000 for < 1M rows)." + } + } + } + } + }, + "location": { + "$ref": "#/$defs/bindingLocation" + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Binding configuration tags for infrastructure." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Binding configuration labels for automation." + }, + "properties": { + "type": "object", + "description": "Provider-specific binding properties (e.g., cluster_by, comment for Snowflake).", + "additionalProperties": true + }, + "governance": { + "$ref": "#/$defs/bindingGovernance", + "description": "Per-resource governance: registerLocation, principal grants, tag associations, row/column filters. Account-wide governance (admins, tag definitions) lives at the contract top-level governance block." + }, + "packaging": { + "$ref": "#/$defs/packaging", + "description": "NEW in v0.7.6: Per-exposure packaging override β€” key-wise precedence over the contract-wide top-level `packaging` block (binding.packaging > packaging > absent-legacy)." + }, + "encryption": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.6: Encryption at rest for an AWS S3 binding or a GCP BigQuery binding. Absent: nothing is emitted, and the cloud's default applies: SSE-S3 for new S3 objects, Google-managed keys for BigQuery.", + "properties": { + "kms": { + "type": "string", + "default": "product", + "anyOf": [ + { + "enum": [ + "product", + "none" + ] + }, + { + "pattern": "^alias/[A-Za-z0-9/_-]{1,250}$" + }, + { + "pattern": "^arn:aws[a-z-]*:kms:[a-z0-9-]+:[0-9]{12}:(key|alias)/[A-Za-z0-9/_-]{1,250}$" + }, + { + "pattern": "^projects/[a-z0-9.:-]{1,100}/locations/[a-z0-9-]{1,63}/keyRings/[A-Za-z0-9_-]{1,63}/cryptoKeys/[A-Za-z0-9_-]{1,63}$" + } + ], + "description": "NEW in v0.7.6: The KMS key the bucket's objects are encrypted with (SSE-KMS with an S3 Bucket Key). 'product' (the default): `fluid apply` creates a customer managed key for each bucket this product owns (aws_kms_key, rotation on, alias alias/fluid//) and makes it the bucket's default encryption (aws_s3_bucket_server_side_encryption_configuration). Its key policy lets the account's IAM policies decide who may use it (the KMS default statement for the account root) and, when the binding carries governance.lakeFormation, lets the Lake Formation service-linked role AWSServiceRoleForLakeFormationDataAccess encrypt and decrypt, matched by aws:PrincipalArn so the role need not exist yet. A principal querying through Athena with credentials Lake Formation vends then needs no KMS permission of its own; a grantee in another account that the bucket policy lets read the objects directly (governance.lakeFormation.bucketPolicy) is also allowed kms:Decrypt, through S3 only. The writer (the identity the build runs as) needs kms:GenerateDataKey and kms:Decrypt from IAM. `tofu destroy` deletes the alias and schedules the key for deletion after 7 days, the minimum KMS allows; kms:CancelKeyDeletion recovers it until then. Dropping the block, or naming another key, removes the product key while the bucket keeps the objects encrypted under it, which become unreadable once the deletion window ends unless they are rewritten or the deletion is cancelled; `fluid apply` refuses that plan without --allow-data-loss. On a shared (pool) bucket the bucket's default encryption is the pool owner's, so 'product' is refused there. 'alias/' or a key or alias ARN: an existing key, looked up at plan time (data aws_kms_key, so the identity running `fluid apply` needs kms:DescribeKey on it) and made the default encryption of a bucket this product owns by its key ARN, since S3 resolves an alias in the account of whoever writes; the plan fails unless the key is Enabled and a symmetric encryption key (SYMMETRIC_DEFAULT). On a pool bucket it is only checked. Its key policy is its owner's, and must let the Lake Formation role use it when the location is registered: an AWS managed key cannot be used with that role, so alias/aws/s3 is refused with registerLocation, and the plan fails for any key that is not customer managed. 'none': nothing is emitted. `fluid verify` checks that the key is Enabled and that the objects under the binding's prefix are SSE-KMS with it. On a gcp binding (fluid-schema 0.7.6, BigQuery tables): 'product' creates a Cloud KMS key ring and crypto key for each dataset this product writes (google_kms_key_ring fluid-- and google_kms_crypto_key 'bigquery', in the dataset's location, EU and US mapping to the europe and us key locations, rotated every 90 days), grants the project's BigQuery service agent roles/cloudkms.cryptoKeyEncrypterDecrypter on it, and makes it the dataset's default_encryption_configuration and the table's encryption_configuration. The project must have the Cloud KMS API (cloudkms.googleapis.com) enabled. A key ring and key cannot be deleted on GCP: `tofu destroy` removes them from state and schedules the key's versions for destruction (30 days by default), and a later apply adopts the same names. 'projects/

/locations//keyRings//cryptoKeys/': an existing key in the dataset's location, used as given; its owner grants the BigQuery service agent. 'none': Google-managed keys. An alias/... or arn:... value is refused on a gcp binding, and a Cloud KMS name on an aws binding. Adding a key to a table that exists replaces the table (BigQuery cannot re-key it in place), so `fluid apply` refuses that plan without --allow-data-loss; the next build lands the data again. `fluid verify` checks kmsKeyName on the dataset and the table." + } + } + }, + "principals": { + "type": "object", + "description": "NEW in v0.7.6: Maps the contract's LOGICAL principals (accessPolicy.grants[].principal and exposes[].policy.authz.columnRestrictions[].principal, keyed exactly as the contract writes them) to the identities they stand for on this binding's platform, so the base contract stays cloud-neutral and each environment's overlay binds it. On gcp an identity is an IAM member (user:, group:, serviceAccount: or domain:); on aws an IAM principal ARN. A value is one identity or a list; [] says the principal has no identity on this cloud, and nothing is granted to it there. When the block is present, every principal the contract names for this expose must be a key: an unmapped one is refused, never emitted as written. Without the block the principals are used as written, except that a GCP principal in a reserved top-level domain (.example, .test, .invalid, .localhost) is refused as a placeholder. Modelled on ODCS v3's roles[] bound per server (servers[].roles) and on dbt grants resolved per target.", + "propertyNames": { + "minLength": 1 + }, + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + } + ] + } + }, + "serialization": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.6: How message VALUES (and keys) are serialized on the wire. A Kafka binding today declares only `format: kafka_topic`, which tells a consumer nothing about whether the values are Avro or JSON β€” the one thing it has to know to read the topic at all. The first four values are the message formats ODCS v3.1.0 documents for its own Kafka server `format` field; the two `-confluent` values name those same wire formats carried through a Confluent schema registry, spelled as Apache Flink spells them.", + "properties": { + "value": { + "type": "string", + "enum": [ + "json", + "avro", + "protobuf", + "xml", + "avro-confluent", + "json-confluent" + ], + "description": "How message values are serialized." + }, + "key": { + "type": "string", + "enum": [ + "json", + "avro", + "protobuf", + "xml", + "avro-confluent", + "json-confluent" + ], + "description": "How message keys are serialized, for a topic whose keys are not opaque bytes." + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "platform": { + "const": "gcp" + } + }, + "required": [ + "platform" + ] + }, + "then": { + "properties": { + "encryption": { + "properties": { + "kms": { + "not": { + "pattern": "^(alias/|arn:)" + } + } + } + }, + "principals": { + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "pattern": "^(user|group|serviceAccount|domain):.+$" + }, + { + "type": "array", + "items": { + "type": "string", + "pattern": "^(user|group|serviceAccount|domain):.+$" + } + } + ] + } + } + } + } + }, + { + "if": { + "properties": { + "platform": { + "const": "aws" + } + }, + "required": [ + "platform" + ] + }, + "then": { + "properties": { + "encryption": { + "properties": { + "kms": { + "not": { + "pattern": "^projects/" + } + } + } + }, + "principals": { + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "pattern": "^arn:aws[a-z0-9-]*:iam::" + }, + { + "type": "array", + "items": { + "type": "string", + "pattern": "^arn:aws[a-z0-9-]*:iam::" + } + } + ] + } + } + } + } + } + ] + }, + "bindingLocation": { + "type": "object", + "description": "Provider-native addressing with tightened structure for better interop.", + "additionalProperties": false, + "properties": { + "account": { + "type": "string", + "description": "Cloud account ID" + }, + "project": { + "type": "string", + "description": "GCP project ID" + }, + "dataset": { + "type": "string", + "description": "BigQuery dataset or database name" + }, + "database": { + "type": "string", + "description": "Database name" + }, + "schema": { + "type": "string", + "description": "Schema name" + }, + "table": { + "type": "string", + "description": "Table name" + }, + "bucket": { + "type": "string", + "description": "Storage bucket name" + }, + "path": { + "type": "string", + "description": "File or object path" + }, + "gateway": { + "type": "string", + "description": "API gateway endpoint" + }, + "host": { + "type": "string", + "description": "NEW in v0.7.6: Host or endpoint; for a Kafka binding, the bootstrap servers. The ODCS server mapper already reads `host` and falls back to `account`, so until now the only valid way to set the bootstrap servers was a field named for something else β€” and with neither set, the exported ODCS server is the placeholder `${KAFKA_BOOTSTRAP_SERVERS}`. Usually set per environment in an overlay rather than in the base contract." + }, + "baseUrl": { + "type": "string", + "format": "uri", + "description": "Base URL for API endpoints" + }, + "topic": { + "type": "string", + "description": "Pub/Sub or Kafka topic name" + }, + "subscription": { + "type": "string", + "description": "Pub/Sub subscription name" + }, + "region": { + "type": "string", + "description": "Cloud region" + }, + "zone": { + "type": "string", + "description": "Cloud zone" + }, + "environment_id": { + "type": "string", + "description": "NEW in v0.7.5: Confluent Cloud environment id (env-xxxxx) for the Tableflow emitter." + }, + "kafka_cluster_id": { + "type": "string", + "description": "NEW in v0.7.5: Confluent Cloud Kafka cluster id (lkc-xxxxx) the Tableflow topic belongs to." + }, + "stream": { + "type": "string", + "description": "NEW in v0.7.5: AWS Kinesis Data Stream name β€” maps to aws_kinesis_stream in the streaming IaC emitter." + }, + "namespace": { + "type": "string", + "description": "NEW in v0.7.5: Amazon Redshift Serverless namespace name β€” maps to aws_redshiftserverless_namespace." + }, + "workgroup": { + "type": "string", + "description": "NEW in v0.7.5: Amazon Redshift Serverless workgroup name β€” maps to aws_redshiftserverless_workgroup." + }, + "iam_role_arn": { + "type": "string", + "description": "NEW in v0.7.5: ARN of the IAM role attached to the Redshift Serverless namespace / used by the redshift-data CREATE EXTERNAL SCHEMA bridge." + }, + "external_schema": { + "type": "string", + "description": "NEW in v0.7.5: Redshift external (Spectrum) schema name created over a Glue database via the redshift-data CREATE EXTERNAL SCHEMA bridge." + }, + "glue_database": { + "type": "string", + "description": "NEW in v0.7.5: Glue Data Catalog database that backs the Redshift external schema." + }, + "confluent_role_arn": { + "type": "string", + "description": "NEW in v0.7.5: ARN of the pre-created AWS IAM role Confluent Tableflow assumes (byob_aws). Its trust policy is updated post-apply with the provider integration's external_id (two-phase IAM)." + }, + "catalog": { + "type": "string", + "description": "NEW in v0.7.5: The Iceberg catalog that owns this table: glue, rest, lakekeeper, polaris, unity, nessie, bigquery, hive, jdbc, hadoop, dynamodb or snowflake-managed. Spellings fold case and '-'/'_', so iceberg_rest == rest and snowflake == snowflake-managed; fluid validate refuses any other value on an Iceberg expose. When absent: glue on platform aws, snowflake-managed on snowflake, rest elsewhere. The REST catalogs (rest, lakekeeper, polaris, unity) need location.uri plus location.warehouse as the catalog's warehouse NAME, not an object-store URI; Lakekeeper serves its Iceberg REST API under /catalog (uri: http://lakekeeper:8181/catalog). On platform aws, a kind other than glue gets no Glue table and no Glue grant: that catalog creates and governs the table. Every emitter (streaming sinks, dbt catalogs.yml, the IaC, policy compile) reads one table, providers/_iceberg_catalog.py." + }, + "warehouse": { + "type": "string", + "description": "NEW in v0.7.5: Iceberg warehouse β€” the canonical Iceberg REST 'warehouse' property (maps to iceberg.catalog.warehouse): an object-store URI (s3://|gs://|abfss://) for object-store catalogs, or the catalog/warehouse name for a REST catalog." + }, + "uri": { + "type": "string", + "description": "NEW in v0.7.5: REST Iceberg catalog endpoint URL β€” the canonical Iceberg REST 'uri' property (maps to iceberg.catalog.uri)." + }, + "partitionBy": { + "type": "array", + "items": { + "type": "string" + }, + "description": "NEW in v0.7.5: Iceberg partition columns β€” a FLUID abstraction over the Iceberg PARTITIONED BY / PartitionSpec (maps to iceberg.tables.default-partition-by). On a gcp BigQuery table with exposes[].lifecycle.expire: true (fluid-schema 0.7.6), one DATE, TIMESTAMP or DATETIME column to partition by (time_partitioning.field); without it the table is partitioned by ingestion time. With a column, retention counts from the date in that column, not from landing: BigQuery deletes a partition `retention` after its date, so a backfill of older rows is deleted at once. Leave it out for the S3 rule's semantics (age since written)." + } + } + }, + "observability": { + "type": "object", + "additionalProperties": false, + "properties": { + "metrics": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "source" + ], + "properties": { + "name": { + "type": "string" + }, + "source": { + "type": "string" + }, + "sli": { + "type": "string", + "description": "Human/machine-readable SLI check description" + } + } + } + }, + "onBreach": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "slack", + "email", + "webhook", + "pagerduty", + "custom" + ] + }, + "channel": { + "type": "string" + }, + "target": { + "type": "string" + }, + "url": { + "type": "string", + "format": "uri" + }, + "severity": { + "type": "string", + "enum": [ + "info", + "warning", + "critical" + ] + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Alert configuration tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Alert configuration labels." + } + } + } + }, + "defaultSLIs": { + "$ref": "#/$defs/defaultSLIs", + "description": "Default SLI trio (freshness, completeness, latency) by kind." + }, + "alert": { + "type": "object", + "additionalProperties": false, + "description": "Alert dispatcher config consumed by the acquisition Alerter (DLQ overflow, schema-fingerprint changes, anomaly signals).", + "properties": { + "channels": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "log", + "file", + "webhook" + ] + }, + "path": { + "type": "string", + "description": "For kind=file: NDJSON sink path (env-vars expanded)." + }, + "url": { + "type": "string", + "format": "uri", + "description": "For kind=webhook: Slack-compatible incoming webhook URL. Validated against the SSRF guard at construction time." + } + } + } + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Observability configuration tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Observability configuration labels." + } + } + }, + "defaultSLIs": { + "type": "object", + "description": "Default SLI trio that CLI can auto-generate based on expose kind.", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "default": true, + "description": "Enable auto-generation of default SLIs." + }, + "freshness": { + "type": "object", + "description": "Default freshness SLI configuration.", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "default": true + }, + "threshold": { + "$ref": "#/$defs/isoDuration", + "description": "Freshness threshold (auto-set by kind: table=PT6H, stream=PT1M, api=PT1S)" + }, + "severity": { + "type": "string", + "enum": [ + "info", + "warning", + "critical" + ], + "default": "warning" + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Freshness SLI tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Freshness SLI labels." + } + } + }, + "completeness": { + "type": "object", + "description": "Default completeness SLI configuration.", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "default": true + }, + "threshold": { + "type": "number", + "minimum": 0, + "maximum": 1, + "default": 0.95, + "description": "Completeness threshold (auto-set by kind: table=0.95, stream=0.99)" + }, + "severity": { + "type": "string", + "enum": [ + "info", + "warning", + "critical" + ], + "default": "warning" + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Completeness SLI tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Completeness SLI labels." + } + } + }, + "latency": { + "type": "object", + "description": "Default latency SLI configuration.", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "default": true + }, + "threshold": { + "$ref": "#/$defs/isoDuration", + "description": "P95 latency threshold (auto-set by kind: api=PT200MS, table=PT5S, stream=PT100MS)" + }, + "percentile": { + "type": "number", + "minimum": 0, + "maximum": 100, + "default": 95, + "description": "Latency percentile to measure." + }, + "severity": { + "type": "string", + "enum": [ + "info", + "warning", + "critical" + ], + "default": "warning" + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Latency SLI tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Latency SLI labels." + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Default SLI configuration tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Default SLI configuration labels." + } + } + }, + "lifecycle": { + "type": "object", + "additionalProperties": false, + "properties": { + "state": { + "$ref": "#/$defs/lifecycleState", + "description": "Unified lifecycle state vocabulary." + }, + "retention": { + "$ref": "#/$defs/isoDuration" + }, + "deprecationPolicy": { + "type": "object", + "additionalProperties": false, + "properties": { + "noticePeriod": { + "$ref": "#/$defs/isoDuration" + }, + "contact": { + "type": "string" + }, + "replacement": { + "type": "string", + "description": "Reference to replacement data product or expose." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Deprecation policy tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Deprecation policy labels." + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Lifecycle tags for automation." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Lifecycle labels for governance." + } + } + }, + "exposeLifecycle": { + "type": "object", + "additionalProperties": false, + "properties": { + "state": { + "$ref": "#/$defs/lifecycleState", + "description": "Unified lifecycle state vocabulary." + }, + "retention": { + "$ref": "#/$defs/isoDuration", + "description": "How long this expose's data is kept (ISO-8601 duration, e.g. P30D, P7Y). A declaration unless `expire` is true. With `expire: true` on an AWS S3 binding, `fluid apply` expires the objects under the binding's prefix this long after they are written; on a GCP BigQuery table it deletes each daily partition this long after its day ends. Years and months are counted as 365 and 30 days, and a part of a day rounds up to a whole one." + }, + "expire": { + "type": "boolean", + "default": false, + "description": "NEW in v0.7.6: Delete stored data once it is older than `retention`. Default false, so a contract that declares a retention period does not start deleting data when forge-cli is upgraded. Honoured by the AWS emitter on an S3 binding and by the GCP emitter on a BigQuery table; on a gcp binding that is not a BigQuery table (GCS, Pub/Sub, Iceberg storage) it is refused rather than dropped. AWS: `fluid apply` writes one rule per expose into the bucket's aws_s3_bucket_lifecycle_configuration, filtered to the binding's prefix (location.path, else {database}/{table}/; a binding with neither is refused, so no rule ever expires a whole bucket). The rule expires the current version `retention` after it is written, removes a noncurrent version (on a versioning-enabled bucket) one day after it stops being current, and aborts an incomplete multipart upload after 7 days or the retention period if shorter. The lifecycle configuration is AUTHORITATIVE for the whole bucket: S3 keeps one per bucket, so rules that `fluid apply` did not write are replaced. It is therefore written only for a bucket this product owns; on a shared (pool) bucket (packaging) nothing is written, the pool's owner must hold the rule, and `fluid verify` checks that an enabled rule covering the prefix expires objects after exactly this period. Objects already older than the period when the rule is first applied are expired on S3's next lifecycle run, typically within a day. Removing `expire` later deletes the lifecycle configuration, which `fluid apply` refuses without --allow-data-loss (it refuses every plan that removes a resource holding data or policy; removing an access grant or a policy tag is a revocation and is not gated); `tofu destroy` deletes it. GCP (fluid-schema 0.7.6): the BigQuery table is partitioned by day (time_partitioning type DAY, on binding.location.partitionBy when it names one date or timestamp column, else by ingestion time) with expiration_ms = `retention` in days, so BigQuery deletes each partition that long after its day ends. BigQuery counts from the partition's date, not from when its rows were written: by ingestion time (no partitionBy, the default and the S3 rule's semantics) no row is deleted sooner than `retention` after it landed; with partitionBy, retention is the age of the date in that column, so a backfill of rows whose date is already older than `retention` lands in expired partitions and is deleted at once. It is never a whole-table expiration, which would delete the product. BigQuery cannot partition a table that exists, so adding `expire` to a live table replaces it: `fluid apply` refuses that plan without --allow-data-loss, and the next build lands the data again. A later change of `retention` is an in-place update. Removing `expire` from a live table removes the partitioning's replacement trigger, which `fluid apply` also refuses without --allow-data-loss; BigQuery cannot un-partition a table, so to keep data longer set a longer `retention` instead. `fluid verify` checks the live table's partition expiration." + }, + "deprecationPolicy": { + "type": "object", + "additionalProperties": false, + "properties": { + "noticePeriod": { + "$ref": "#/$defs/isoDuration" + }, + "contact": { + "type": "string" + }, + "replacement": { + "type": "string", + "description": "Reference to replacement data product or expose." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Deprecation policy tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Deprecation policy labels." + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Lifecycle tags for automation." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Lifecycle labels for governance." + } + }, + "description": "NEW in v0.7.6: An expose's lifecycle. The same fields as the contract-root lifecycle, plus `expire`, which is only meaningful where there is storage to expire." + }, + "docs": { + "type": "object", + "additionalProperties": false, + "properties": { + "homepage": { + "type": "string", + "format": "uri" + }, + "runbook": { + "type": "string", + "format": "uri" + }, + "dictionary": { + "type": "string", + "format": "uri" + }, + "changeLog": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Documentation tags for organization." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Documentation labels for automation." + } + } + }, + "agentPolicy": { + "type": "object", + "description": "NEW in v0.7.1: AI/LLM usage governance. Controls which AI models can consume data and defines usage boundaries. Enables data owners to enforce 'reporting yes, training no' policies.", + "additionalProperties": false, + "properties": { + "allowedModels": { + "type": "array", + "description": "Whitelist of AI models permitted to consume this data. Examples: ['gpt-4', 'claude-3-opus', 'gemini-pro']. Empty array = no AI access.", + "items": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-_.]*[a-z0-9]$|^[a-z0-9]$", + "examples": [ + "gpt-4", + "claude-3-opus", + "claude-3-sonnet", + "gemini-pro", + "gemini-ultra", + "llama-3", + "mixtral-8x7b" + ] + }, + "uniqueItems": true + }, + "deniedModels": { + "type": "array", + "description": "Blacklist of AI models explicitly prohibited from consuming this data. Takes precedence over allowedModels.", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "maxTokensPerRequest": { + "type": "integer", + "minimum": 1, + "description": "Maximum tokens per AI request. Prevents excessive data exposure in single queries.", + "examples": [ + 4096, + 8192, + 16384 + ] + }, + "maxTokensPerDay": { + "type": "integer", + "minimum": 1, + "description": "Daily token limit for AI consumption. Enforces usage quotas." + }, + "allowedUseCases": { + "type": "array", + "description": "Permitted AI use cases. Controls whether data can be used for training, inference, reasoning, etc.", + "items": { + "type": "string", + "enum": [ + "inference", + "reasoning", + "analysis", + "summarization", + "classification", + "embedding", + "search", + "qa", + "code_generation", + "fine_tuning", + "training", + "rag" + ] + }, + "uniqueItems": true, + "examples": [ + [ + "inference", + "reasoning", + "summarization" + ], + [ + "search", + "qa", + "rag" + ] + ] + }, + "deniedUseCases": { + "type": "array", + "description": "Prohibited AI use cases. Example: Allow reporting but deny model training.", + "items": { + "type": "string", + "enum": [ + "inference", + "reasoning", + "analysis", + "summarization", + "classification", + "embedding", + "search", + "qa", + "code_generation", + "fine_tuning", + "training", + "rag" + ] + }, + "uniqueItems": true, + "examples": [ + [ + "training", + "fine_tuning" + ] + ] + }, + "canReason": { + "type": "boolean", + "default": false, + "description": "Whether AI agents can use this data for multi-step reasoning/chain-of-thought. False = simple retrieval only." + }, + "canStore": { + "type": "boolean", + "default": false, + "description": "Whether AI systems can cache/store this data. False = ephemeral processing only." + }, + "retentionPolicy": { + "type": "object", + "description": "Data retention requirements for AI systems consuming this data.", + "properties": { + "maxRetentionDays": { + "type": "integer", + "minimum": 0, + "description": "Maximum days AI system can retain this data. 0 = no retention allowed." + }, + "requireDeletion": { + "type": "boolean", + "default": true, + "description": "Whether AI system must delete data after use." + } + } + }, + "auditRequired": { + "type": "boolean", + "default": true, + "description": "Whether AI consumption must be logged/audited." + }, + "purposeLimitation": { + "type": "string", + "description": "Free-text description of allowed purposes. Example: 'Customer service chatbot only, not for marketing'." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Agent policy tags for categorization." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Agent policy labels for automation." + } + } + }, + "sovereignty": { + "type": "object", + "description": "NEW in v0.7.1: Data sovereignty and jurisdiction requirements. Declares WHERE data must reside and enables compliance enforcement before deployment.", + "additionalProperties": false, + "properties": { + "jurisdiction": { + "type": "string", + "description": "Required legal jurisdiction for data storage and processing. Used to validate binding.location matches sovereignty intent.", + "enum": [ + "EU", + "US", + "UK", + "CA", + "AU", + "JP", + "CN", + "IN", + "BR", + "Global", + "Multi-Region" + ], + "examples": [ + "EU", + "US", + "Global" + ] + }, + "allowedRegions": { + "type": "array", + "description": "Explicit list of allowed cloud regions. Example: ['eu-west-1', 'eu-central-1'] for EU-only.", + "items": { + "type": "string" + }, + "uniqueItems": true, + "examples": [ + [ + "eu-west-1", + "eu-central-1" + ], + [ + "us-east-1", + "us-west-2" + ] + ] + }, + "deniedRegions": { + "type": "array", + "description": "Explicit list of prohibited cloud regions. Takes precedence over allowedRegions.", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "dataResidency": { + "type": "boolean", + "default": true, + "description": "Whether data must remain within jurisdiction boundaries at rest and in transit. True = strict residency." + }, + "crossBorderTransfer": { + "type": "boolean", + "default": false, + "description": "Whether cross-border data transfer is permitted. False = data never leaves jurisdiction." + }, + "transferMechanisms": { + "type": "array", + "description": "Allowed legal mechanisms for cross-border transfer (if permitted). Example: ['SCCs', 'BCRs', 'Adequacy'].", + "items": { + "type": "string", + "enum": [ + "SCCs", + "BCRs", + "Adequacy", + "DPF", + "Consent", + "Derogation" + ] + }, + "uniqueItems": true + }, + "regulatoryFramework": { + "type": "array", + "description": "Applicable regulatory frameworks. Example: ['GDPR', 'CCPA', 'HIPAA'].", + "items": { + "type": "string", + "enum": [ + "GDPR", + "CCPA", + "CPRA", + "HIPAA", + "PIPEDA", + "LGPD", + "PDPA", + "POPIA", + "DPA", + "APPI" + ] + }, + "uniqueItems": true, + "examples": [ + [ + "GDPR" + ], + [ + "GDPR", + "CCPA" + ], + [ + "HIPAA" + ] + ] + }, + "enforcementMode": { + "type": "string", + "enum": [ + "strict", + "advisory", + "audit" + ], + "default": "strict", + "description": "How sovereignty violations are handled. strict = block deployment, advisory = warn, audit = log only." + }, + "validationRequired": { + "type": "boolean", + "default": true, + "description": "Whether binding.location must be validated against sovereignty rules before deployment." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Sovereignty policy tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Sovereignty policy labels." + } + }, + "examples": [ + { + "jurisdiction": "EU", + "allowedRegions": [ + "eu-west-1", + "eu-central-1" + ], + "dataResidency": true, + "crossBorderTransfer": false, + "regulatoryFramework": [ + "GDPR" + ], + "enforcementMode": "strict" + }, + { + "jurisdiction": "US", + "allowedRegions": [ + "us-east-1", + "us-west-2" + ], + "dataResidency": true, + "regulatoryFramework": [ + "CCPA", + "HIPAA" + ], + "enforcementMode": "strict" + } + ] + }, + "accessPolicy": { + "type": "object", + "description": "Root-level access control policy for automated IAM binding generation. Defines who can access this data product and with what permissions.", + "additionalProperties": false, + "properties": { + "grants": { + "type": "array", + "description": "List of access grants to principals (users, groups, service accounts).", + "items": { + "type": "object", + "required": [ + "principal" + ], + "additionalProperties": false, + "properties": { + "principal": { + "type": "string", + "description": "Identity receiving access. Format: 'user:email@domain.com', 'group:name@domain.com', 'serviceAccount:name@project.iam.gserviceaccount.com'", + "examples": [ + "group:data-analytics@company.com", + "serviceAccount:pipeline@project.iam.gserviceaccount.com", + "user:analyst@company.com" + ] + }, + "permissions": { + "type": "array", + "description": "Permissions to grant. Will be mapped to provider-specific roles (e.g., BigQuery roles).", + "items": { + "type": "string", + "enum": [ + "read", + "select", + "query", + "write", + "insert", + "update", + "delete", + "create", + "admin", + "manage" + ] + }, + "uniqueItems": true, + "examples": [ + [ + "read", + "select" + ], + [ + "write", + "insert", + "update" + ] + ] + }, + "resources": { + "type": "array", + "description": "Resources this grant applies to. JSONPath expressions referencing exposes[] items. If omitted, applies to all exposed resources.", + "items": { + "type": "string" + }, + "examples": [ + "$.exposes[?(@.exposeId=='customer_table')]", + "$.exposes[?(@.kind=='table')]" + ] + }, + "conditions": { + "type": "object", + "description": "Optional conditions for conditional access (e.g., IP restrictions, time windows).", + "additionalProperties": true + } + } + } + } + }, + "examples": [ + { + "grants": [ + { + "principal": "group:data-analytics@company.com", + "permissions": [ + "read", + "select", + "query" + ] + }, + { + "principal": "serviceAccount:pipeline@project.iam.gserviceaccount.com", + "permissions": [ + "write", + "insert", + "update", + "delete" + ] + } + ] + } + ] + }, + "consumeRef": { + "type": "object", + "additionalProperties": false, + "required": [ + "productId", + "exposeId" + ], + "$comment": "productId identifies data product (renamed from 'provider' for clarity)", + "properties": { + "productId": { + "$ref": "#/$defs/identifier", + "description": "Upstream data product identifier (e.g., 'silver.hr.people_v2'). Renamed from 'provider' for clarity." + }, + "exposeId": { + "$ref": "#/$defs/identifier", + "description": "The upstream exposeId being consumed." + }, + "contractId": { + "type": "string", + "description": "NEW in v0.7.6: The upstream contract's OWN id, for an upstream that has one of its own β€” a product not built with FLUID, registered in a catalog under an id FLUID did not derive. The canonical-port builder already prefers an explicit contractId over the derived `.`; until now `consumes[]` was closed, so there was no way to supply it and the derived id was exported as if the upstream answered to it." + }, + "versionConstraint": { + "$ref": "#/$defs/semverRange" + }, + "contractVersion": { + "$ref": "#/$defs/semver", + "description": "NEW in v0.7.6: The exact upstream contract version this product was built and tested against, within `versionConstraint`. Exported as the ODPS `inputPorts[].version`, and as the version of the matching `outputPorts[].inputContracts[]` entry. Without it the export derives the range's lower bound; for a range it cannot read it falls back to 1.0.0, because ODPS v1.0.0 requires a version on every input port and the field cannot simply be omitted. `versionConstraint` stays the range this product accepts; this is the single version its exported lineage claims. `fluid validate` does not check that it falls inside `versionConstraint`.", + "examples": [ + "2.1.0" + ] + }, + "qosExpectations": { + "type": "object", + "additionalProperties": false, + "properties": { + "freshnessMax": { + "$ref": "#/$defs/isoDuration" + }, + "maxStaleness": { + "$ref": "#/$defs/isoDuration", + "description": "Maximum acceptable staleness for consumed data." + }, + "minCompleteness": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Minimum completeness expectation." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "QoS expectation tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "QoS expectation labels." + } + } + }, + "requiredPolicies": { + "type": "array", + "items": { + "type": "string" + } + }, + "purpose": { + "type": "string", + "description": "Business purpose for consuming this data." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Consumption relationship tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Consumption relationship labels." + }, + "upstreamWorkspace": { + "type": "string", + "minLength": 1, + "description": "Cross-mesh only. Id of the federated workspace that owns this upstream, as declared in federation/upstreams.yaml. Set it when the upstream lives in another mesh rather than this workspace; omit it for a local upstream, which is resolved by walking the workspace." + }, + "upstreamDigest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$", + "description": "Cross-mesh only. The contract digest this product was composed against, as sha256:<64 hex>. `fluid apply` re-fetches the upstream's live digest and compares: a mismatch means the upstream changed since composition. Required whenever upstreamWorkspace is set." + } + }, + "dependentRequired": { + "upstreamWorkspace": [ + "upstreamDigest" + ] + } + }, + "build": { + "type": "object", + "description": "Enhanced build configuration supporting v0.4.0 patterns with v0.5.5 improvements.", + "additionalProperties": false, + "properties": { + "id": { + "$ref": "#/$defs/identifier", + "description": "Build identifier for multi-build scenarios (e.g., 'batch-processing', 'ml-training')." + }, + "description": { + "type": "string", + "description": "Human-readable description of this build configuration." + }, + "pattern": { + "type": "string", + "enum": [ + "hybrid-reference", + "embedded-logic", + "multi-stage", + "acquisition" + ], + "default": "hybrid-reference", + "description": "Build pattern: hybrid-reference (dbt-style), embedded-logic (raw SQL/code), multi-stage (complex pipelines), or acquisition (source-aligned ingestion from external systems)." + }, + "engine": { + "type": "string", + "anyOf": [ + { + "enum": [ + "dbt", + "sql", + "python", + "spark", + "glue", + "custom", + "duckdb", + "airbyte", + "meltano", + "dlt", + "kafka-connect", + "debezium", + "flink" + ] + }, + { + "pattern": "^dbt-[a-z0-9]+([_-][a-z0-9]+)*$" + } + ], + "default": "dbt", + "description": "Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-qualified dbt engines (dbt-, e.g. dbt-glue, dbt-snowflake, dbt-bigquery) are accepted too and route through the dbt build runner, which derives the adapter from the suffix. Ingestion engines (acquisition pattern only): duckdb (zero-infra files/JDBC), airbyte (350+ SaaS), meltano (Singer 600+ taps), dlt (Python-native), kafka-connect (streaming), debezium (CDC). AWS Glue: glue (Spark ETL jobs managed by Glue). NEW in v0.7.6: flink (Apache Flink jobs, usually long-running streaming SQL with properties.language: flink_sql). forge-cli validates and plans a flink build but ships no runner for it: the platform that runs the job, such as the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, executes it. Put deployment settings under properties.parameters, for example parallelism and upgradeMode, as the Flink Kubernetes Operator's job spec names them, and flinkConfiguration." + }, + "capabilities": { + "type": "array", + "description": "Capabilities the build asks for (acquisition pattern). Validation enforces capabilities βŠ† runner.declared_capabilities.", + "items": { + "type": "string", + "enum": [ + "full_refresh", + "incremental_append", + "incremental_dedup", + "incremental_merge", + "cdc", + "streaming", + "schema_discovery", + "schema_evolution", + "dlp_scan", + "at_most_once", + "at_least_once", + "exactly_once" + ] + }, + "uniqueItems": true + }, + "repository": { + "type": "string" + }, + "properties": { + "$ref": "#/$defs/buildProperties", + "description": "Pattern-specific build properties with conditional validation." + }, + "execution": { + "$ref": "#/$defs/buildExecution", + "description": "Build execution configuration (triggers, runtime, retries)." + }, + "outputs": { + "type": "array", + "description": "List of exposeIds produced by this build.", + "items": { + "$ref": "#/$defs/identifier" + } + }, + "dependencies": { + "type": "array", + "description": "Dependencies on other builds (for multi-build orchestration).", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "buildId": { + "$ref": "#/$defs/identifier" + }, + "condition": { + "type": "string", + "enum": [ + "success", + "completion", + "always" + ] + } + } + } + }, + "transformations": { + "type": "array", + "description": "Legacy transformations array for backward compatibility.", + "items": { + "$ref": "#/$defs/transformation" + } + }, + "tags": { + "$ref": "#/$defs/tags" + }, + "labels": { + "$ref": "#/$defs/labels" + } + }, + "allOf": [ + { + "if": { + "properties": { + "pattern": { + "const": "hybrid-reference" + } + } + }, + "then": { + "properties": { + "properties": { + "$ref": "#/$defs/hybridReferencePattern" + } + } + } + }, + { + "if": { + "properties": { + "pattern": { + "const": "embedded-logic" + } + } + }, + "then": { + "properties": { + "properties": { + "$ref": "#/$defs/embeddedLogicPattern" + } + } + } + }, + { + "if": { + "properties": { + "pattern": { + "const": "multi-stage" + } + } + }, + "then": { + "properties": { + "properties": { + "$ref": "#/$defs/multiStagePattern" + } + } + } + }, + { + "if": { + "properties": { + "pattern": { + "const": "acquisition" + } + } + }, + "then": { + "properties": { + "properties": { + "$ref": "#/$defs/acquisitionPattern" + } + } + } + } + ] + }, + "buildProperties": { + "type": "object", + "description": "Base properties for all build patterns." + }, + "hybridReferencePattern": { + "type": "object", + "description": "Hybrid-reference pattern from v0.4.0 (dbt-style).", + "additionalProperties": false, + "properties": { + "model": { + "type": "string" + }, + "target": { + "type": "string", + "description": "dbt profile target to run against. Forwarded to dbt as `--target ` by the build runner (allowlisted against the profile's real targets; falls back to the profile default when the requested target is unavailable)." + }, + "select": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": "dbt node selector(s) forwarded to `dbt --select`. Accepts a single selector string or an array of selector strings; the build runner drops flag-like values (those starting with '-')." + }, + "models": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": "Legacy alias for `select` (dbt's older `--models` flag). Same string-or-array shape; `select` takes precedence when both are set." + }, + "vars": { + "type": "object", + "additionalProperties": true + }, + "materializations": { + "type": "object", + "additionalProperties": { + "type": "string", + "enum": [ + "table", + "view", + "incremental", + "ephemeral" + ] + } + }, + "tags": { + "$ref": "#/$defs/tags" + }, + "labels": { + "$ref": "#/$defs/labels" + } + }, + "required": [ + "model" + ] + }, + "embeddedLogicPattern": { + "type": "object", + "description": "Embedded logic pattern from v0.4.0.", + "additionalProperties": false, + "properties": { + "sql": { + "type": "string" + }, + "language": { + "type": "string", + "enum": [ + "sql", + "flink_sql", + "pyspark", + "scala", + "python", + "r" + ] + }, + "parameters": { + "type": "object", + "additionalProperties": true + }, + "stream": { + "$ref": "#/$defs/streamSource", + "description": "NEW in v0.7.7 (preview): The stream the build reads. On a managed Flink build (parameters.flink.management) sql is optional: the lane writes the query from the mapping instead. Elsewhere the stream describes what the build's own sql reads." + }, + "mapping": { + "$ref": "#/$defs/recordMapping", + "description": "NEW in v0.7.7 (preview): How the stream's records become the output's rows. Requires stream, and replaces sql." + }, + "tags": { + "$ref": "#/$defs/tags" + }, + "labels": { + "$ref": "#/$defs/labels" + } + }, + "if": { + "not": { + "required": [ + "stream", + "parameters" + ], + "properties": { + "parameters": { + "required": [ + "flink" + ], + "properties": { + "flink": { + "required": [ + "management" + ] + } + } + } + } + } + }, + "then": { + "required": [ + "sql" + ] + }, + "dependentRequired": { + "mapping": [ + "stream" + ] + } + }, + "multiStagePattern": { + "type": "object", + "description": "NEW in v0.5.5: Multi-stage orchestration pattern.", + "additionalProperties": false, + "properties": { + "stages": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "pattern": { + "type": "string", + "enum": [ + "hybrid-reference", + "embedded-logic" + ], + "default": "hybrid-reference" + }, + "properties": { + "$ref": "#/$defs/buildProperties" + }, + "dependsOn": { + "type": "array", + "items": { + "type": "string" + } + }, + "outputs": { + "type": "array", + "items": { + "$ref": "#/$defs/identifier" + } + }, + "tags": { + "$ref": "#/$defs/tags" + }, + "labels": { + "$ref": "#/$defs/labels" + } + } + } + }, + "orchestration": { + "type": "object", + "properties": { + "parallelism": { + "type": "integer", + "minimum": 1 + }, + "retryPolicy": { + "$ref": "#/$defs/retryPolicy" + } + } + }, + "tags": { + "$ref": "#/$defs/tags" + }, + "labels": { + "$ref": "#/$defs/labels" + } + } + }, + "acquisitionPattern": { + "type": "object", + "description": "NEW in v0.7.3: Source-aligned acquisition pattern. The build ingests data from an external system (no transformation). Engine ∈ {duckdb, airbyte, meltano, dlt, kafka-connect, debezium}.", + "additionalProperties": false, + "required": [ + "source" + ], + "properties": { + "source": { + "$ref": "#/$defs/acquisitionSource" + }, + "sink": { + "$ref": "#/$defs/acquisitionSink" + }, + "delivery": { + "$ref": "#/$defs/acquisitionDelivery" + }, + "schemaEvolution": { + "$ref": "#/$defs/acquisitionSchemaEvolution" + }, + "preLand": { + "type": "array", + "description": "Pre-land hook chain - runs on each batch before destination write.", + "items": { + "type": "string", + "enum": [ + "dlp_scan", + "tokenize_pii", + "quality_gate", + "emit_lineage_input" + ] + } + }, + "quality": { + "$ref": "#/$defs/acquisitionQuality" + }, + "cost": { + "$ref": "#/$defs/acquisitionCost" + }, + "catalog": { + "$ref": "#/$defs/acquisitionCatalog" + }, + "concurrency": { + "$ref": "#/$defs/acquisitionConcurrency" + }, + "lineage": { + "type": "object", + "additionalProperties": false, + "properties": { + "emit": { + "type": "boolean", + "default": true + } + } + }, + "duckdb": { + "type": "object", + "additionalProperties": true, + "properties": { + "extensions": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "airbyte": { + "type": "object", + "additionalProperties": true, + "properties": { + "connector_image": { + "type": "string" + }, + "version": { + "type": "string" + }, + "normalization": { + "type": "boolean" + }, + "image_signature": { + "$ref": "#/$defs/acquisitionImageSignature" + }, + "deployment": { + "$ref": "#/$defs/acquisitionDeployment" + } + } + }, + "meltano": { + "type": "object", + "additionalProperties": true, + "properties": { + "tap": { + "type": "string" + }, + "project_dir": { + "type": "string" + }, + "deployment": { + "$ref": "#/$defs/acquisitionDeployment" + } + } + }, + "dlt": { + "type": "object", + "additionalProperties": true, + "properties": { + "source_module": { + "type": "string" + }, + "pipeline_name": { + "type": "string" + } + } + }, + "kafka-connect": { + "type": "object", + "additionalProperties": true, + "properties": { + "connector_class": { + "type": "string" + }, + "connector_name": { + "type": "string" + }, + "tasks_max": { + "type": "integer", + "minimum": 1, + "default": 1 + }, + "schema_registry": { + "type": "object", + "additionalProperties": false, + "description": "Confluent-compatible Schema Registry config. When ``url`` is set, the runner injects ``key.converter``/``value.converter`` for Avro into the emitted connector config.", + "properties": { + "url": { + "type": "string", + "format": "uri", + "description": "Schema Registry HTTP endpoint, e.g. http://schema-registry:8081" + }, + "auth": { + "type": "object", + "additionalProperties": false, + "properties": { + "secretRef": { + "type": "string" + } + } + } + } + }, + "sink_connector_name": { + "type": "string" + }, + "sink_connector_config": { + "type": "object", + "additionalProperties": true + }, + "image_signature": { + "$ref": "#/$defs/acquisitionImageSignature" + }, + "deployment": { + "$ref": "#/$defs/acquisitionDeployment" + }, + "iceberg_sink_enabled": { + "type": "boolean", + "description": "Opt-in: derive the Iceberg sink connector config. Defaults OFF when a hand-written sink_connector_config is present (RFC Β§6.2)." + }, + "sink_topics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Explicit topics the derived Iceberg sink consumes (else derived from source streams)." + }, + "iceberg_catalog_overrides": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Operator escape hatch: connector config keys merged LAST over the derived Iceberg sink config." + }, + "streamingSink": { + "type": "object", + "additionalProperties": false, + "description": "Optional Iceberg streaming-sink tuning (RFC Β§6.2). All keys optional.", + "properties": { + "commitIntervalMs": { + "type": "integer", + "minimum": 1 + }, + "routeField": { + "type": "string" + }, + "dynamicEnabled": { + "type": "boolean" + }, + "autoCreate": { + "type": "boolean" + }, + "evolveSchema": { + "type": "boolean" + }, + "upsertMode": { + "type": "boolean" + }, + "controlTopic": { + "type": "string" + } + } + } + } + }, + "debezium": { + "type": "object", + "additionalProperties": true, + "properties": { + "connector_class": { + "type": "string" + }, + "snapshot_mode": { + "type": "string", + "enum": [ + "initial", + "schema_only", + "never", + "when_needed", + "always" + ] + }, + "deployment": { + "$ref": "#/$defs/acquisitionDeployment" + } + } + } + } + }, + "acquisitionSource": { + "type": "object", + "description": "External source system specification.", + "additionalProperties": false, + "required": [ + "kind", + "mode" + ], + "properties": { + "kind": { + "type": "string", + "description": "Source system kind: filesystem, postgres, mysql, sqlite, http, salesforce, stripe, github, kafka, etc. Engine-specific catalog of supported kinds." + }, + "connection": { + "type": "object", + "description": "Connection details. Use ${VAR} placeholders for env values; use secretRef for credentials. Inline secrets are forbidden by validator.", + "additionalProperties": true, + "properties": { + "uri": { + "type": "string" + }, + "host": { + "type": "string" + }, + "port": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "string", + "description": "Allows ${ENV_VAR} placeholders that resolve to an integer at runtime." + } + ] + }, + "database": { + "type": "string" + }, + "instance_url": { + "type": "string" + }, + "secretRef": { + "type": "string", + "pattern": "^[a-z][a-z0-9_+.-]*://.+", + "description": "Reference to a secret in the configured backend (vault://, aws://, gcp://, azure://, env://)." + }, + "user": { + "type": "string", + "description": "NEW in v0.7.3: Username/principal for SQL- or REST-style sources. Required by most database drivers (Postgres/MySQL/Oracle/etc.) β€” without it the underlying client falls back to the OS-user identity and typically fails authentication. Goes alongside ``secretRef`` (or an inline ``password``) for the credential pair." + }, + "schema": { + "type": "string", + "description": "NEW in v0.7.3: Default schema/namespace within the source database to read from. Maps to engine-specific config: dlt sql_database schema=, Meltano tap-postgres filter_schemas=[], Airbyte source-postgres schemas=[], Debezium schema.include.list, Kafka Connect JDBC schema.pattern. Use ``schemas`` (array) to scope to multiple schemas at once." + }, + "schemas": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true, + "description": "NEW in v0.7.3: Multi-schema variant of ``schema``. When both are set, ``schemas`` wins and ``schema`` is treated as a deprecated alias. Engines that don't natively support multi-schema reads will iterate the list." + } + } + }, + "mode": { + "type": "string", + "enum": [ + "full_refresh", + "incremental_append", + "incremental_dedup", + "incremental_merge", + "cdc", + "streaming" + ] + }, + "cursor_field": { + "type": "string", + "description": "Column used for incremental cursor (e.g., updated_at, SystemModstamp)." + }, + "watermark": { + "type": "object", + "additionalProperties": false, + "properties": { + "strategy": { + "type": "string", + "enum": [ + "high_water_mark", + "log_position", + "lsn" + ] + }, + "allowedLateness": { + "$ref": "#/$defs/isoDuration" + } + } + }, + "streams": { + "type": "array", + "description": "List of streams/tables/objects to ingest.", + "items": { + "type": "string" + } + }, + "reader": { + "type": "object", + "additionalProperties": true, + "properties": { + "format": { + "type": "string", + "enum": [ + "csv", + "parquet", + "json", + "ndjson", + "avro", + "orc" + ] + }, + "options": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, + "acquisitionSink": { + "type": "object", + "description": "Destination format for ingested data. binding.platform on the expose remains the source of truth for where; sink.format describes how.", + "additionalProperties": false, + "properties": { + "format": { + "type": "string", + "enum": [ + "iceberg", + "delta", + "parquet", + "csv", + "json", + "snowflake_table", + "bigquery_table", + "redshift_table", + "duckdb_table" + ] + }, + "catalog": { + "type": "string", + "enum": [ + "rest", + "glue", + "nessie", + "unity", + "snowflake-managed", + "hive", + "lakekeeper", + "polaris" + ] + }, + "partitionBy": { + "type": "array", + "items": { + "type": "string" + } + }, + "catalogAuth": { + "type": "object", + "description": "NEW in v0.7.6: How the Iceberg sink config forge-cli DERIVES (kafka-connect with sink.format iceberg, embedded debezium) authenticates to an Iceberg REST catalog (rest, lakekeeper, polaris, unity, snowflake-managed). Names an environment variable, never a secret: forge-cli emits the secret as the catalog property value env:, which Apache Iceberg's REST client resolves from the environment of the process that runs the sink (the Kafka Connect worker, or the Debezium Server forge-cli launches). Iceberg drops a property whose variable is unset, so the embedded Debezium run refuses to start without it; forge-cli cannot see a Connect worker's environment. type oauth2 (client credentials): credentialEnv names a variable holding client_id:client_secret, oauth2ServerUri is the identity provider's token endpoint, scope is required (lakekeeper on a Keycloak-fronted Lakekeeper; PRINCIPAL_ROLE:ALL on Polaris with internal auth, whose oauth2ServerUri is /api/catalog/v1/oauth/tokens). type bearer: tokenEnv names a variable holding a bearer token (Unity Catalog OSS). oauth2ServerUri, scope, audience and resource are written as they are and may not hold a ${...} expression or start with env: (Debezium Server would expand it, Kafka Connect would not). fluid validate refuses it on a hand-written sink config, on a catalog not reached over REST, and next to an override that sets rest.auth.type, credential, token, oauth2-server-uri, scope, audience, resource, rest.sigv4-enabled or a header.Authorization.", + "additionalProperties": false, + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "oauth2", + "bearer" + ], + "description": "oauth2: OAuth2 client credentials against oauth2ServerUri. bearer: a static bearer token." + }, + "credentialEnv": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])", + "not": { + "pattern": "(?:^|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_|$)" + }, + "description": "type oauth2: the NAME of the environment variable holding client_id:client_secret (Iceberg splits it on the first ':'). Emitted as credential=env:. A value shaped like a token (a run of 20 or more letters and digits, with a digit, between underscores) is refused." + }, + "oauth2ServerUri": { + "type": "string", + "pattern": "^https?://[^\\s/?#@]+(?:[/?#]\\S*)?$", + "not": { + "pattern": "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" + }, + "description": "type oauth2: absolute URL starting with a lowercase http:// or https:// of the identity provider's token endpoint, with no user:password@ part (the client id and secret go in credentialEnv's variable), e.g. a Keycloak realm's /realms//protocol/openid-connect/token. Emitted as oauth2-server-uri; without it Iceberg would send the credential to the catalog's own v1/oauth/tokens." + }, + "scope": { + "type": "string", + "minLength": 1, + "not": { + "pattern": "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" + }, + "description": "type oauth2: the OAuth2 scope to request (Iceberg's default, catalog, is not one most identity providers know). Emitted as scope." + }, + "audience": { + "type": "string", + "minLength": 1, + "not": { + "pattern": "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" + }, + "description": "type oauth2, optional: sent as the audience parameter on the token request." + }, + "resource": { + "type": "string", + "minLength": 1, + "not": { + "pattern": "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" + }, + "description": "type oauth2, optional: sent as the resource parameter on the token request." + }, + "tokenEnv": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])", + "not": { + "pattern": "(?:^|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_|$)" + }, + "description": "type bearer: the NAME of the environment variable holding the bearer token. Emitted as token=env:. A value shaped like a token (a run of 20 or more letters and digits, with a digit, between underscores, such as a dapi... personal access token) is refused." + } + }, + "allOf": [ + { + "if": { + "required": [ + "type" + ], + "properties": { + "type": { + "const": "oauth2" + } + }, + "type": "object" + }, + "then": { + "required": [ + "credentialEnv", + "oauth2ServerUri", + "scope" + ], + "allOf": [ + { + "not": { + "required": [ + "tokenEnv" + ] + } + } + ] + } + }, + { + "if": { + "required": [ + "type" + ], + "properties": { + "type": { + "const": "bearer" + } + }, + "type": "object" + }, + "then": { + "required": [ + "tokenEnv" + ], + "allOf": [ + { + "not": { + "required": [ + "credentialEnv" + ] + } + }, + { + "not": { + "required": [ + "oauth2ServerUri" + ] + } + }, + { + "not": { + "required": [ + "scope" + ] + } + }, + { + "not": { + "required": [ + "audience" + ] + } + }, + { + "not": { + "required": [ + "resource" + ] + } + } + ] + } + } + ] + } + } + }, + "acquisitionDelivery": { + "type": "object", + "description": "Delivery semantics + idempotency + DLQ configuration.", + "additionalProperties": false, + "properties": { + "guarantee": { + "type": "string", + "enum": [ + "at_most_once", + "at_least_once", + "exactly_once" + ], + "default": "at_least_once" + }, + "idempotencyKey": { + "type": "string", + "description": "Template for the idempotency key. Default: {run_id}:{stream}:{record_pk}", + "default": "{run_id}:{stream}:{record_pk}" + }, + "dlq": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "default": true + }, + "sink": { + "type": "object", + "additionalProperties": false, + "properties": { + "format": { + "type": "string", + "enum": [ + "parquet", + "json", + "ndjson" + ] + }, + "location": { + "type": "string" + } + } + }, + "maxRecordsBeforeAbort": { + "type": "integer", + "minimum": 0, + "default": 10000 + }, + "alertOn": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "pii_classification_failed", + "schema_violation", + "destination_write_failed", + "quality_gate_failed" + ] + } + } + } + } + } + }, + "acquisitionSchemaEvolution": { + "type": "object", + "description": "Schema evolution behavior. Stricter-wins between policy and per-change overrides.", + "additionalProperties": false, + "properties": { + "policy": { + "type": "string", + "enum": [ + "strict", + "discover_and_freeze", + "evolve_safe", + "evolve_all" + ], + "default": "strict" + }, + "onAddedColumn": { + "type": "string", + "enum": [ + "include", + "warn", + "fail" + ] + }, + "onRemovedColumn": { + "type": "string", + "enum": [ + "drop", + "warn", + "fail" + ] + }, + "onTypeChange": { + "type": "string", + "enum": [ + "cast", + "warn", + "fail" + ] + }, + "sourceFingerprint": { + "type": "string", + "enum": [ + "required", + "optional", + "disabled" + ], + "default": "required" + } + } + }, + "acquisitionQuality": { + "type": "object", + "description": "Pre-land quality gates + anomaly signals.", + "additionalProperties": false, + "properties": { + "gates": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true, + "required": [ + "rule", + "severity" + ], + "properties": { + "rule": { + "type": "string", + "enum": [ + "not_null", + "unique", + "regex", + "range", + "row_count_anomaly", + "freshness" + ] + }, + "columns": { + "type": "array", + "items": { + "type": "string" + } + }, + "column": { + "type": "string" + }, + "pattern": { + "type": "string" + }, + "min": { + "type": "number" + }, + "max": { + "type": "number" + }, + "algorithm": { + "type": "string", + "enum": [ + "ewma", + "iqr", + "exact" + ] + }, + "sensitivity": { + "type": "number" + }, + "severity": { + "type": "string", + "enum": [ + "info", + "warn", + "error" + ] + } + } + } + }, + "onError": { + "type": "string", + "enum": [ + "route_to_dlq", + "abort_run", + "best_effort" + ], + "default": "route_to_dlq" + }, + "anomalies": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "signal", + "severity" + ], + "properties": { + "signal": { + "type": "string", + "enum": [ + "record_count_drop", + "record_count_surge", + "bytes_per_record_outlier", + "latency_outlier", + "dlq_rate_spike", + "cursor_stalled", + "schema_fingerprint_changed" + ] + }, + "algorithm": { + "type": "string", + "enum": [ + "ewma", + "iqr", + "exact" + ] + }, + "sensitivity": { + "type": "number" + }, + "severity": { + "type": "string", + "enum": [ + "info", + "warn", + "error" + ] + } + } + } + } + } + }, + "acquisitionCost": { + "type": "object", + "description": "Cost tracking + budget enforcement.", + "additionalProperties": false, + "properties": { + "budget": { + "type": "object", + "additionalProperties": false, + "properties": { + "monthly": { + "type": "object", + "additionalProperties": false, + "properties": { + "rows": { + "type": "integer", + "minimum": 0 + }, + "bytes": { + "type": "string", + "description": "Human-readable size with unit suffix (e.g., '50GB', '500MB')." + }, + "computeMinutes": { + "type": "integer", + "minimum": 0 + } + } + }, + "onExceed": { + "type": "string", + "enum": [ + "warn", + "abort" + ], + "default": "warn" + } + } + }, + "chargeback": { + "type": "object", + "additionalProperties": true, + "properties": { + "team": { + "type": "string" + }, + "project": { + "type": "string" + }, + "costCenter": { + "type": "string" + } + } + } + } + }, + "acquisitionCatalog": { + "type": "object", + "description": "Catalog auto-registration on first apply.", + "additionalProperties": false, + "properties": { + "register": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "datahub", + "openmetadata", + "datamesh_manager", + "unity", + "glue", + "snowflake_horizon" + ] + } + }, + "documentation": { + "type": "string", + "enum": [ + "auto", + "manual", + "none" + ], + "default": "auto" + } + } + }, + "acquisitionConcurrency": { + "type": "object", + "description": "Single-flight concurrency control.", + "additionalProperties": false, + "properties": { + "lock": { + "type": "object", + "additionalProperties": false, + "properties": { + "scope": { + "type": "string", + "enum": [ + "product", + "build" + ], + "default": "product" + }, + "timeout": { + "$ref": "#/$defs/isoDuration" + }, + "onContended": { + "type": "string", + "enum": [ + "abort", + "queue", + "replace" + ], + "default": "abort" + } + } + } + } + }, + "acquisitionImageSignature": { + "type": "object", + "description": "Connector image supply-chain verification.", + "additionalProperties": false, + "properties": { + "verifier": { + "type": "string", + "enum": [ + "cosign" + ], + "default": "cosign" + }, + "publicKey": { + "type": "string" + }, + "slsaProvenance": { + "type": "string", + "enum": [ + "required", + "optional", + "disabled" + ], + "default": "optional" + } + } + }, + "acquisitionDeployment": { + "type": "object", + "description": "Engine deployment mode and target.", + "additionalProperties": false, + "required": [ + "mode" + ], + "properties": { + "mode": { + "type": "string", + "enum": [ + "embedded", + "bring-your-own", + "managed" + ], + "default": "embedded" + }, + "server_url": { + "type": "string", + "description": "URL for bring-your-own mode." + }, + "auth": { + "type": "object", + "additionalProperties": true, + "properties": { + "secretRef": { + "type": "string" + } + } + }, + "managed": { + "type": "object", + "additionalProperties": false, + "required": [ + "target" + ], + "properties": { + "target": { + "type": "string", + "enum": [ + "docker", + "kubernetes", + "terraform", + "opentofu" + ] + }, + "profile": { + "type": "string", + "enum": [ + "small", + "medium", + "large" + ], + "default": "small" + }, + "chart": { + "type": "object", + "additionalProperties": true, + "properties": { + "repo": { + "type": "string" + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "values_overlay": { + "type": "object", + "additionalProperties": true + }, + "secrets": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "ref": { + "type": "string" + } + } + } + }, + "network": { + "type": "object", + "additionalProperties": false, + "properties": { + "egressAllowList": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "buildExecution": { + "type": "object", + "description": "Build execution configuration from v0.4.0 enhanced.", + "additionalProperties": false, + "properties": { + "trigger": { + "$ref": "#/$defs/trigger" + }, + "runtime": { + "$ref": "#/$defs/runtime" + }, + "retries": { + "$ref": "#/$defs/retryPolicy" + }, + "notifications": { + "type": "array", + "items": { + "$ref": "#/$defs/notification" + } + }, + "orchestration": { + "type": "object", + "required": [ + "engine" + ], + "properties": { + "engine": { + "type": "string", + "enum": [ + "airflow", + "dagster", + "prefect", + "kubeflow", + "custom", + "none" + ] + }, + "mode": { + "type": "string", + "enum": [ + "generated", + "manual", + "hybrid" + ], + "default": "generated" + }, + "generateOnChange": { + "type": "boolean", + "default": true + }, + "airflow": { + "$ref": "#/$defs/airflowOrchestration" + }, + "dagster": { + "type": "object" + }, + "prefect": { + "type": "object" + } + } + } + } + }, + "trigger": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "schedule", + "event", + "manual", + "dependency", + "dataset", + "schedule_and_dataset", + "timetable", + "continuous" + ], + "description": "NEW in v0.7.6: continuous, for a build that runs as a long-lived job on its own platform, such as an engine: flink job on the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, or a kafka-connect or debezium connector. The platform keeps the job running, so no scheduler starts runs: fluid generate schedule, fluid schedule-sync and the fluid apply DAGs leave the build out. A continuous trigger takes no other trigger, so cron, schedule, datasets, event and timetable are refused next to it, as Databricks Jobs refuses a continuous trigger next to any other." + }, + "schedule": { + "type": "string" + }, + "event": { + "type": "string" + }, + "condition": { + "type": "string" + }, + "datasets": { + "type": "array", + "description": "NEW in v0.7.0: Can reference data products directly via exposeRef for strong typing and auto-URI generation. Backward compatible with raw URIs.", + "items": { + "oneOf": [ + { + "type": "string", + "description": "Raw dataset URI (legacy, e.g., 'dataset://gcp/project/dataset/table')" + }, + { + "type": "object", + "description": "NEW in v0.7.0: Data product reference with strong typing", + "required": [ + "productId", + "exposeId" + ], + "properties": { + "productId": { + "$ref": "#/$defs/identifier", + "description": "Data product ID to depend on" + }, + "exposeId": { + "$ref": "#/$defs/identifier", + "description": "Expose ID within the data product" + }, + "versionConstraint": { + "$ref": "#/$defs/semverRange", + "description": "Optional version constraint (e.g., '^2.0.0')" + } + } + } + ] + } + }, + "datasetsOperator": { + "type": "string", + "enum": [ + "any", + "all" + ], + "default": "all" + }, + "cron": { + "type": "string", + "description": "Cron expression for schedule trigger" + }, + "timezone": { + "type": "string", + "default": "UTC" + }, + "timetable": { + "type": "object", + "description": "Custom timetable configuration" + } + }, + "allOf": [ + { + "if": { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "continuous" + } + } + }, + "then": { + "allOf": [ + { + "description": "cron is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove cron, or set type: schedule.", + "not": { + "required": [ + "cron" + ] + } + }, + { + "description": "schedule is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove schedule, or set type: schedule.", + "not": { + "required": [ + "schedule" + ] + } + }, + { + "description": "datasets is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove datasets, or set type: dataset.", + "not": { + "required": [ + "datasets" + ] + } + }, + { + "description": "event is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove event, or set type: event.", + "not": { + "required": [ + "event" + ] + } + }, + { + "description": "timetable is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove timetable, or set type: timetable.", + "not": { + "required": [ + "timetable" + ] + } + } + ] + } + } + ] + }, + "runtime": { + "type": "object", + "properties": { + "resources": { + "type": "object", + "properties": { + "cpu": { + "type": "string" + }, + "memory": { + "type": "string" + }, + "disk": { + "type": "string" + } + } + }, + "timeout": { + "type": "string" + }, + "environment": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "platform": { + "type": "string", + "enum": [ + "gcp", + "aws", + "azure", + "local", + "kubernetes", + "composer", + "mwaa", + "databricks", + "snowflake", + "athena", + "glue", + "redshift" + ] + }, + "executor": { + "type": "string", + "enum": [ + "LocalExecutor", + "CeleryExecutor", + "KubernetesExecutor", + "SequentialExecutor" + ] + }, + "image": { + "type": "string", + "description": "Container image for execution" + }, + "serviceAccount": { + "type": "string", + "description": "Service account for cloud execution" + } + } + }, + "retryPolicy": { + "type": "object", + "properties": { + "maxAttempts": { + "type": "integer", + "minimum": 1 + }, + "backoffStrategy": { + "type": "string", + "enum": [ + "fixed", + "exponential", + "linear" + ] + }, + "initialDelay": { + "type": "string" + }, + "maxDelay": { + "type": "string" + } + } + }, + "notification": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "email", + "slack", + "webhook", + "pagerduty" + ] + }, + "target": { + "type": "string" + }, + "condition": { + "type": "string", + "enum": [ + "success", + "failure", + "always" + ] + } + } + }, + "transformation": { + "type": "object", + "additionalProperties": false, + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "model": { + "type": "string", + "description": "Path/script identifier (dbt model, sql file, etc.)" + }, + "outputs": { + "type": "array", + "description": "List of exposeIds materialized by this step. VALIDATION: Tools must validate that every output exists in exposes[].exposeId.", + "items": { + "$ref": "#/$defs/identifier" + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Transformation step tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Transformation step labels." + } + } + }, + "lineage": { + "type": "object", + "additionalProperties": false, + "properties": { + "granularity": { + "type": "string", + "enum": [ + "table_level", + "field_level" + ], + "default": "table_level", + "description": "Level of lineage tracking granularity." + }, + "upstream": { + "type": "array", + "description": "Upstream dependencies and their field mappings.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "productId", + "exposeId" + ], + "properties": { + "productId": { + "$ref": "#/$defs/identifier", + "description": "Upstream data product ID." + }, + "exposeId": { + "$ref": "#/$defs/identifier", + "description": "Upstream expose ID being consumed." + }, + "fieldMappings": { + "type": "array", + "description": "Field-level lineage mappings (optional).", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "sourceField", + "targetField" + ], + "properties": { + "sourceField": { + "type": "string" + }, + "targetField": { + "type": "string" + }, + "transformation": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "direct_mapping", + "aggregation", + "calculation", + "lookup" + ] + }, + "expression": { + "type": "string", + "description": "Transformation logic or SQL expression." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Transformation tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Transformation labels." + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Field mapping tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Field mapping labels." + } + } + } + }, + "relationship": { + "type": "string", + "enum": [ + "direct_consumption", + "lookup_enrichment", + "aggregation" + ], + "description": "Type of data relationship." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Upstream relationship tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Upstream relationship labels." + } + } + } + }, + "downstream": { + "type": "array", + "description": "Known downstream consumers (optional).", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "consumer": { + "$ref": "#/$defs/identifier", + "description": "Downstream data product ID." + }, + "impact": { + "type": "string", + "enum": [ + "critical", + "high", + "medium", + "low" + ], + "description": "Impact level if this data product changes." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Downstream relationship tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Downstream relationship labels." + } + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Lineage configuration tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Lineage configuration labels." + } + } + }, + "schemaEvolution": { + "type": "object", + "additionalProperties": false, + "properties": { + "strategy": { + "type": "string", + "enum": [ + "semantic_versioning", + "date_based", + "sequential" + ], + "description": "Versioning strategy for schema changes." + }, + "compatibility": { + "type": "string", + "enum": [ + "backward_compatible", + "forward_compatible", + "full_compatible", + "breaking" + ], + "description": "Compatibility approach for schema evolution." + }, + "changePolicy": { + "type": "object", + "additionalProperties": false, + "properties": { + "changeWindowDays": { + "type": "integer", + "minimum": 0, + "description": "Advance notice period for breaking changes." + }, + "approvalRequired": { + "type": "boolean", + "description": "Whether schema changes require approval." + }, + "approvers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of approvers for schema changes." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Change policy tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Change policy labels." + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Schema evolution tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Schema evolution labels." + } + } + }, + "machineLearning": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "default": false + }, + "framework": { + "type": "string", + "enum": [ + "mlflow", + "kubeflow", + "sagemaker", + "vertex_ai", + "custom" + ] + }, + "models": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "version", + "type" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "$ref": "#/$defs/semver" + }, + "type": { + "type": "string", + "enum": [ + "classification", + "regression", + "clustering", + "recommendation", + "other" + ] + }, + "algorithm": { + "type": "string" + }, + "features": { + "type": "array", + "description": "List of feature names used by this model.", + "items": { + "type": "string" + } + }, + "target": { + "type": "string", + "description": "Target variable for supervised learning." + }, + "validation": { + "type": "object", + "additionalProperties": false, + "properties": { + "metrics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Validation metrics (e.g., accuracy, auc_roc)." + }, + "thresholds": { + "type": "object", + "additionalProperties": { + "type": "number" + }, + "description": "Minimum acceptable metric values." + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Model validation tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Model validation labels." + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "ML model tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "ML model labels." + } + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "ML configuration tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "ML configuration labels." + } + } + }, + "environmentConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "metadata": { + "type": "object", + "description": "Environment-specific metadata overrides.", + "additionalProperties": false, + "properties": { + "owner": { + "type": "object", + "additionalProperties": false, + "properties": { + "team": { + "type": "string" + }, + "email": { + "type": "string" + }, + "slack": { + "type": "string" + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Environment-specific metadata tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Environment-specific metadata labels." + } + } + }, + "exposes": { + "type": "array", + "description": "Environment-specific expose overrides.", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "exposeId": { + "$ref": "#/$defs/identifier" + }, + "binding": { + "$ref": "#/$defs/binding" + }, + "qos": { + "$ref": "#/$defs/exposeQoS" + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Environment-specific expose tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Environment-specific expose labels." + } + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Environment configuration tags." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Environment configuration labels." + } + } + }, + "orchestration": { + "type": "object", + "required": [ + "engine" + ], + "properties": { + "engine": { + "type": "string", + "enum": [ + "airflow", + "dagster", + "prefect", + "kubeflow", + "custom", + "none" + ] + }, + "mode": { + "type": "string", + "enum": [ + "generated", + "manual", + "hybrid" + ], + "default": "generated" + }, + "generateOnChange": { + "type": "boolean", + "default": true + }, + "airflow": { + "$ref": "#/$defs/airflowOrchestration" + }, + "dagster": { + "type": "object" + }, + "prefect": { + "type": "object" + } + } + }, + "airflowOrchestration": { + "type": "object", + "required": [ + "dagId" + ], + "properties": { + "dagId": { + "type": "string", + "pattern": "^[a-z0-9_][a-z0-9_.-]*[a-z0-9_]$|^[a-z0-9_]$" + }, + "dagConfig": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "schedule": { + "type": "string" + }, + "startDate": { + "type": "string" + }, + "endDate": { + "type": "string" + }, + "catchup": { + "type": "boolean", + "default": false + }, + "maxActiveRuns": { + "type": "integer", + "minimum": 1 + }, + "dagRunTimeout": { + "$ref": "#/$defs/isoDuration" + }, + "defaultArgs": { + "type": "object" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Airflow DAG tags (different from FLUID tags)" + }, + "onSuccessCallback": { + "type": "string" + }, + "onFailureCallback": { + "type": "string" + }, + "slaCallback": { + "type": "string" + } + } + }, + "taskDefaults": { + "type": "object", + "description": "Default configuration for all tasks (can be overridden per task)", + "properties": { + "retries": { + "type": "integer", + "minimum": 0 + }, + "retryDelay": { + "$ref": "#/$defs/isoDuration" + }, + "executionTimeout": { + "$ref": "#/$defs/isoDuration" + }, + "pool": { + "type": "string" + }, + "queue": { + "type": "string" + }, + "priority": { + "type": "integer" + }, + "triggerRule": { + "type": "string", + "enum": [ + "all_success", + "all_failed", + "all_done", + "one_success", + "one_failed", + "none_failed", + "none_skipped" + ] + }, + "errorHandling": { + "type": "object", + "description": "NEW in v0.7.0: Enhanced error categorization and handling", + "properties": { + "strategy": { + "type": "string", + "enum": [ + "fixed", + "exponential_backoff", + "linear_backoff" + ], + "default": "exponential_backoff" + }, + "retryableErrors": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Error types that should trigger retries", + "examples": [ + [ + "TemporaryFailure", + "RateLimitExceeded", + "ConnectionTimeout", + "DeadlineExceeded" + ] + ] + }, + "nonRetryableErrors": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Error types that should fail immediately", + "examples": [ + [ + "ValidationError", + "SchemaViolation", + "PermissionDenied", + "InvalidCredentials" + ] + ] + }, + "onError": { + "type": "array", + "items": { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ + "notify", + "create_incident", + "log", + "custom" + ] + }, + "channel": { + "type": "string" + }, + "severity": { + "type": "string", + "enum": [ + "low", + "medium", + "high", + "critical" + ] + }, + "when": { + "type": "string", + "enum": [ + "always", + "maxAttemptsExceeded", + "nonRetryableError" + ] + } + } + } + } + } + } + } + }, + "tasks": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/airflowTask" + } + }, + "sensors": { + "type": "array", + "items": { + "$ref": "#/$defs/airflowSensor" + } + }, + "taskGroups": { + "type": "array", + "items": { + "type": "object", + "required": [ + "groupId" + ], + "properties": { + "groupId": { + "type": "string" + }, + "description": { + "type": "string" + }, + "tasks": { + "type": "array", + "items": { + "$ref": "#/$defs/airflowTask" + } + } + } + } + } + } + }, + "airflowTask": { + "type": "object", + "required": [ + "taskId" + ], + "properties": { + "taskId": { + "type": "string", + "pattern": "^[a-z0-9_][a-z0-9_.-]*[a-z0-9_]$|^[a-z0-9_]$" + }, + "type": { + "type": "string", + "description": "NEW in v0.7.0: Simplified task type taxonomy. Replaces 'operator' field (backward compatible). Types: provider_action (NEW!), fluid_validate, fluid_plan, fluid_apply, fluid_execute, fluid_verify, fluid_dq_check, bash, python, sensor, or provider-specific operators.", + "enum": [ + "provider_action", + "fluid_validate", + "fluid_plan", + "fluid_apply", + "fluid_execute", + "fluid_verify", + "fluid_dq_check", + "bash", + "python", + "branch_python", + "sensor", + "email", + "http", + "snowflake_query", + "bigquery_query", + "bigquery_job", + "glue_job", + "databricks_job", + "custom" + ] + }, + "operator": { + "type": "string", + "description": "DEPRECATED in v0.7.0: Use 'type' instead. Maintained for backward compatibility with v0.6.1 contracts. Maps to 'type' automatically (FluidExecuteOperator β†’ fluid_execute, BashOperator β†’ bash)." + }, + "provider": { + "type": "string", + "description": "NEW in v0.7.0: Provider name when type=provider_action. Examples: aws, gcp, azure, snowflake, databricks.", + "enum": [ + "aws", + "gcp", + "azure", + "snowflake", + "databricks", + "kafka", + "kubernetes", + "local", + "custom" + ] + }, + "action": { + "type": "string", + "pattern": "^[a-z0-9_]+\\.[a-z0-9_]+$", + "description": "NEW in v0.7.0: Provider action when type=provider_action. Format: service.operation (e.g., s3.ensure_bucket, table.ensure, kinesis.create_stream).", + "examples": [ + "s3.ensure_bucket", + "table.ensure", + "database.ensure", + "kinesis.create_stream", + "glue.create_job", + "bigquery.create_table" + ] + }, + "buildStepRef": { + "type": "string", + "description": "NEW in v0.7.0: Reference to transformation step in builds[].transformations[]. Links orchestration tasks to build steps for clearer mapping." + }, + "dependencies": { + "type": "array", + "items": { + "type": "string" + } + }, + "params": { + "type": "object", + "additionalProperties": true + }, + "produces": { + "type": "array", + "description": "NEW in v0.7.0: Declares what exposes[] this task produces. Enables automatic dataset registration and lineage tracking.", + "items": { + "type": "object", + "required": [ + "exposeId" + ], + "properties": { + "exposeId": { + "$ref": "#/$defs/identifier", + "description": "The exposeId from exposes[] that this task produces" + }, + "dataset": { + "type": "string", + "description": "Optional: Airflow dataset URI. Auto-generated from binding.location if not specified." + }, + "updateStrategy": { + "type": "string", + "enum": [ + "replace", + "append", + "merge", + "upsert" + ], + "default": "replace", + "description": "How this task updates the data" + } + } + } + }, + "dataProductDependencies": { + "type": "array", + "description": "NEW in v0.7.0: Tasks can declare dependencies on other data products with version constraints and freshness requirements.", + "items": { + "type": "object", + "required": [ + "productId", + "exposeId" + ], + "properties": { + "productId": { + "$ref": "#/$defs/identifier", + "description": "Data product ID to depend on" + }, + "exposeId": { + "$ref": "#/$defs/identifier", + "description": "Specific expose within the product" + }, + "versionConstraint": { + "$ref": "#/$defs/semverRange", + "description": "Required version (e.g., '^2.0.0', '>=1.5.0')" + }, + "awaitFreshness": { + "$ref": "#/$defs/isoDuration", + "description": "Maximum age of data to wait for (e.g., PT6H = 6 hours)" + } + } + } + }, + "overrides": { + "type": "object", + "description": "Task-specific overrides for taskDefaults" + }, + "doc": { + "type": "string" + }, + "docMd": { + "type": "string" + }, + "uiColor": { + "type": "string" + }, + "uiFgcolor": { + "type": "string" + }, + "xcom": { + "type": "object", + "description": "XCom configuration for inter-task communication (from v0.6.1)", + "properties": { + "push": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "value": {}, + "serializer": { + "type": "string", + "enum": [ + "json", + "pickle", + "cloudpickle", + "custom" + ] + } + } + } + }, + "pull": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "dag_id": { + "type": "string" + }, + "map_indexes": { + "type": "array" + } + } + } + } + } + }, + "accessControl": { + "type": "object", + "description": "Task-level access control (from v0.6.1)", + "properties": { + "requiredRoles": { + "type": "array", + "items": { + "type": "string" + } + }, + "approvalRequired": { + "type": "boolean" + }, + "approvers": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "costEstimate": { + "type": "object", + "description": "Cost tracking (enhanced in v0.7.0)", + "properties": { + "compute": { + "type": "object", + "properties": { + "units": { + "type": "number" + }, + "pricePerUnit": { + "type": "number" + }, + "currency": { + "type": "string", + "default": "USD" + } + } + }, + "storage": { + "type": "object", + "properties": { + "units": { + "type": "number" + }, + "pricePerUnit": { + "type": "number" + } + } + }, + "total": { + "type": "number" + }, + "trackActual": { + "type": "boolean", + "default": false, + "description": "NEW in v0.7.0: Track actual costs vs estimate" + }, + "alertThreshold": { + "type": "number", + "description": "NEW in v0.7.0: Alert if actual > threshold * estimate (e.g., 1.5 = 150%)" + }, + "budget": { + "type": "object", + "description": "NEW in v0.7.0: Budget enforcement", + "properties": { + "monthly": { + "type": "number" + }, + "stopOnExceed": { + "type": "boolean", + "default": false + }, + "alertOnExceed": { + "type": "boolean", + "default": true + } + } + } + } + } + }, + "allOf": [ + { + "description": "Backward compatibility: If 'operator' is specified but not 'type', map operator to type", + "if": { + "required": [ + "operator" + ], + "not": { + "required": [ + "type" + ] + } + }, + "then": { + "description": "Operator field is deprecated but still works. Maps: FluidExecuteOperatorβ†’fluid_execute, BashOperatorβ†’bash, etc." + } + }, + { + "description": "NEW in v0.7.0: Validate provider_action tasks have required fields", + "if": { + "properties": { + "type": { + "const": "provider_action" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "provider", + "action", + "params" + ], + "errorMessage": "Tasks with type=provider_action must specify provider, action, and params" + } + }, + { + "description": "Validate fluid_execute tasks", + "if": { + "anyOf": [ + { + "properties": { + "type": { + "const": "fluid_execute" + } + } + }, + { + "properties": { + "operator": { + "pattern": "FluidExecute" + } + } + } + ] + }, + "then": { + "properties": { + "params": { + "type": "object", + "required": [ + "contract_path" + ], + "properties": { + "contract_path": { + "type": "string" + }, + "build_id": { + "type": "string" + } + } + } + } + } + }, + { + "description": "Validate bash tasks", + "if": { + "anyOf": [ + { + "properties": { + "type": { + "const": "bash" + } + } + }, + { + "properties": { + "operator": { + "pattern": "Bash" + } + } + } + ] + }, + "then": { + "properties": { + "params": { + "anyOf": [ + { + "required": [ + "bash_command" + ] + }, + { + "required": [ + "command" + ] + } + ] + } + } + } + }, + { + "description": "Validate python tasks", + "if": { + "anyOf": [ + { + "properties": { + "type": { + "const": "python" + } + } + }, + { + "properties": { + "operator": { + "pattern": "Python" + } + } + } + ] + }, + "then": { + "properties": { + "params": { + "anyOf": [ + { + "required": [ + "python_callable" + ] + }, + { + "required": [ + "callable" + ] + } + ] + } + } + } + } + ] + }, + "airflowSensor": { + "type": "object", + "required": [ + "taskId", + "sensorType" + ], + "properties": { + "taskId": { + "type": "string" + }, + "sensorType": { + "type": "string", + "enum": [ + "FileSensor", + "ExternalTaskSensor", + "TimeSensor", + "HttpSensor", + "S3KeySensor", + "GCSObjectSensor", + "custom" + ] + }, + "params": { + "type": "object" + }, + "pokInterval": { + "type": "integer", + "minimum": 1, + "default": 60 + }, + "timeout": { + "type": "integer", + "minimum": 1, + "default": 3600 + }, + "mode": { + "type": "string", + "enum": [ + "poke", + "reschedule" + ], + "default": "poke" + }, + "dependencies": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "semanticModel": { + "type": "object", + "description": "NEW in v0.7.2: Semantic model definition for an exposed interface. Maps physical columns to business concepts (entities, measures, dimensions, metrics) enabling AI agents and BI tools to generate correct queries without semantic hallucination. Compatible with dbt MetricFlow, Snowflake Semantic Views, and OSI-aligned formats.", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Human-readable name for this semantic model." + }, + "description": { + "type": "string", + "description": "Business context explaining what this semantic model represents." + }, + "defaultAggTimeDimension": { + "type": "string", + "description": "Default time dimension used for measure aggregation. Can be overridden per-measure." + }, + "entities": { + "type": "array", + "description": "Join keys with type annotations. Entities connect this table to others in the semantic graph.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type" + ], + "properties": { + "name": { + "type": "string", + "description": "Entity name (e.g., 'order', 'customer')." + }, + "type": { + "type": "string", + "enum": [ + "primary", + "foreign", + "unique", + "natural" + ], + "description": "Entity key type: primary (unique, complete), foreign (references another table), unique (unique, possibly partial), natural (real-world identifier)." + }, + "expr": { + "type": "string", + "description": "Column expression. Defaults to entity name if omitted." + }, + "description": { + "type": "string" + } + } + } + }, + "measures": { + "type": "array", + "description": "Aggregatable column expressions. Measures are the atomic building blocks from which metrics are composed.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "agg" + ], + "properties": { + "name": { + "type": "string", + "description": "Measure name (e.g., 'total_amount', 'order_count')." + }, + "description": { + "type": "string" + }, + "agg": { + "type": "string", + "enum": [ + "sum", + "avg", + "count", + "count_distinct", + "min", + "max", + "median", + "percentile" + ], + "description": "Aggregation function applied to the expression." + }, + "expr": { + "type": "string", + "description": "SQL expression to aggregate. Defaults to measure name if omitted." + }, + "aggParams": { + "type": "object", + "additionalProperties": false, + "description": "Aggregation parameters (mirrors dbt-semantic-interfaces agg_params). Currently used by agg=percentile.", + "properties": { + "percentile": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Percentile in [0, 1] for agg=percentile (0.5 = median). Consumers default to 0.5 when omitted." + }, + "useDiscretePercentile": { + "type": "boolean", + "description": "Use the discrete percentile (PERCENTILE_DISC / use_discrete_percentile) instead of continuous interpolation." + } + } + }, + "aggTimeDimension": { + "type": "string", + "description": "Override the model-level default time dimension for this measure." + }, + "nonAdditiveDimension": { + "type": "object", + "additionalProperties": false, + "description": "Dimension over which this measure cannot be naively aggregated (e.g., account balances).", + "properties": { + "name": { + "type": "string" + }, + "windowChoice": { + "type": "string", + "enum": [ + "min", + "max" + ], + "description": "Which value to pick when the non-additive dimension is present." + } + } + }, + "createMetric": { + "type": "boolean", + "default": false, + "description": "If true, automatically create a simple metric from this measure." + } + } + } + }, + "dimensions": { + "type": "array", + "description": "Categorical and time-based grouping axes. Dimensions define how data can be sliced in queries.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type" + ], + "properties": { + "name": { + "type": "string", + "description": "Dimension name (e.g., 'order_date', 'region', 'product_category')." + }, + "type": { + "type": "string", + "enum": [ + "categorical", + "time" + ], + "description": "Dimension type: categorical (text/enum grouping) or time (temporal axis)." + }, + "expr": { + "type": "string", + "description": "SQL expression. Defaults to dimension name if omitted." + }, + "description": { + "type": "string" + }, + "typeParams": { + "type": "object", + "additionalProperties": false, + "description": "Type-specific parameters.", + "properties": { + "timeGranularity": { + "type": "string", + "enum": [ + "day", + "week", + "month", + "quarter", + "year", + "hour", + "minute" + ], + "description": "Default time granularity for this dimension." + } + } + } + } + } + }, + "metrics": { + "type": "array", + "description": "Composable KPI definitions built from measures. Metrics are the named business calculations that agents and tools resolve.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type" + ], + "properties": { + "name": { + "type": "string", + "description": "Metric name (e.g., 'net_revenue', 'customer_churn_rate')." + }, + "description": { + "type": "string", + "description": "Business definition of this metric." + }, + "type": { + "type": "string", + "enum": [ + "simple", + "derived", + "ratio" + ], + "description": "Metric type: simple (single measure + filter), derived (expression over other metrics), ratio (numerator/denominator)." + }, + "measure": { + "type": "string", + "description": "Reference to a measure name (for simple metrics)." + }, + "filter": { + "type": "string", + "description": "SQL WHERE clause filter (e.g., \"status = 'completed' AND is_refunded = FALSE\")." + }, + "inputMetrics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "References to other metric names (for derived/ratio metrics)." + }, + "expr": { + "type": "string", + "description": "Mathematical expression combining input metrics (for derived metrics)." + }, + "numerator": { + "type": "string", + "description": "Numerator metric name (for ratio metrics)." + }, + "denominator": { + "type": "string", + "description": "Denominator metric name (for ratio metrics)." + }, + "owner": { + "type": "string", + "description": "Business owner of this metric definition." + } + } + } + }, + "tags": { + "$ref": "#/$defs/tags", + "description": "Semantic model tags for discovery and categorization." + }, + "labels": { + "$ref": "#/$defs/labels", + "description": "Semantic model labels for automation and metadata." + } + }, + "examples": [ + { + "name": "orders_revenue_model", + "description": "Canonical semantic model for revenue analytics", + "defaultAggTimeDimension": "order_date", + "entities": [ + { + "name": "order", + "type": "primary", + "expr": "order_id" + }, + { + "name": "customer", + "type": "foreign", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "order_amount", + "agg": "sum", + "expr": "amount", + "description": "Sum of all order amounts" + }, + { + "name": "order_count", + "agg": "count_distinct", + "expr": "order_id", + "createMetric": true + } + ], + "dimensions": [ + { + "name": "order_date", + "type": "time", + "expr": "completed_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "net_revenue", + "description": "GAAP-compliant net revenue: completed orders minus discounts, excluding refunds", + "type": "derived", + "inputMetrics": [ + "gross_revenue", + "total_discounts" + ], + "expr": "gross_revenue - total_discounts", + "owner": "finance-data@company.com" + }, + { + "name": "gross_revenue", + "description": "Total value of completed orders", + "type": "simple", + "measure": "order_amount", + "filter": "status = 'completed'" + } + ] + } + ] + }, + "exposeMcp": { + "type": "object", + "additionalProperties": false, + "description": "NEW in v0.7.4: Per-expose overrides for the MCP output-port server β€” sampling caps + classification. The presence of this (non-empty) block opts the expose into the Fluid MCP gateway, where policy.agentPolicy is enforced at every read.", + "properties": { + "sampling": { + "type": "object", + "additionalProperties": false, + "properties": { + "maxRows": { + "type": "integer", + "minimum": 1, + "description": "Hard cap for the `sample` MCP tool. Defends against an over-curious agent asking for petabyte-scale row counts. Default: 100." + } + } + }, + "classification": { + "type": "object", + "additionalProperties": false, + "properties": { + "dataClass": { + "type": "string", + "enum": [ + "public", + "internal", + "confidential", + "restricted" + ], + "description": "Surfaced on the `describe` tool so consumer agents can declare downstream handling. Advisory; the real gate is `policy.agentPolicy`." + } + } + } + } + }, + "governance": { + "type": "object", + "description": "Account/project-wide governance settings. Today only the lakeFormation block is supported; future cloud-specific blocks may live alongside it (e.g. snowflake masking-policy registries, gcp organization-policies).", + "additionalProperties": false, + "properties": { + "lakeFormation": { + "type": "object", + "additionalProperties": false, + "description": "AWS Lake Formation account-wide settings: data-lake admins and LF-tag (TBAC) definitions. Applies to the AWS account the contract's IaC apply targets.", + "properties": { + "admins": { + "type": "array", + "description": "IAM principal ARNs (roles or users) granted Lake Formation admin privileges. Emitted as aws_lakeformation_data_lake_settings.admins. AUTHORITATIVE, not additive: Lake Formation's PutDataLakeSettings replaces the account's admin list with the one sent, so an admin NOT listed here is REMOVED on apply. List every principal that must stay a data lake admin, including the role or user that runs the apply: left out, it loses Lake Formation admin on that apply, and the Lake Formation grants and registrations that need an admin can then fail. The terraform-provider-aws resource also clears the data lake settings its configuration omits, and this block sets only the admins, so the account's create-database and create-table default permissions (the IAMAllowedPrincipals default for new databases and tables), trusted resource owners and parameters are cleared too. Destroying the resource empties the admin list and resets CROSS_ACCOUNT_VERSION to 1. Re-applying the same list is a no-op.", + "items": { + "type": "string", + "pattern": "^arn:aws[a-z0-9-]*:iam::" + }, + "uniqueItems": true + }, + "tagDefinitions": { + "type": "object", + "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and a_b. Field idiom borrowed from OpenMetadata's classification-label model β€” see ODCS roles + OpenMetadata data contracts spec.", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true, + "minItems": 1 + } + } + } + } + } + }, + "bindingGovernance": { + "type": "object", + "description": "Per-resource (per-exposure) governance. Currently scoped to AWS Lake Formation; future cloud-specific blocks may sit alongside (e.g. gcpIamPolicy, snowflakeMaskingPolicy).", + "additionalProperties": false, + "properties": { + "lakeFormation": { + "type": "object", + "additionalProperties": false, + "description": "Per-resource LF settings: location registration, principal grants, LF-tag associations, row/column filters. Applies only to formats backed by S3 + Glue catalog (iceberg/parquet/csv/json/avro/orc/delta).", + "properties": { + "registerLocation": { + "type": "boolean", + "default": false, + "description": "When true, emit aws_lakeformation_resource that registers the binding's S3 path with Lake Formation. Required before LF can manage access to objects under that path." + }, + "bucketPolicy": { + "type": "string", + "enum": [ + "cross-account", + "none", + "all-grantees" + ], + "default": "cross-account", + "description": "NEW in v0.7.6: Which grants[] principals get a statement in the aws_s3_bucket_policy emitted beside the grants (s3:ListBucket and s3:GetBucketLocation on the bucket, s3:GetObject under location.path). A statement lets its principal read the objects straight from S3, which skips Lake Formation's column, row and cell filters and its revocations. 'cross-account' (the default): only principals whose ARN names an AWS account other than the one running the apply, decided at plan time against the caller identity; same-account grantees get no statement, since on a registered location Lake Formation vends them credentials. 'none': no bucket policy at all; the tightest setting when the location is registered (registerLocation) and every reader, cross-account ones included, queries through a Lake Formation integrated engine, or when the bucket policy is managed elsewhere. 'all-grantees': every grantee, same-account included, which is the output before this field existed and reopens the direct S3 read path. An emitted aws_s3_bucket_policy is authoritative: it replaces every other statement on the bucket. When no grantee is left none is emitted." + }, + "grants": { + "type": "array", + "description": "Principal-based LF grants on this exposure's database/table. Each entry maps a principal to a permission set (and optional column projection). Emitted as aws_lakeformation_permissions resources. Field idiom adapted from ODCS v3 roles[] (principal + access-type), bound to IAM ARNs since LF is cloud-specific.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "principal", + "permissions" + ], + "properties": { + "principal": { + "type": "string", + "description": "IAM principal ARN (role or user) receiving the grant.", + "pattern": "^arn:aws[a-z0-9-]*:iam::" + }, + "permissions": { + "type": "array", + "description": "LF permissions granted. Valid values per the hashicorp/aws aws_lakeformation_permissions resource.", + "items": { + "type": "string", + "enum": [ + "ALL", + "SELECT", + "ALTER", + "DROP", + "DELETE", + "INSERT", + "DESCRIBE", + "CREATE_TABLE", + "CREATE_DATABASE", + "DATA_LOCATION_ACCESS" + ] + }, + "uniqueItems": true, + "minItems": 1 + }, + "permissionsWithGrantOption": { + "type": "array", + "description": "Subset of permissions the principal can re-grant. Maps to aws_lakeformation_permissions.permissions_with_grant_option.", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "columns": { + "type": "array", + "description": "Optional column-level restriction. When set, the grant is emitted against aws_lakeformation_permissions.table_with_columns instead of .table, allowing the principal to read only the listed columns.", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "excludedColumns": { + "type": "array", + "description": "Opposite of columns: every column EXCEPT these is granted. Maps to table_with_columns.excluded_column_names. Mutually exclusive with columns.", + "items": { + "type": "string" + }, + "uniqueItems": true + } + } + } + }, + "tags": { + "type": "object", + "description": "LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as aws_lakeformation_resource_lf_tags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).", + "additionalProperties": { + "type": "string" + } + }, + "rowFilter": { + "type": "object", + "additionalProperties": false, + "description": "Row-level (and optionally column-level) filter applied to all reads of this table via LF. Emitted as aws_lakeformation_data_cells_filter. The filter must then be granted to principals via grants[].permissions including SELECT β€” LF enforces both layers.", + "properties": { + "name": { + "type": "string", + "description": "Filter name (must be unique per table)." + }, + "rowExpression": { + "type": "string", + "description": "PartiQL-flavoured row predicate. ``ALL`` lets every row through (use when only column filtering is intended)." + }, + "columnNames": { + "type": "array", + "description": "Columns visible through the filter. Mutually exclusive with allColumns / excludedColumnNames.", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "excludedColumnNames": { + "type": "array", + "description": "Columns hidden through the filter (everything else is visible).", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "allColumns": { + "type": "boolean", + "default": false, + "description": "Convenience for the LF column-wildcard form (every column visible). When true, columnNames / excludedColumnNames must be omitted." + } + }, + "required": [ + "name", + "rowExpression" + ] + }, + "maskedViews": { + "type": "object", + "additionalProperties": false, + "required": [ + "definer", + "validationConnection" + ], + "description": "Where the protected Glue Data Catalog views of the expose's masked columns come from (NEW in v0.7.6). Lake Formation cannot mask, so a principal an access: mask column restriction names reads a protected, SECURITY DEFINER multi-dialect view, _masked, that returns the columns masked by the restriction's rule and applies its row filter; its grant on the table excludes the columns. The definer is granted SELECT with grant option on the table and DESCRIBE on the database; the principal SELECT and DESCRIBE on the view.", + "properties": { + "definer": { + "type": "string", + "description": "ARN of the IAM role the views are defined as: Glue and Lake Formation must be allowed to assume it (sts:AssumeRole, sts:SetContext), and the applying principal to pass it (iam:PassRole)." + }, + "validationConnection": { + "type": "string", + "description": "The Glue connection of type VIEW_VALIDATION_ATHENA (WORKGROUP_NAME) Glue validates each view's SQL through; the applying principal and the definer need glue:PassConnection on it." + } + } + } + } + } + } + }, + "consumerRef": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type" + ], + "$comment": "dbt-exposures-shaped (name/label/type/owner/url/maturity/description); exposeIds is the FLUID-native refinement tying a consumer to specific output ports.", + "properties": { + "name": { + "$ref": "#/$defs/identifier", + "description": "Stable identifier for this consumer (unique within the contract)." + }, + "label": { + "type": "string", + "description": "Human-facing display name, e.g. \"Weekly Revenue Dashboard\"." + }, + "type": { + "type": "string", + "enum": [ + "dashboard", + "notebook", + "analysis", + "ml", + "application" + ], + "description": "What kind of artifact consumes this product." + }, + "owner": { + "type": "object", + "additionalProperties": false, + "$comment": "Mirrors metadata.owner's vocabulary so 'owner' means one thing across the standard: team is the routable identity, email the contact. A dbt exposure's owner.name maps to team on import.", + "properties": { + "team": { + "type": "string", + "description": "Owning team for this consumer (routable identity, as in metadata.owner)." + }, + "email": { + "type": "string", + "format": "email", + "description": "Contact address for the consumer's owner." + } + } + }, + "url": { + "type": "string", + "description": "Link to the live artifact (dashboard URL, notebook, app)." + }, + "maturity": { + "type": "string", + "enum": [ + "high", + "medium", + "low" + ], + "description": "How production-hardened the consumer is (dbt exposures semantics)." + }, + "description": { + "type": "string" + }, + "exposeIds": { + "type": "array", + "items": { + "$ref": "#/$defs/identifier" + }, + "description": "Which output ports this consumer reads. Absent = all exposes." + } + } + } + } +} \ No newline at end of file diff --git a/scripts/schema-versions.json b/scripts/schema-versions.json index 45e5153..54c50b7 100644 --- a/scripts/schema-versions.json +++ b/scripts/schema-versions.json @@ -11,12 +11,12 @@ "repository": "Agenticstiger/forge-cli", "schemaDir": "fluid_build/schemas", "previewSource": "fluid_build/schema_manager.py", - "ref": "v0.21.0", - "commit": "91e071e2c2273e2ba06fd0deabfe42d78de82f2e" + "ref": "v0.27.0", + "commit": "c0e97c076f735205f4767f67f0bd05bcd78a47cd" }, - "latestStable": "0.7.5", - "preview": ["0.7.6"], - "synced": ["0.7.2", "0.7.3", "0.7.4", "0.7.5", "0.7.6"], + "latestStable": "0.7.6", + "preview": ["0.7.7"], + "synced": ["0.7.2", "0.7.3", "0.7.4", "0.7.5", "0.7.6", "0.7.7"], "notSynced": { "0.7.1": "The reference implementation bundles a different document under the same $id (it adds orchestration, icebergConfig, binding.properties and more). Re-vendoring it would change which documents the published 0.7.1 accepts, so it is a decision for the maintainers rather than a sync, and the published copy is left as it is.", "0.0.1-0.5.7": "Predate the compatibility promise; the reference implementation no longer bundles them." diff --git a/specs/0.7.6/fluid-spec.html b/specs/0.7.6/fluid-spec.html index 5a43501..43d2378 100644 --- a/specs/0.7.6/fluid-spec.html +++ b/specs/0.7.6/fluid-spec.html @@ -13,16 +13,18 @@ - FLUID 0.7.6 β€” Declarative Packaging Modes (preview) + FLUID 0.7.6 β€” Declarative Packaging Modes -

FLUID 0.7.6 β€” Declarative Packaging Modes (preview)

Type: object
+

FLUID 0.7.6 β€” Declarative Packaging Modes

Type: object
-

FLUID contract schema 0.7.6 β€” the current PREVIEW version, opened when 0.7.5 was promoted to stable.

+

FLUID contract schema 0.7.6 β€” opened as the preview when 0.7.5 was promoted to stable, and stable itself since 0.7.7 opened.

πŸ”₯ NEW in v0.7.6:
β€’ Declarative packaging modes β€” a top-level packaging block (plus a per-exposure binding.packaging override) declares per-container ownership for the IaC emit: isolated (this product creates and owns the container) vs shared (a pre-existing, platform-owned pool referenced via an OpenTofu data source), with a per-kind containers override map for hybrid tiers and a pool id for tenancy/cost attribution (RFC-packaging-modes.md).
+β€’ Optional Bitol-shaped fields the exporters already wanted, each one opt-in: metadata.team (a roster β€” name plus members, every member a $defs.teamMember whose only required field is username), the root description as {purpose, usage, limitations} as well as a plain string, consumes[].contractId (an upstream that has a contract id of its own), consumes[].contractVersion (the exact upstream version this product was built against, inside that range), exposes[].contract.id (used verbatim as the ODCS id and the ODPS output port contractId instead of the derived <product id>.<exposeId>; two exposes may not share one) and binding.location.host plus binding.serialization.{value, key} for Kafka. A contract that sets none of them validates and exports exactly as it does today.
+β€’ exposes[].contract.schema[].quality β€” column-level data quality rules in ODCS's own shape. The ODCS exporter always replayed this key verbatim and read it back on import, but $defs.column is closed and did not define it, so the one way to publish a hand-written column rule was to write a contract fluid validate refuses. Optional: a column without it exports exactly as it does today.
β€’ Backward compatible with v0.7.5 β€” every 0.7.5 contract validates as 0.7.6 unchanged; an ABSENT packaging block keeps the legacy byte-identical emit.

@@ -211,7 +213,7 @@

/> fluidVersionType: enum (of string)
-

Contract schema version. Accepts '0.7.3' (source-aligned data products + acquisition pattern + ingestion engines) or '0.7.4' (adds the MCP output-port gateway + runtime agentPolicy enforcement). 0.7.4 is fully backward-compatible with 0.7.3 β€” every 0.7.3 contract validates as 0.7.4 unchanged. '0.7.6' is the current preview.

+

Contract schema version. Accepts '0.7.3' (source-aligned data products + acquisition pattern + ingestion engines) or '0.7.4' (adds the MCP output-port gateway + runtime agentPolicy enforcement). 0.7.4 is fully backward-compatible with 0.7.3 β€” every 0.7.3 contract validates as 0.7.4 unchanged.

Must be one of:

  • "0.7.3"
  • "0.7.4"
  • "0.7.5"
  • "0.7.6"
@@ -359,8 +361,264 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - description

Type: string
+ description
+
+

NEW in v0.7.6: Either a plain string β€” unchanged meaning, it becomes purpose on export β€” or the structured object. While this was a string, an export of a valid contract could only ever carry description.purpose: usage and limitations had no home a contract could declare, so every ODCS and ODPS export left them empty. fluid odps import restores the object to the root description when the schema version the imported contract declares accepts an object, and also keeps a verbatim copy in extensions.odps.metadata.description so the ODPS leg stays an exact fixed point. All three fields therefore survive a FLUID β†’ ODPS β†’ FLUID cycle and reach both the ODCS and the ODPS export afterwards. Importing against a version whose root description is a string still roots purpose alone, with the object recoverable from that bucket.

+
+
+ +

+ +

+
+ + + Type: string
+

Prose description. Exported as description.purpose.

+
+ + + + + + +
+ + + Type: object
+

NEW in v0.7.6: The structured form of the root description. The three field names are Bitol's β€” ODPS v1.1.0 $defs.Description and the ODCS v3.1.0 root description both use exactly purpose/usage/limitations β€” so import and export are a pass-through rather than a translation.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

What this data product is for.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

How a consumer is expected to use it.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

What it must not be used for, and what it does not cover.

+
+ + + + + + +
+
+
+
+
@@ -1417,18 +1675,18 @@

-
+
-
+

- +

-
+
Type: stringFormat: date-time
- + team
Type: object
+

NEW in v0.7.6: The team behind this product, as a roster rather than a single contact. metadata.owner holds one team name and one mailbox and carries no role, so a product with a data owner, a steward and two engineers had nowhere to say so and the exported ODCS/ODPS team block came out empty. The field names are Bitol's, so an imported team round-trips without a translation. Optional: a contract without it exports exactly as it does today.

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Business alignment information for data mesh organization.

-
No Additional Properties + team + + + + name
Type: string
+

Team name, e.g. 'Sales Analytics'.

+
-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

Business domain this product belongs to.

+ members
Type: array
+

The people on the team, each with the role they hold on this product.

- -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

Business subdomain for more granular organization.

+ members + + + + teamMember
Type: object
+

NEW in v0.7.6: One member of metadata.team.members. The field names and the username-only required list are Bitol's ODCS v3.1.0 / ODPS v1.1.0 $defs.TeamMember, copied verbatim so an imported team exports back unchanged.

+ + No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

Business capability this product supports.

+ members + + + + members items + + + + username
Type: string
+

The member's identifier in the organisation's directory β€” the one field Bitol requires.

@@ -1638,18 +1909,18 @@

-
+
-
+

- +

-
+
Type: string
-

Value stream this product participates in.

+ members + + + + members items + + + + name
Type: string
+

Display name.

@@ -1686,22 +1971,18 @@

-
-
-
-
-
+
-
+

- +

-
+
Type: array of string
-

Product-level tags on metadata for discovery and categorization.

-
Same definition as tags -
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: array
- - - - - - -

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

-
-
- - - Type: object
- - - No Additional Properties + role
Type: string
+

The role this member holds on this product, e.g. 'owner', 'steward', 'data engineer'. Free-form: Bitol does not constrain it.

+
-
-
Example:
-
{
-    "exposeId": "customer_profiles",
-    "title": "Customer Profiles Table",
-    "kind": "table",
-    "tags": [
-        "pii",
-        "customer-facing",
-        "analytical"
-    ],
-    "labels": {
-        "sensitivity": "high",
-        "retention": "7-years",
-        "team": "customer-analytics"
-    },
-    "contract": {
-        "schema": [
-            {
-                "name": "customer_id",
-                "type": "STRING",
-                "required": true,
-                "sensitivity": "cleartext",
-                "tags": [
-                    "identifier",
-                    "primary-key"
-                ],
-                "labels": {
-                    "business-name": "Customer ID",
-                    "data-category": "identifier"
-                }
-            }
-        ]
-    },
-    "binding": {
-        "platform": "gcp",
-        "format": "bigquery_table",
-        "location": {
-            "project": "company-data",
-            "dataset": "customer",
-            "table": "profiles"
-        }
-    }
-}
-
+ +
+
+
-
+
-
+

- +

-
+
Type: string
-

Stable interface handle (used by consumers & build.outputs).

-
Same definition as id -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: string
+ email
Type: stringFormat: email
@@ -1920,18 +2094,28 @@

-
+
+
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: string
-

Detailed description of the exposed resource.

-
+ createdAt
Type: stringFormat: date-time
+ @@ -1968,18 +2144,18 @@

-
+
-
+

- +

-
+
Type: string
-

Semantic version (semver.org).

-
- - Must match regular expression: ^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$ + businessContext
Type: object
+

Business alignment information for data mesh organization.

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "table"
  • "view"
  • "api"
  • "file"
  • "stream"
  • "topic"
  • "feature_store"
  • "model"
  • "vector"
  • "graph"
  • "time_series"
  • "other"
-
+ domain
Type: string
+

Business domain this product belongs to.

+
@@ -2068,18 +2229,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: array of string
-

Expose-level tags for categorization and routing.

-
Same definition as tags + subdomain
Type: string
+

Business subdomain for more granular organization.

+
+ + + + + +
-
+
-
+

- +

-
+
Type: object
-

Expose-level labels for metadata and automation.

-
Same definition as labels + businessCapability
Type: string
+

Business capability this product supports.

+
+ + + + + +
-
+
-
+

- +

-
+
Type: object
+ valueStream
Type: string
+

Value stream this product participates in.

+
+ + + - No Additional Properties

- -

-
- + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: array of string
+

Product-level tags on metadata for discovery and categorization.

+
Same definition as tags +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ exposes
Type: array
-
-

The following properties are required:

-
  • schema
-
+ - -
+

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

+
+
Type: object
- -
-

The following properties are required:

-
  • openapiRef
-
+ expose
Type: object
- - - -
+ No Additional Properties - -
+
+
Example:
+
{
+    "exposeId": "customer_profiles",
+    "title": "Customer Profiles Table",
+    "kind": "table",
+    "tags": [
+        "pii",
+        "customer-facing",
+        "analytical"
+    ],
+    "labels": {
+        "sensitivity": "high",
+        "retention": "7-years",
+        "team": "customer-analytics"
+    },
+    "contract": {
+        "schema": [
+            {
+                "name": "customer_id",
+                "type": "STRING",
+                "required": true,
+                "sensitivity": "cleartext",
+                "tags": [
+                    "identifier",
+                    "primary-key"
+                ],
+                "labels": {
+                    "business-name": "Customer ID",
+                    "data-category": "identifier"
+                }
+            }
+        ]
+    },
+    "binding": {
+        "platform": "gcp",
+        "format": "bigquery_table",
+        "location": {
+            "project": "company-data",
+            "dataset": "customer",
+            "table": "profiles"
+        }
+    }
+}
+
+
+
-
+

- +

-
+
Type: string
-

Hash of canonical schema/OpenAPI. Used by contract-tests gates.

-
Must match regular expression: ^sha256:[0-9a-fA-F]{64}$ - - - - - - + exposeId
Type: string
+

Stable interface handle (used by consumers & build.outputs).

+
Same definition as id
-
+
-
+

- +

-
+
Type: enum (of string) Default: "strict"
-

NEW in v0.7.3: Schema evolution policy. strict = fail on any schema change; discoverandfreeze = capture from source on first run, fail thereafter; evolvesafe = accept additive changes; evolveall = accept all changes (cast on type narrow, route failures to DLQ).

-
-

Must be one of:

-
  • "strict"
  • "discover_and_freeze"
  • "evolve_safe"
  • "evolve_all"
-
+ title
Type: string
+ @@ -2423,18 +2607,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: array
-

Tabular schema (for table/view/file).

+ description
Type: string
+

Detailed description of the exposed resource.

- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
- + version
Type: string
+

Semantic version (semver.org).

+
- No Additional Properties + Must match regular expression: ^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$ -
+
+
+
+
+
-
+

- +

-
+
Type: string
- + kind
Type: enum (of string)
+
+

Must be one of:

+
  • "table"
  • "view"
  • "api"
  • "file"
  • "stream"
  • "topic"
  • "feature_store"
  • "model"
  • "vector"
  • "graph"
  • "time_series"
  • "other"
+
-

Must be at least 1 characters long

+
-
+
-
+

- +

-
+
Type: array of string
+

Expose-level tags for categorization and routing.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
-

Column data type. Accepts a canonical FLUID/SQL type name (case-insensitive) or a parameterized form such as decimal(18,4) or varchar(255). The enum lists the canonical bare types; the pattern additionally permits case variants and parameter suffixes. Intentionally generous β€” it rejects obvious garbage (e.g. MYSTERY_TYPE) while admitting the cross-dialect superset.

-

- -

-
- + exposes items + + + + labels
Type: object
+

Expose-level labels for metadata and automation.

+
Same definition as labels +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
+ + + No Additional Properties

+ +

+
+ + + Type: enum (of string)
-
-

Must be one of:

-
  • "string"
  • "text"
  • "varchar"
  • "varchar2"
  • "nvarchar"
  • "char"
  • "nchar"
  • "character"
  • "clob"
  • "int"
  • "integer"
  • "int2"
  • "int4"
  • "int8"
  • "int16"
  • "int32"
  • "int64"
  • "tinyint"
  • "smallint"
  • "mediumint"
  • "bigint"
  • "long"
  • "longint"
  • "serial"
  • "bigserial"
  • "float"
  • "float4"
  • "float8"
  • "float32"
  • "float64"
  • "double"
  • "real"
  • "decimal"
  • "dec"
  • "numeric"
  • "number"
  • "bignumeric"
  • "money"
  • "boolean"
  • "bool"
  • "bit"
  • "date"
  • "time"
  • "datetime"
  • "datetime2"
  • "smalldatetime"
  • "timestamp"
  • "timestamptz"
  • "timestamp_tz"
  • "timestamp_ntz"
  • "timestamp_ltz"
  • "timestampntz"
  • "interval"
  • "year"
  • "variant"
  • "object"
  • "array"
  • "struct"
  • "map"
  • "record"
  • "row"
  • "json"
  • "jsonb"
  • "super"
  • "binary"
  • "varbinary"
  • "bytes"
  • "blob"
  • "bytea"
  • "raw"
  • "geography"
  • "geometry"
  • "geom"
  • "point"
  • "uuid"
  • "uniqueidentifier"
  • "guid"
  • "enum"
  • "hll"
-
- + item 0
Type: object
+ +
+

The following properties are required:

+
  • schema
+
+ id="tab-pane_exposes_items_contract_anyOf_i1" role="tabpanel"> Type: string
-Must match regular expression: (?i)^\s*(string|text|varchar|varchar2|nvarchar|char|nchar|character|clob|int|integer|int2|int4|int8|int16|int32|int64|tinyint|smallint|mediumint|bigint|long|longint|serial|bigserial|float|float4|float8|float32|float64|double|real|double\s+precision|decimal|dec|numeric|number|bignumeric|money|boolean|bool|bit|date|time|datetime|datetime2|smalldatetime|timestamp|timestamptz|timestamp_tz|timestamp_ntz|timestamp_ltz|timestampntz|timestamp\s+with(out)?\s+time\s+zone|interval|year|variant|object|array|struct|map|record|row|json|jsonb|super|binary|varbinary|bytes|blob|bytea|raw|geography|geometry|geom|point|uuid|uniqueidentifier|guid|enum|hll)\s*(\(\s*[0-9A-Za-z_,\s'\"-]*\s*\))?\s*$ - + item 1
Type: object
+ +
+

The following properties are required:

+
  • openapiRef
+
@@ -2808,22 +2997,18 @@

Must be one of:

-
-
-
-
-
+
-
+

- +

-
+
Type: boolean
- + id
Type: string
+

NEW in v0.7.6: An explicit, stable contract id, used verbatim as the ODCS id and as the ODPS output port contractId. Without it the id is derived as <product id>.<exposeId>, so renaming the product or the expose silently changes the contract id and breaks every consumer reference to it, and a catalog that requires an id format of its own (a UUID, say) cannot be served at all. Each expose needs its own: two exposes sharing one id is refused by fluid validate.

+
- +

Must be at least 1 characters long

-
+
-
+

- +

-
+
Type: string
- + schemaSignature
Type: string
+

Hash of canonical schema/OpenAPI. Used by contract-tests gates.

+
Must match regular expression: ^sha256:[0-9a-fA-F]{64}$ @@ -2948,18 +3107,18 @@

-
+
-
+

- +

-
+
Type: enum (of string)
-

Field-level sensitivity classification for improved policy ergonomics.

-
- - Type: enum (of string) Default: "strict"
+

NEW in v0.7.3: Schema evolution policy. strict = fail on any schema change; discoverandfreeze = capture from source on first run, fail thereafter; evolvesafe = accept additive changes; evolveall = accept all changes (cast on type narrow, route failures to DLQ).

+

Must be one of:

-
  • "none"
  • "internal"
  • "confidential"
  • "restricted"
  • "pii"
  • "phi"
  • "cleartext"
  • "treated"
  • "anonymized"
  • "pseudonymized"
  • "tokenized"
  • "encrypted"
+
  • "strict"
  • "discover_and_freeze"
  • "evolve_safe"
  • "evolve_all"
@@ -3022,18 +3165,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: string
-

Semantic meaning (e.g., 'email', 'phone', 'identifier').

+ schema
Type: array
+

Tabular schema (for table/view/file).

- -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

Business-friendly name for this field.

-
+ column
Type: object
+ + + No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

Business definition and meaning of this field.

-
- + name
Type: string
+ +

Must be at least 1 characters long

-
+
-
+

- +

-
+
Type: array of object
-

Field-level validation constraints.

-
- - - - - - No Additional Items

Each item of this array must be:

-
-
+ type

+

Column data type. Accepts a canonical FLUID/SQL type name (case-insensitive) or a parameterized form such as decimal(18,4) or varchar(255). The enum lists the canonical bare types; the pattern additionally permits case variants and parameter suffixes. Intentionally generous β€” it rejects obvious garbage (e.g. MYSTERY_TYPE) while admitting the cross-dialect superset.

+

+ +

+
Type: object
- No Additional Properties + anyOf + + + + item 0
Type: enum (of string)
+
+

Must be one of:

+
  • "string"
  • "text"
  • "varchar"
  • "varchar2"
  • "nvarchar"
  • "char"
  • "nchar"
  • "character"
  • "clob"
  • "int"
  • "integer"
  • "int2"
  • "int4"
  • "int8"
  • "int16"
  • "int32"
  • "int64"
  • "tinyint"
  • "smallint"
  • "mediumint"
  • "bigint"
  • "long"
  • "longint"
  • "serial"
  • "bigserial"
  • "float"
  • "float4"
  • "float8"
  • "float32"
  • "float64"
  • "double"
  • "real"
  • "decimal"
  • "dec"
  • "numeric"
  • "number"
  • "bignumeric"
  • "money"
  • "boolean"
  • "bool"
  • "bit"
  • "date"
  • "time"
  • "datetime"
  • "datetime2"
  • "smalldatetime"
  • "timestamp"
  • "timestamptz"
  • "timestamp_tz"
  • "timestamp_ntz"
  • "timestamp_ltz"
  • "timestampntz"
  • "interval"
  • "year"
  • "variant"
  • "object"
  • "array"
  • "struct"
  • "map"
  • "record"
  • "row"
  • "json"
  • "jsonb"
  • "super"
  • "binary"
  • "varbinary"
  • "bytes"
  • "blob"
  • "bytea"
  • "raw"
  • "geography"
  • "geometry"
  • "geom"
  • "point"
  • "uuid"
  • "uniqueidentifier"
  • "guid"
  • "enum"
  • "hll"
+
-
-
-
-

- -

-
- -
-
+
+ Type: enum (of string)
-
-

Must be one of:

-
  • "range"
  • "regex"
  • "enum"
  • "custom"
-
+ item 1
Type: string
+Must match regular expression: (?i)^\s*(string|text|varchar|varchar2|nvarchar|char|nchar|character|clob|int|integer|int2|int4|int8|int16|int32|int64|tinyint|smallint|mediumint|bigint|long|longint|serial|bigserial|float|float4|float8|float32|float64|double|real|double\s+precision|decimal|dec|numeric|number|bignumeric|money|boolean|bool|bit|date|time|datetime|datetime2|smalldatetime|timestamp|timestamptz|timestamp_tz|timestamp_ntz|timestamp_ltz|timestampntz|timestamp\s+with(out)?\s+time\s+zone|interval|year|variant|object|array|struct|map|record|row|json|jsonb|super|binary|varbinary|bytes|blob|bytea|raw|geography|geometry|geom|point|uuid|uniqueidentifier|guid|enum|hll)\s*(\(\s*[0-9A-Za-z_,\s'\"-]*\s*\))?\s*$ + + + + + + +
@@ -3440,18 +3554,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: string
+ required
Type: boolean
@@ -3522,18 +3622,18 @@

-
+
-
+

- +

-
+
Type: string
+ description
Type: string
@@ -3604,24 +3690,18 @@

-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: array of string
-

Field-level tags for categorization and data governance.

-
Same definition as tags -
+ sensitivity
Type: enum (of string)
+

Field-level sensitivity classification for improved policy ergonomics.

+
+ +
+

Must be one of:

+
  • "none"
  • "internal"
  • "confidential"
  • "restricted"
  • "pii"
  • "phi"
  • "cleartext"
  • "treated"
  • "anonymized"
  • "pseudonymized"
  • "tokenized"
  • "encrypted"
+
+ + + + + + +
-
+
-
+

- +

-
+
Type: object
-

Field-level labels for metadata and automation.

-
Same definition as labels -
-
-
-
-
-
+ semanticType
Type: string
+

Semantic meaning (e.g., 'email', 'phone', 'identifier').

+
+ + + + + +
-
+
-
+

- +

-
+
Type: stringFormat: uri
-

OpenAPI document URL (for APIs).

+ schema + + + + schema items + + + + businessName
Type: string
+

Business-friendly name for this field.

@@ -3797,18 +3902,18 @@

-
+
-
+

- +

-
+
Type: object
- No Additional Properties + schema + + + + schema items + + + + businessDefinition
Type: string
+

Business definition and meaning of this field.

+
-
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-

Compatibility promise for non-breaking evolution.

-
-

Must be one of:

-
  • "none"
  • "patch"
  • "minor"
  • "backward"
  • "forward"
  • "strict"
-
+ schema items + + + + validationRules
Type: array of object
+

Field-level validation constraints.

+
- -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: array of enum (of string)
- + schema items + + + + validationRules + + + + validationRules items
Type: object
+ No Additional Properties - No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: enum (of string)
-Type: enum (of string)
+

Must be one of:

-
  • "additive_columns"
  • "additive_fields"
  • "additive_endpoints"
  • "nullable_to_required_disallowed"
  • "field_removal_with_notice"
  • "type_widening_allowed"
+
  • "range"
  • "regex"
  • "enum"
  • "custom"
@@ -4026,28 +4178,22 @@

Must be one of:

-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Enhanced data quality rules including anomaly detection.

-
- - No Additional Properties - - - - - - -
-
-
-

- -

-
- -
- Type: string
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
- + validationRules + + + + validationRules items + + + + message
Type: string
- No Additional Properties -
+
+
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: string
+ quality
Type: array of object
+
+

NEW in v0.7.6: Column-level data quality rules, in ODCS's own shape β€” the array that is written straight into schema[].properties[].quality[] of the exported ODCS v3.1.0 contract, and the array an imported ODCS column's rules are read back into, inline on the column rather than in the importer's pass-through bucket: the ODPS importer merges a resolved port's contract block into its output-port stub and discards the sub-import's root extensions, so rules left in that bucket were dropped outright by a FLUID β†’ ODPS β†’ FLUID cycle. The ODCS exporter has always replayed this key verbatim and always read it back on import, but $defs.column is additionalProperties: false and did not define it, so a column written the way the exporter supports was REFUSED by fluid validate: the only way to get a hand-written quality rule into a published contract was to write an invalid contract. Optional and additive β€” a column without it exports exactly as it does today, with the rules forge derives from required, the primary-key tag and validationRules. Setting it REPLACES those derived rules for that column rather than adding to them, which is what makes an imported ODCS column round-trip byte-for-byte. The authority for an item's contents is ODCS v3.1.0 $defs.DataQuality, deliberately not re-declared here (a copy would drift from the vendored schema, and fluid odcs validate already checks an emitted document against it); the keys below are the common ones. Every rule should carry an id: it is optional in ODCS, but a DQ engine that labels its results by rule id rejects a contract whose rules have none.

+
+
+ +
- -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: enum (of string)
-
-

Must be one of:

-
  • "freshness"
  • "completeness"
  • "uniqueness"
  • "valid_values"
  • "accuracy"
  • "schema"
  • "anomaly_detection"
  • "drift_detection"
-
+ quality items
Type: object
+ -
-
-
-
-
+
-
+

- +

-
+
Type: string
- + quality items + + + + id
Type: string
+

Stable rule identifier, unique within this column's rules. The pattern is ODCS $defs.StableId. Rules forge derives itself are named <column>-<metric>.

+
Must match regular expression: ^[A-Za-z0-9_-]+$ @@ -4430,18 +4568,18 @@

-
+
-
+

- +

-
+
Type: number
- + quality items + + + + type
Type: enum (of string)
+

ODCS quality check type. text = prose; library = one of ODCS's predefined metrics; sql = a query whose result is compared; custom = a named engine's own check.

+
+

Must be one of:

+
  • "text"
  • "library"
  • "sql"
  • "custom"
+
@@ -4505,18 +4654,18 @@

-
+
-
+

- +

-
+
Type: enum (of string)
-Type: enum (of string)
+

type: library only. The five predefined metrics ODCS v3.1.0 defines; a name outside this list is not a library rule.

+

Must be one of:

-
  • ">="
  • ">"
  • "<="
  • "<"
  • "=="
  • "!="
+
  • "nullValues"
  • "missingValues"
  • "invalidValues"
  • "duplicateValues"
  • "rowCount"
@@ -4583,18 +4740,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: string
-

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+ quality items + + + + arguments
Type: object
+

type: library only. Extra arguments for the metric β€” pattern (a regex) or validValues (a list) for invalidValues, as DataHub's ODCS ingestion source reads them.

- - Must match regular expression: ^P(?!$)(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T(\d+H)?(\d+M)?(\d+S)?)?$ @@ -4661,18 +4823,18 @@

-
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "info"
  • "warn"
  • "error"
  • "critical"
-
+ quality items + + + + query
Type: string
+

type: sql only. The query whose result the comparison operator is applied to.

+
@@ -4739,18 +4906,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: string
- + quality items + + + + engine
Type: string
+

type: custom only. The engine that runs the check, e.g. soda, great-expectations, dbt.

+
@@ -4814,18 +4989,18 @@

-
+
-
+

- +

-
+
Type: array of string
-

DQ rule tags for categorization and automation.

-
Same definition as tags + quality items + + + + implementation
Type: object
+

type: custom only. The check as that engine expresses it.

+
+ + + + + +
-
+
-
+

- +

-
+
Type: object
-

DQ rule labels for metadata and routing.

-
Same definition as labels + quality items + + + + dimension
Type: enum (of string)
+

The data quality dimension the rule measures.

+
+

Must be one of:

+
  • "accuracy"
  • "completeness"
  • "conformity"
  • "consistency"
  • "coverage"
  • "timeliness"
  • "uniqueness"
-
-
-
-
-
+ + + + + +
-
+
-
+

- +

-
+
Type: object
- No Additional Properties + schema items + + + + quality + + + + quality items + + + + description
Type: string
+ -
+
+
+
+
+
-
+

- +

-
+
Type: boolean Default: true
- + quality + + + + quality items + + + + severity
Type: string
+

Severity of a failure. ODCS types this as a free string and documents info / warning / error by example.

+
@@ -5085,18 +5323,18 @@

-
+
-
+

- +

-
+
Type: string
-

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

-
Same definition as window + quality + + + + quality items + + + + unit
Type: string
+

Unit the comparison is in, usually rows (the default) or percent.

+
+ + + + + +
-
+
+
+
+
+
+
+
-
+

- +

-
+
Type: string
- - - - - - - -
-
-
-
+ tags
Type: array of string
+

Field-level tags for categorization and data governance.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: array of string
-

Data quality specification tags.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

Data quality specification labels.

+ labels
Type: object
+

Field-level labels for metadata and automation.

Same definition as labels
+
+
-
+
-
+

- +

-
+
Type: array of object
-

Simplified SQL-expression data quality rules (alternate form of dq.rules for inline assertions).

+ openapiRef
Type: stringFormat: uri
+

OpenAPI document URL (for APIs).

- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
- + guarantees
Type: object
+ No Additional Properties -
+
-
+

- +

-
+
Type: string
-

Rule identifier.

-
+ compatibility
Type: enum (of string)
+

Compatibility promise for non-breaking evolution.

+
+

Must be one of:

+
  • "none"
  • "patch"
  • "minor"
  • "backward"
  • "forward"
  • "strict"
+
@@ -5503,18 +5714,18 @@

-
+
-
+

- +

-
+
Type: string
-

SQL or boolean expression that must evaluate to true.

-
+ evolution
Type: array of enum (of string)
+ - -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: enum (of string)
-

Severity when the rule fails.

-
Type: enum (of string)
+

Must be one of:

-
  • "error"
  • "warning"
  • "info"
+
  • "additive_columns"
  • "additive_fields"
  • "additive_endpoints"
  • "nullable_to_required_disallowed"
  • "field_removal_with_notice"
  • "type_widening_allowed"
@@ -5640,28 +5828,28 @@

Must be one of:

-
-
+
+
-
+
-
+

- +

-
+
Type: array of string
-

Contract-level tags for data quality and governance.

-
Same definition as tags -
-
-
-
-
+ dq
Type: object
+

Enhanced data quality rules including anomaly detection.

+
+ + No Additional Properties + + + + + + +
-
+

- +

-
+
Type: object
-

Contract-level labels for automation and metadata.

-
Same definition as labels -
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: array
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
- - - No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: string
-

Availability percentage (e.g., 99.9%).

-
+ dqRule
Type: object
- Must match regular expression: ^(100(\.0+)?|\d{2}(\.\d+)?|\d{1}\d(\.\d+)?)%$ + + No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

-
Same definition as window -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: string
-

e.g., '0 rows'

-
+ id
Type: string
+ @@ -5958,18 +6079,18 @@

-
+
-
+

- +

-
+
Type: string
-

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

-
Same definition as window -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: number
-

Target completeness ratio (0.0 to 1.0).

-
+ type
Type: enum (of string)
+
+

Must be one of:

+
  • "freshness"
  • "completeness"
  • "uniqueness"
  • "valid_values"
  • "accuracy"
  • "schema"
  • "anomaly_detection"
  • "drift_detection"
+
-

Value must be greater or equal to 0 and lesser or equal to 1

+
-
+
-
+

- +

-
+
Type: number
-

Acceptable error rate (0.0 to 1.0).

-
+ dq + + + + rules + + + + rules items + + + + selector
Type: string
+ -

Value must be greater or equal to 0 and lesser or equal to 1

+
-
+
-
+

- +

-
+
Type: array of string
-

QoS-level tags for SLA management.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

QoS-level labels for automation and reporting.

-
Same definition as labels -
-
-
-
+ threshold
Type: number
+ + + + + + +
-
+
-
+

- +

-
+
Type: object
- - - No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: enum (of string)
-Type: enum (of string)
+

Must be one of:

-
  • "oidc"
  • "oauth2"
  • "api_key"
  • "none"
  • "custom"
  • "iam"
  • "jwt"
+
  • ">="
  • ">"
  • "<="
  • "<"
  • "=="
  • "!="
@@ -6321,18 +6385,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: object
- No Additional Properties + dq + + + + rules + + + + rules items + + + + window
Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
+ + Must match regular expression: ^P(?!$)(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T(\d+H)?(\d+M)?(\d+S)?)?$ -
+
+
+
+
+
-
+

- +

-
+
Type: array of string
- + rules + + + + rules items + + + + severity
Type: enum (of string)
+
+

Must be one of:

+
  • "info"
  • "warn"
  • "error"
  • "critical"
+
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
+ rules items + + + + description
Type: string
@@ -6482,24 +6612,22 @@

-

-
-
+
-
+

- +

-
+
Type: array of string
- - - - - - - No Additional Items

Each item of this array must be:

+ rules + + + + rules items + + + + tags
Type: array of string
+

DQ rule tags for categorization and automation.

+
Same definition as tags +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
- - - - - - - + rules items + + + + labels
Type: object
+

DQ rule labels for metadata and routing.

+
Same definition as labels +
+
+
+
-
+
-
+

- +

-
+
Type: array of object
-
-

Column-level access control. A column named in any restriction is restricted. 'deny': the principal may not read the columns. 'allow': the columns are readable only by the principals an allow names. A deny beats an allow, and a restriction never grants access: the readers are the expose's readers. Principals are logical and resolve through binding.principals. NEW in v0.7.6 (the field itself is older): fluid apply enforces it. GCP BigQuery: a Data Catalog taxonomy per product and dataset with fine-grained access control, a policy tag per set of restricted columns that share their readers, attached through the table schema's policyTags, and roles/datacatalog.categoryFineGrainedReader for exactly the expose's readers allowed to read them: the accessPolicy read grantees and the expose's own policy.authz.readers (a denied principal gets an access error on the columns). A restriction on an expose with no reader on the binding is refused, as it would lock the columns for everyone. A restriction's tags and labels are written into the policy tag's description; AWS has no place for them. AWS: each governance.lakeFormation grant with SELECT excludes the restricted columns its principal may not read (tablewithcolumns.excludedcolumnnames); a grant's hand-written excludedColumns must agree, and a binding with no Lake Formation grants is refused. fluid verify checks the policy tags and their readers on GCP, and the principals' Lake Formation permissions on AWS. NEW in v0.7.6: access: mask lets the principal read the columns masked by the platform (see mask): a deny beats a mask, and a mask beats an allow for the principal it names.

- -
-
- -
+ monitoring
Type: object
+ No Additional Properties - No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+

- No Additional Properties

- -

-
- + enabled
Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

- - -
- -
- - - Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
- - - - - - - -
+ tags
Type: array of string
+

Data quality specification tags.

+
Same definition as tags +
+
+
+
+
-
+

- +

-
+
Type: const
-Specific value: "mask" - - - - - - + labels
Type: object
+

Data quality specification labels.

+
Same definition as labels
+
+
+
+
+
+
+

+ +

+
- - -
- +
+
Type: array of object
+

Simplified SQL-expression data quality rules (alternate form of dq.rules for inline assertions).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
- -
-

The following properties are required:

-
  • mask
-
+ quality items
Type: object
- -
- +
+
+
+

+ +

+
- -
- +
+
Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

-
Type: string
+

Rule identifier.

+
+ - - + -
- -
+
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
- -
-

The following properties are required:

-
  • mask
-
+ expression
Type: string
+

SQL or boolean expression that must evaluate to true.

+
+ +
+
+
+
+
+
+

+ +

+
- - -
- +
+
Type: object
- + severity
Type: enum (of string)
+

Severity when the rule fails.

+
+

Must be one of:

+
  • "error"
  • "warning"
  • "info"
+
-
+
+
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: array of string
+

Contract-level tags for data quality and governance.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: const
-Specific value: "mask" - - - - - - + labels
Type: object
+

Contract-level labels for automation and metadata.

+
Same definition as labels
-
- - - - - - -
- - - - - - -
+
+
+
+
+
-
+

- +

-
+
Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
-

Logical principal the restriction applies to, written as in accessPolicy (e.g. group:analysts@company.example); binding.principals maps it per cloud.

+ availability
Type: string
+

Availability percentage (e.g., 99.9%).

+ + Must match regular expression: ^(100(\.0+)?|\d{2}(\.\d+)?|\d{1}\d(\.\d+)?)%$ @@ -7505,18 +7656,18 @@

-
+
-
+

- +

-
+
Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
-

Columns of this expose's schema.

-
- - - - + qos + + + + dataLossSLO
Type: string
+

e.g., '0 rows'

+
+ - No Additional Items

Each item of this array must be:

+ + + + +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+ Type: number
+

Target completeness ratio (0.0 to 1.0).

+
- +

Value must be greater or equal to 0 and lesser or equal to 1

-
-
-
+
-
+

- +

-
+
Type: enum (of string)
-

deny: the principal may not read the columns. allow: only the principals an allow names may read them. mask: the principal reads the columns masked by the platform, by the rule in mask, and never their raw values (NEW in v0.7.6).

-
-

Must be one of:

-
  • "allow"
  • "deny"
  • "mask"
-
+ errorBudget
Type: number
+

Acceptable error rate (0.0 to 1.0).

+
- +

Value must be greater or equal to 0 and lesser or equal to 1

-
+
-
+

- +

-
+
Type: array of string
-

Access control rule tags.

+ tags
Type: array of string
+

QoS-level tags for SLA management.

Same definition as tags
-
+
-
+

- +

-
+
Type: object
-

Access control rule labels.

+ labels
Type: object
+

QoS-level labels for automation and reporting.

Same definition as labels
-
+
+
+
+
+
-
+

- +

-
+
Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
-
-

The masking rule an access: mask restriction applies, required with it and refused without it. lastfour: 'XXXXX' and the value's last four characters; firstfour: its first four characters and 'XXXXX' (a value of four characters or fewer is hashed instead, by each platform); nullify: NULL. GCP BigQuery: a data policy (DATAMASKINGPOLICY, LASTFOURCHARACTERS, FIRSTFOURCHARACTERS or ALWAYS_NULL) on the columns' policy tag, and roles/bigquerydatapolicy.maskedReader for the masked principals, who read the same column masked. AWS Glue: Lake Formation cannot mask, so the masked principals' grant on the table excludes the columns and they read a protected Glue Data Catalog view (SECURITY DEFINER, binding.governance.lakeFormation.maskedViews) that returns them masked by the same rule (NEW in v0.7.6).

- -
-
- -
Type: enum (of string)
+

Must be one of:

-
  • "last_four"
  • "first_four"
  • "nullify"
+
  • "oidc"
  • "oauth2"
  • "api_key"
  • "none"
  • "custom"
  • "iam"
  • "jwt"
@@ -7956,24 +8123,18 @@

Must be one of:

-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: array of string
-

Authorization policy tags.

-
Same definition as tags -
-
-
-
-Type: object
+ No Additional Properties + + + + + + +
-
+

- +

-
+
Type: object
-

Authorization policy labels.

-
Same definition as labels -
-
-
-
-
-
-
-

- -

-
+ readers
Type: array of string
-
-
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ Type: array of object
-
-

Row-level access control (NEW in v0.7.6). Each filter lets its principal read only the rows its where predicate selects; a reader no filter names reads every row, and a principal may have one filter per expose. The predicate is one SQL boolean expression over this expose's columns, with no statement separator or comment. Principals are logical and resolve through binding.principals. GCP BigQuery: a row access policy (filterpredicate, grantees) per filter, plus one that selects every row for every other reader and writer, since a principal no row access policy names reads no row at all. AWS Glue: a Lake Formation data cells filter (rowfilter and the principal's excluded columns), and the principal's SELECT is granted on the filter instead of the table.

+ readers + + + + readers items
Type: string
-
-
- -
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
- No Additional Properties + writers
Type: array of string
+ - -
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

Logical principal the filter applies to, written as in accessPolicy; binding.principals maps it per cloud.

-
+ writers items
Type: string
+ +
+
-
+
-
+

- +

-
+
Type: string
-

The filter's name on the platform (the row access policy or data cells filter); derived from the principal when omitted.

-
Must match regular expression: ^[a-z][a-z0-9_]{0,62}$ + columnRestrictions
Type: array of object
+
+

Column-level access control. A column named in any restriction is restricted. 'deny': the principal may not read the columns. 'allow': the columns are readable only by the principals an allow names. A deny beats an allow, and a restriction never grants access: the readers are the expose's readers. Principals are logical and resolve through binding.principals. NEW in v0.7.6 (the field itself is older): fluid apply enforces it. GCP BigQuery: a Data Catalog taxonomy per product and dataset with fine-grained access control, a policy tag per set of restricted columns that share their readers, attached through the table schema's policyTags, and roles/datacatalog.categoryFineGrainedReader for exactly the expose's readers allowed to read them: the accessPolicy read grantees and the expose's own policy.authz.readers (a denied principal gets an access error on the columns). A restriction on an expose with no reader on the binding is refused, as it would lock the columns for everyone. A restriction's tags and labels are written into the policy tag's description; AWS has no place for them. AWS: each governance.lakeFormation grant with SELECT excludes the restricted columns its principal may not read (tablewithcolumns.excludedcolumnnames); a grant's hand-written excludedColumns must agree, and a binding with no Lake Formation grants is refused. fluid verify checks the policy tags and their readers on GCP, and the principals' Lake Formation permissions on AWS. NEW in v0.7.6: access: mask lets the principal read the columns masked by the platform (see mask): a deny beats a mask, and a mask beats an allow for the principal it names.

+ +
+
+ +
- -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

One SQL boolean expression over the expose's columns, e.g. consentdataanalytics = true AND is_suppressed = false.

-
- - - -

Must be at least 1 characters long

- - -
-
-
-
-
-
-
-

- -

-
- -
-
+ No Additional Properties

+ +

+
+ Type: string
- - + allOf + + + + item 0
Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ +
+ +
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
Type: object
- No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: array of object
-
-

Columns to treat before they land. The DuckDB acquisition runner rewrites each named column inside the write, so the file, S3 object or BigQuery load and the DLQ receive the treated value; quality gates judge the source value first. A treated column lands as a string, so declare it with a string type (string, or VARCHAR under a schemaPolicy that compares source type names). A rule the build cannot apply (an unset salt or key, k_anonymity, a column the stream lacks, a non-string declared type, the incremental cursor) refuses the build rather than landing the column untreated. fluid verify checks every non-null value of a masked column for its strategy's shape, on a local file and, through Athena, on an S3+Glue table. The MCP output-port gateway drops masked columns from what it serves.

+ if
Type: object
-
-
- -
- No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: object
- No Additional Properties + columnRestrictions items + + + + allOf + + + + item 0 + + + + if + + + + access
Type: const
+Specific value: "mask" -
-
-
-

- -

+
+
+
+
-
-
+ + +
+ Type: string
+ allOf + + + + item 0 + + + + then
Type: object
+ +
+

The following properties are required:

+
  • mask
+
+ +
-
-
-
-
-
-
-
-

- -

-
+ -
-
+ +
+ Type: enum (of string)
-
-

hash: lowercase hex SHA-256 of salt || value, deterministic so joins work (64 hex characters). mask: every character but the first params.keepFirst and the last params.keepLast replaced by '*', length preserved. tokenize: HMAC-SHA256 token, 32 hex characters, the token the tokenizepii preLand hook makes. encrypt: AES-GCM, written as 'aesgcm:v1:' + base64url(nonce || ciphertext || tag) with the column name as associated data; reversible with the key, not deterministic. kanonymity: accepted by the schema but refused at landing: it is a property of a whole table over its quasi-identifiers, not of one value.

- -
-
- -
-

Must be one of:

-
  • "mask"
  • "hash"
  • "tokenize"
  • "encrypt"
  • "k_anonymity"
-
- + allOf + + + + item 1
Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ +
+ +
-
-
-
-
-
-
-
-

- -

-
- -
-
Type: object
-

Strategy parameters. The runner accepts exactly these keys and refuses any other, including a literal salt or key: secrets come only from the environment, never the contract, which is published with the plan, the bundle and the Glue table's fluid_contract parameter. Contracts on 0.7.5 and earlier may use the same keys; their schemas leave params untyped, and the runner checks them at build time.

-
- + allOf + + + + item 1 + + + + if
Type: object
+ +
+

The following properties are required:

+
  • mask
+
-
-
-
-

- -

-
+
-
-
+ + +
+ Type: string Default: "FLUID_PII_HASH_SECRET"
-

NEW in v0.7.6: hash only. The environment variable holding the salt, at least 16 bytes. Unset or empty, the build is refused: an unsalted SHA-256 of a phone number is reversed by enumerating the numbering plan.

-
Must match regular expression: ^[A-Z_][A-Z0-9_]{0,127}$ + item 1 + + + + then
Type: object
+ -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.6: tokenize and encrypt. The environment variable holding the key. tokenize defaults to FLUIDPIITOKENIZATIONKEY and needs at least 32 bytes; encrypt defaults to FLUIDPIIENCRYPTIONSECRET_KEY and needs the base64 of a 16, 24 or 32-byte AES key (openssl rand -base64 32). Unset or empty, the build is refused.

-
Must match regular expression: ^[A-Z_][A-Z0-9_]{0,127}$ + item 1 + + + + then + + + + access
Type: const
+Specific value: "mask" @@ -9074,18 +9217,32 @@

-
+
+ + + + + + +
+ + + + + + +
-
+

- +

-
+
Type: integer Default: 0
-

NEW in v0.7.6: mask only. Leading characters left as they are.

+ principal
Type: string
+

Logical principal the restriction applies to, written as in accessPolicy (e.g. group:analysts@company.example); binding.principals maps it per cloud.

-

Value must be greater or equal to 0 and lesser or equal to 64

+
-
+
-
+

- +

-
+
Type: integer Default: 4
-

NEW in v0.7.6: mask only. Trailing characters left as they are, so +46701234567 lands as ********4567. A value no longer than keepFirst + keepLast is masked entirely.

+ columns
Type: array of string
+

Columns of this expose's schema.

-

Value must be greater or equal to 0 and lesser or equal to 64

- -
-
-
-
-
-
-
-

- -

-
-
-

Additional Properties of any type are allowed.

+ No Additional Items

Each item of this array must be:

+
+
+ Type: object
+ columns items
Type: string
@@ -9318,26 +9447,24 @@

-

-
-
+
-
+

- +

-
+
Type: array of string
-

Privacy masking rule tags.

-
Same definition as tags + access
Type: enum (of string)
+

deny: the principal may not read the columns. allow: only the principals an allow names may read them. mask: the principal reads the columns masked by the platform, by the rule in mask, and never their raw values (NEW in v0.7.6).

+
+

Must be one of:

+
  • "allow"
  • "deny"
  • "mask"
+
+ + + + + +
-
+
-
+

- +

-
+
Type: object
-

Privacy masking rule labels.

-
Same definition as labels -
-
-
-
-
-
+ tags
Type: array of string
+

Access control rule tags.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: object
- No Additional Properties - - - - - - -Type: object
+

Access control rule labels.

+
Same definition as labels +
+
+
+
+
-
+

- +

-
+
Type: string
-

Provider-specific predicate

-
+ columnRestrictions items + + + + mask
Type: enum (of string)
+
+

The masking rule an access: mask restriction applies, required with it and refused without it. lastfour: 'XXXXX' and the value's last four characters; firstfour: its first four characters and 'XXXXX' (a value of four characters or fewer is hashed instead, by each platform); nullify: NULL. GCP BigQuery: a data policy (DATAMASKINGPOLICY, LASTFOURCHARACTERS, FIRSTFOURCHARACTERS or ALWAYS_NULL) on the columns' policy tag, and roles/bigquerydatapolicy.maskedReader for the masked principals, who read the same column masked. AWS Glue: Lake Formation cannot mask, so the masked principals' grant on the table excludes the columns and they read a protected Glue Data Catalog view (SECURITY DEFINER, binding.governance.lakeFormation.maskedViews) that returns them masked by the same rule (NEW in v0.7.6).

+ +
+
+ +
+

Must be one of:

+
  • "last_four"
  • "first_four"
  • "nullify"
+
@@ -9598,18 +9758,24 @@

-
+
+
+
+
+
+
+
-
+

- +

-
+
Type: array of string
-

Row-level policy tags.

+ tags
Type: array of string
+

Authorization policy tags.

Same definition as tags
-
+
-
+

- +

-
+
Type: object
-

Row-level policy labels.

+ labels
Type: object
+

Authorization policy labels.

Same definition as labels
-
-
-
-
-
+
-
+

- +

-
+
Type: array of string
-

Privacy policy tags.

-
Same definition as tags -
+ rowFilters
Type: array of object
+
+

Row-level access control (NEW in v0.7.6). Each filter lets its principal read only the rows its where predicate selects; a reader no filter names reads every row, and a principal may have one filter per expose. The predicate is one SQL boolean expression over this expose's columns, with no statement separator or comment. Principals are logical and resolve through binding.principals. GCP BigQuery: a row access policy (filterpredicate, grantees) per filter, plus one that selects every row for every other reader and writer, since a principal no row access policy names reads no row at all. AWS Glue: a Lake Formation data cells filter (rowfilter and the principal's excluded columns), and the principal's SELECT is granted on the filter instead of the table.

+
-
-
-
-
-
-

- -

+
+
+ -
-
+ + + + No Additional Items

Each item of this array must be:

+
+
+ Type: object
-

Privacy policy labels.

-
Same definition as labels -
-
-
-
-
-
-
-
-Type: object
+ No Additional Properties + + + + + + +
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "Public"
  • "Internal"
  • "Confidential"
  • "Restricted"
-
+ authz + + + + rowFilters + + + + rowFilters items + + + + principal
Type: string
+

Logical principal the filter applies to, written as in accessPolicy; binding.principals maps it per cloud.

+
@@ -9901,18 +10071,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: object
-

NEW in v0.7.1: AI/LLM usage governance - controls which AI models can consume this data and for what purposes.

-
- - No Additional Properties + authz + + + + rowFilters + + + + rowFilters items + + + + name
Type: string
+

The filter's name on the platform (the row access policy or data cells filter); derived from the principal when omitted.

+
Must match regular expression: ^[a-z][a-z0-9_]{0,62}$ -
+
+
+
+
+
-
+

- +

-
+
Type: array of string
-

Whitelist of AI models permitted to consume this data. Examples: ['gpt-4', 'claude-3-opus', 'gemini-pro']. Empty array = no AI access.

+ rowFilters + + + + rowFilters items + + + + where
Type: string
+

One SQL boolean expression over the expose's columns, e.g. consentdataanalytics = true AND is_suppressed = false.

- +

Must be at least 1 characters long

-

All items must be unique

No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
-Must match regular expression: ^[a-z0-9][a-z0-9-_.]*[a-z0-9]$|^[a-z0-9]$ + rowFilters items + + + + description
Type: string
+ -
-
Examples:
-
"gpt-4"
-
-
"claude-3-opus"
-
-
"claude-3-sonnet"
-
-
"gemini-pro"
-
-
"gemini-ultra"
-
-
"llama-3"
-
-
"mixtral-8x7b"
-
+ +
+
+
@@ -10088,18 +10304,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of string
-

Blacklist of AI models explicitly prohibited from consuming this data. Takes precedence over allowedModels.

-
+ exposes + + + + exposes items + + + + policy + + + + privacy + + + + masking
Type: array of object
+
+

Columns to treat before they land. The DuckDB acquisition runner rewrites each named column inside the write, so the file, S3 object or BigQuery load and the DLQ receive the treated value; quality gates judge the source value first. A treated column lands as a string, so declare it with a string type (string, or VARCHAR under a schemaPolicy that compares source type names). A rule the build cannot apply (an unset salt or key, k_anonymity, a column the stream lacks, a non-string declared type, the incremental cursor) refuses the build rather than landing the column untreated. fluid verify checks every non-null value of a masked column for its strategy's shape, on a local file and, through Athena, on an S3+Glue table. The MCP output-port gateway drops masked columns from what it serves.

+ +
+
+ +
-

All items must be unique

No Additional Items

Each item of this array must be:

+ No Additional Items

Each item of this array must be:

-
+
Type: string
- + masking items
Type: object
+ No Additional Properties -
-
-
-
-
-
-
+
-
+

- +

-
+
Type: integer
-

Maximum tokens per AI request. Prevents excessive data exposure in single queries.

-
+ masking + + + + masking items + + + + column
Type: string
+ -

Value must be greater or equal to 1

+ -
-
Examples:
-
4096
-
-
8192
-
-
16384
-
-
+
-
+
-
+

- +

-
+
Type: integer
-

Daily token limit for AI consumption. Enforces usage quotas.

-
+ masking + + + + masking items + + + + strategy
Type: enum (of string)
+
+

hash: lowercase hex SHA-256 of salt || value, deterministic so joins work (64 hex characters). mask: every character but the first params.keepFirst and the last params.keepLast replaced by '*', length preserved. tokenize: HMAC-SHA256 token, 32 hex characters, the token the tokenizepii preLand hook makes. encrypt: AES-GCM, written as 'aesgcm:v1:' + base64url(nonce || ciphertext || tag) with the column name as associated data; reversible with the key, not deterministic. kanonymity: accepted by the schema but refused at landing: it is a property of a whole table over its quasi-identifiers, not of one value.

+ +
+
+ +
+

Must be one of:

+
  • "mask"
  • "hash"
  • "tokenize"
  • "encrypt"
  • "k_anonymity"
+
-

Value must be greater or equal to 1

+
-
+
-
+

- +

-
+
Type: array of enum (of string)
-

Permitted AI use cases. Controls whether data can be used for training, inference, reasoning, etc.

+ masking + + + + masking items + + + + params
Type: object
+

Strategy parameters. The runner accepts exactly these keys and refuses any other, including a literal salt or key: secrets come only from the environment, never the contract, which is published with the plan, the bundle and the Glue table's fluid_contract parameter. Contracts on 0.7.5 and earlier may use the same keys; their schemas leave params untyped, and the runner checks them at build time.

-

All items must be unique

No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: enum (of string)
-
-

Must be one of:

-
  • "inference"
  • "reasoning"
  • "analysis"
  • "summarization"
  • "classification"
  • "embedding"
  • "search"
  • "qa"
  • "code_generation"
  • "fine_tuning"
  • "training"
  • "rag"
-
- - - - - - -
-

-
Examples:
-
[
-    "inference",
-    "reasoning",
-    "summarization"
-]
-
-
[
-    "search",
-    "qa",
-    "rag"
-]
-
-
+ masking items + + + + params + + + + saltEnv
Type: string Default: "FLUID_PII_HASH_SECRET"
+

NEW in v0.7.6: hash only. The environment variable holding the salt, at least 16 bytes. Unset or empty, the build is refused: an unsalted SHA-256 of a phone number is reversed by enumerating the numbering plan.

+
Must match regular expression: ^[A-Z_][A-Z0-9_]{0,127}$ + + + + + +
-
+
-
+

- +

-
+
Type: array of enum (of string)
-

Prohibited AI use cases. Example: Allow reporting but deny model training.

-
- - - - - -

All items must be unique

No Additional Items

Each item of this array must be:

-
-
- - - Type: enum (of string)
-
-

Must be one of:

-
  • "inference"
  • "reasoning"
  • "analysis"
  • "summarization"
  • "classification"
  • "embedding"
  • "search"
  • "qa"
  • "code_generation"
  • "fine_tuning"
  • "training"
  • "rag"
-
+ keyEnv
Type: string
+

NEW in v0.7.6: tokenize and encrypt. The environment variable holding the key. tokenize defaults to FLUIDPIITOKENIZATIONKEY and needs at least 32 bytes; encrypt defaults to FLUIDPIIENCRYPTIONSECRET_KEY and needs the base64 of a 16, 24 or 32-byte AES key (openssl rand -base64 32). Unset or empty, the build is refused.

+
Must match regular expression: ^[A-Z_][A-Z0-9_]{0,127}$ -
-

-
Example:
-
[
-    "training",
-    "fine_tuning"
-]
-
-
-
+
-
+

- +

-
+
Type: boolean Default: false
-

Whether AI agents can use this data for multi-step reasoning/chain-of-thought. False = simple retrieval only.

+ masking + + + + masking items + + + + params + + + + keepFirst
Type: integer Default: 0
+

NEW in v0.7.6: mask only. Leading characters left as they are.

- +

Value must be greater or equal to 0 and lesser or equal to 64

-
+
-
+

- +

-
+
Type: boolean Default: false
-

Whether AI systems can cache/store this data. False = ephemeral processing only.

+ masking + + + + masking items + + + + params + + + + keepLast
Type: integer Default: 4
+

NEW in v0.7.6: mask only. Trailing characters left as they are, so +46701234567 lands as ********4567. A value no longer than keepFirst + keepLast is masked entirely.

- +

Value must be greater or equal to 0 and lesser or equal to 64

-
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: object
-

Data retention requirements for AI systems consuming this data.

-
+ masking + + + + masking items + + + + params + + + + additionalProperties
Type: object
+ -
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: integer
-

Maximum days AI system can retain this data. 0 = no retention allowed.

-
- - - -

Value must be greater or equal to 0

- - + masking items + + + + tags
Type: array of string
+

Privacy masking rule tags.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: boolean Default: true
-

Whether AI system must delete data after use.

-
- - - - - - + masking items + + + + labels
Type: object
+

Privacy masking rule labels.

+
Same definition as labels
+
+
-
+
-
+

- +

-
+
Type: boolean Default: true
-

Whether AI consumption must be logged/audited.

-
+ rowLevelPolicy
Type: object
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

Free-text description of allowed purposes. Example: 'Customer service chatbot only, not for marketing'.

+ rowLevelPolicy + + + + expression
Type: string
+

Provider-specific predicate

@@ -11049,18 +11400,18 @@

-
+
-
+

- +

-
+
Type: array of string
-

Agent policy tags for categorization.

+ rowLevelPolicy + + + + tags
Type: array of string
+

Row-level policy tags.

Same definition as tags
-
+
-
+

- +

-
+
Type: object
-

Agent policy labels for automation.

+ rowLevelPolicy + + + + labels
Type: object
+

Row-level policy labels.

Same definition as labels
@@ -11165,18 +11530,18 @@

-
+
-
+

- +

-
+
Type: array of string
-

Policy-level tags for governance.

+ privacy + + + + tags
Type: array of string
+

Privacy policy tags.

Same definition as tags
-
+
-
+

- +

-
+
Type: object
-

Policy-level labels for automation.

+ privacy + + + + labels
Type: object
+

Privacy policy labels.

Same definition as labels
@@ -11267,18 +11646,18 @@

-
+
-
+

- +

-
+
Type: object
- - - No Additional Properties

- -

-
- - - Type: enum (of string)
+
+

Must be one of:

+
  • "Public"
  • "Internal"
  • "Confidential"
  • "Restricted"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

-
Type: object
+

NEW in v0.7.1: AI/LLM usage governance - controls which AI models can consume this data and for what purposes.

+
- + No Additional Properties + + + - -
- -
+
+
+
+

+ +

+
+ +
+
Type: object
- + allowedModels
Type: array of string
+

Whitelist of AI models permitted to consume this data. Examples: ['gpt-4', 'claude-3-opus', 'gemini-pro']. Empty array = no AI access.

+
- -
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: const
-Specific value: "gcp" + allowedModels items
Type: string
+Must match regular expression: ^[a-z0-9][a-z0-9-_.]*[a-z0-9]$|^[a-z0-9]$ - +
+
Examples:
+
"gpt-4"
+
+
"claude-3-opus"
+
+
"claude-3-sonnet"
+
+
"gemini-pro"
+
+
"gemini-ultra"
+
+
"llama-3"
+
+
"mixtral-8x7b"
+
+
+
+
-
+
+
+
+

+ +

+
- - -
- +
+
Type: object
- + deniedModels
Type: array of string
+

Blacklist of AI models explicitly prohibited from consuming this data. Takes precedence over allowedModels.

+
- -
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: object
+ deniedModels items
Type: string
@@ -11641,18 +12002,24 @@

-
+
+

+
+
+
+
+
-
+

- +

-
+

-
-

Must not be:

-
- Type: integer
+

Maximum tokens per AI request. Prevents excessive data exposure in single queries.

+
+ + + +

Value must be greater or equal to 1

+ +
+
Examples:
+
4096
+
+
8192
+
+
16384
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
-Must match regular expression: ^(alias/|arn:) - - - - - - -
-
-
+ maxTokensPerDay
Type: integer
+

Daily token limit for AI consumption. Enforces usage quotas.

+
- +

Value must be greater or equal to 1

-
-
-
-
-
+
-
+

- +

-
+
Type: object
- + allowedUseCases
Type: array of enum (of string)
+

Permitted AI use cases. Controls whether data can be used for training, inference, reasoning, etc.

+
- -
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-

Each additional property must conform to the following schema

+
+ Type: enum (of string)
+
+

Must be one of:

+
  • "inference"
  • "reasoning"
  • "analysis"
  • "summarization"
  • "classification"
  • "embedding"
  • "search"
  • "qa"
  • "code_generation"
  • "fine_tuning"
  • "training"
  • "rag"
+
+ + + + + + +
+

+
Examples:
+
[
+    "inference",
+    "reasoning",
+    "summarization"
+]
+
+
[
+    "search",
+    "qa",
+    "rag"
+]
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
-

- -

-
- - - Type: array of enum (of string)
+

Prohibited AI use cases. Example: Allow reporting but deny model training.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
-Must match regular expression: ^(user|group|serviceAccount|domain):.+$ + deniedUseCases items
Type: enum (of string)
+
+

Must be one of:

+
  • "inference"
  • "reasoning"
  • "analysis"
  • "summarization"
  • "classification"
  • "embedding"
  • "search"
  • "qa"
  • "code_generation"
  • "fine_tuning"
  • "training"
  • "rag"
+
-
- +
+

+
Example:
+
[
+    "training",
+    "fine_tuning"
+]
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: boolean Default: false
+

Whether AI agents can use this data for multi-step reasoning/chain-of-thought. False = simple retrieval only.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
- + canStore
Type: boolean Default: false
+

Whether AI systems can cache/store this data. False = ephemeral processing only.

+
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
+

Data retention requirements for AI systems consuming this data.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
-Must match regular expression: ^(user|group|serviceAccount|domain):.+$ - - - - - - -
-
-
+ maxRetentionDays
Type: integer
+

Maximum days AI system can retain this data. 0 = no retention allowed.

+
- +

Value must be greater or equal to 0

-
+
+
+
+

+ +

-
-
-
- - - - - -
- +
+
Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

-
Type: boolean Default: true
+

Whether AI system must delete data after use.

+
+ - - + -
- -
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
- + auditRequired
Type: boolean Default: true
+

Whether AI consumption must be logged/audited.

+
-
+
+
+
+
+
-
+

- +

-
+
Type: const
-Specific value: "aws" + purposeLimitation
Type: string
+

Free-text description of allowed purposes. Example: 'Customer service chatbot only, not for marketing'.

+
@@ -12432,13 +12851,19 @@

-
+
+
+
+

+ +

+
- - -
- +
+
Type: object
- - - - - - - -
+ tags
Type: array of string
+

Agent policy tags for categorization.

+
Same definition as tags +
+
+
+
+
-
+

- +

-
+
Type: object
- - - - - - - -Type: object
+

Agent policy labels for automation.

+
Same definition as labels +
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: array of string
+

Policy-level tags for governance.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
-
-

Must not be:

-
-
- + labels
Type: object
+

Policy-level labels for automation.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
+ + + No Additional Properties

+ +

+
+ + + Type: object
-Must match regular expression: ^projects/ - - - - - - -
-
-
- + item 0
Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ +
+ +
-
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
Type: object
+ if
Type: object
@@ -12797,19 +13239,19 @@

-
+
-
+

- +

-
-

Each additional property must conform to the following schema

+
+

-

- -

-
Type: const
+Specific value: "gcp" + + + + + + +
+
+
+
+
+ + + +
Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
-Must match regular expression: ^arn:aws[a-z0-9-]*:iam:: - - - - - - -
- - - Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of string
- - - - - - - No Additional Items

Each item of this array must be:

-
-
- - -
+
+

Must not be:

+
+
+ + + Type: string
-Must match regular expression: ^arn:aws[a-z0-9-]*:iam:: + not
Type: object
+Must match regular expression: ^(alias/|arn:) -
-
+
+
@@ -13151,32 +13608,18 @@

-
- - - - - - -
- - - - - - -
+
-
+

- +

-
+
Type: enum (of string)
-

Target platform. NEW in v0.7.5: 'pgvector' for the vector/embeddings output port (pgvector-on-PostgreSQL RAG target — emits the embeddings table + ANN index DDL for the ai-embeddable columns). NEW in v0.7.5: 'confluent' for the Confluent Cloud Tableflow managed Kafka→Iceberg emitter. NEW in v0.7.4: 'postgres' for the MCP output-port PostgreSQL driver (AWS Athena uses platform 'aws').

-
-

Must be one of:

-
  • "gcp"
  • "aws"
  • "azure"
  • "snowflake"
  • "databricks"
  • "kafka"
  • "confluent"
  • "local"
  • "kubernetes"
  • "postgres"
  • "pgvector"
  • "other"
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: enum (of string)
-

Binding format. NEW in v0.7.5: 'pgvectortable' β€” a pgvector embeddings table target for the vector output port. NEW in v0.7.4: 'postgrestable' (PostgreSQL driver), 'athenatable' / 'gluetable' (AWS Athena driver) for the MCP output port.

-
-

Must be one of:

-
  • "bigquery_table"
  • "snowflake_table"
  • "snowflake_view"
  • "gcs_file"
  • "s3_file"
  • "http_api"
  • "grpc_api"
  • "pubsub_topic"
  • "kafka_topic"
  • "delta_table"
  • "iceberg"
  • "parquet"
  • "csv"
  • "json"
  • "redshift_table"
  • "redshift_serverless"
  • "redshift_external_schema"
  • "postgres_table"
  • "athena_table"
  • "glue_table"
  • "pgvector_table"
  • "other"
-
+ principals
Type: object
+ -
-
-
-
-
+
-
+

- +

-
-
+
+

Each additional property must conform to the following schema

Type: object
-

Apache Iceberg table format configuration

-
No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: enum (of integer)
-

Iceberg table format version

-
-

Must be one of:

-
  • 1
  • 2
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
+

+ +

+
+ Type: enum (of string)
-

Underlying file format for data files

-
-

Must be one of:

-
  • "parquet"
  • "orc"
  • "avro"
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: array of object
-

Partition transformation specifications

-
+ item 0
Type: string
+Must match regular expression: ^(user|group|serviceAccount|domain):.+$ - No Additional Items

Each item of this array must be:

-
-
+ +
Type: object
- - - - - - - -
-
-
-

- -

-
- -
-
- - Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
-

Partition field name

-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: string
-

Source column for partitioning

-
+ item 1 + + + + item 1 items
Type: string
+Must match regular expression: ^(user|group|serviceAccount|domain):.+$ + + + + + + +
+
+
@@ -13726,19 +14029,20 @@

-
-
-
-

- -

+
+
+
+
+ -
-
+ + + + +
+ Type: enum (of string)
-

Partition transform function

-
-

Must be one of:

-
  • "identity"
  • "year"
  • "month"
  • "day"
  • "hour"
  • "bucket"
  • "truncate"
-
- + item 1
Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ +
+ +
-
-
-
-
-
-
-
-

- -

-
- -
-
Type: integer
-

Number of buckets (for bucket transform)

-
+ if
Type: object
+ -
-
-
-
-
+
-
+

- +

-
+
Type: integer
-

Truncate width (for truncate transform)

-
+ platform
Type: const
+Specific value: "aws" @@ -13957,25 +14234,13 @@

-
-
-
-
-
-
-
-
-

- -

-
-
-
+ + +
+ Type: array of object
-

Sort order specifications for data files

-
+ item 1 + + + + then
Type: object
+ - No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: object
+ then + + + + encryption
Type: object
@@ -14075,18 +14363,18 @@

-
+
-
+

- +

-
+
Type: string
-

Column to sort by

-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
+ encryption + + + + kms

+
+

Must not be:

+
+
+ Type: enum (of string)
-

Sort direction

-
-

Must be one of:

-
  • "asc"
  • "desc"
-
+ encryption + + + + kms + + + + not
Type: object
+Must match regular expression: ^projects/ + + + + + + +
+
+
@@ -14230,18 +14524,22 @@

Must be one of:

-
+
+
+
+

+
-
+

- +

-
+
Type: enum (of string)
-

NULL value ordering

-
-

Must be one of:

-
  • "nulls-first"
  • "nulls-last"
-
+ principals
Type: object
+ -
-
-
-
-
-
-
-
-
-
-
+
-
+

- +

-
-
+
+

Each additional property must conform to the following schema

Type: object
-

Iceberg table properties

-
- - - - - - -
-
-
-

- -

-
- -
-

Each additional property must conform to the following schema

- -
+

+ +

+
+ + + Type: string
- + binding + + + + allOf + + + + item 1 + + + + then + + + + principals + + + + additionalProperties + + + + anyOf + + + + item 0
Type: string
+Must match regular expression: ^arn:aws[a-z0-9-]*:iam:: -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
+
+ Type: object
-

NEW in v0.7.5: vector / embeddings output-port configuration (platform 'pgvector', format 'pgvectortable'). Consumed by fluidbuild.output_ports.vector to emit the pgvector embeddings-table + ANN-index DDL and a RAG provenance manifest for the schema's ai-embeddable columns.

-
No Additional Properties + allOf + + + + item 1 + + + + then + + + + principals + + + + additionalProperties + + + + anyOf + + + + item 1
Type: array of string
+ - -
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: integer
-

Embedding vector width (e.g. 1536 for OpenAI text-embedding-3-small, 1024 for Cohere embed-english-light). Maps to the pgvector column type vector(<dimensions>).

-
- - - -

Value must be greater or equal to 1 and lesser or equal to 16000

- - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: string
-

Provenance: the embedding model that produces the vectors (e.g. 'text-embedding-3-small'). Recorded in the RAG manifest and the embeddings-table's embedding_model column so a retriever can verify model parity.

-
+ item 1 + + + + item 1 items
Type: string
+Must match regular expression: ^arn:aws[a-z0-9-]*:iam:: + + + + + + +
+
+
@@ -14624,18 +14949,36 @@

-
+
+
+
+
+
+ + + + + + +
+ + + + + + +
-
+

- +

-
+
Type: enum (of string) Default: "vector"
-

pgvector column type: 'vector' (float32, up to 2000 dims) or 'halfvec' (float16, half the storage, up to 4000 dims).

-
Type: enum (of string)
+

Target platform. NEW in v0.7.5: 'pgvector' for the vector/embeddings output port (pgvector-on-PostgreSQL RAG target — emits the embeddings table + ANN index DDL for the ai-embeddable columns). NEW in v0.7.5: 'confluent' for the Confluent Cloud Tableflow managed Kafka→Iceberg emitter. NEW in v0.7.4: 'postgres' for the MCP output-port PostgreSQL driver (AWS Athena uses platform 'aws').

+

Must be one of:

-
  • "vector"
  • "halfvec"
+
  • "gcp"
  • "aws"
  • "azure"
  • "snowflake"
  • "databricks"
  • "kafka"
  • "confluent"
  • "local"
  • "kubernetes"
  • "postgres"
  • "pgvector"
  • "other"
@@ -14689,18 +15025,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: enum (of string) Default: "hnsw"
-

ANN index built on the embedding column. 'hnsw' (higher recall / lower query latency, buildable before data lands), 'ivfflat' (lower build cost, needs data first), or 'none' (no index β€” exact search only).

-
Type: enum (of string)
+

Binding format. NEW in v0.7.5: 'pgvectortable' β€” a pgvector embeddings table target for the vector output port. NEW in v0.7.4: 'postgrestable' (PostgreSQL driver), 'athenatable' / 'gluetable' (AWS Athena driver) for the MCP output port.

+

Must be one of:

-
  • "hnsw"
  • "ivfflat"
  • "none"
+
  • "bigquery_table"
  • "snowflake_table"
  • "snowflake_view"
  • "gcs_file"
  • "s3_file"
  • "http_api"
  • "grpc_api"
  • "pubsub_topic"
  • "kafka_topic"
  • "delta_table"
  • "iceberg"
  • "parquet"
  • "csv"
  • "json"
  • "redshift_table"
  • "redshift_serverless"
  • "redshift_external_schema"
  • "postgres_table"
  • "athena_table"
  • "glue_table"
  • "pgvector_table"
  • "other"
@@ -14754,18 +15083,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: enum (of string) Default: "cosine"
-

Distance function the index optimises for → pgvector operator class: cosine→vectorcosineops, l2→vectorl2ops, innerproduct→vectoripops, l1→vectorl1_ops.

-
-

Must be one of:

-
  • "cosine"
  • "l2"
  • "inner_product"
  • "l1"
-
+ icebergConfig
Type: object
+

Apache Iceberg table format configuration

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

Target embeddings table name. Defaults to '<exposeId>_embeddings' when omitted.

-
+ writeVersion
Type: enum (of integer)
+

Iceberg table format version

+
+

Must be one of:

+
  • 1
  • 2
+
@@ -14881,18 +15199,18 @@

-
+
-
+

- +

-
+
Type: string
-

Provenance: the source primary-key column that ties each embedding row back to its origin row (written to the embeddings-table's source_key column). Enables update/delete propagation from the source product to the vector store.

-
+ fileFormat
Type: enum (of string)
+

Underlying file format for data files

+
+

Must be one of:

+
  • "parquet"
  • "orc"
  • "avro"
+
@@ -14943,18 +15264,18 @@

-
+
-
+

- +

-
+
Type: object
-

HNSW index tuning (used when indexType is 'hnsw').

-
No Additional Properties + partitionSpec
Type: array of object
+

Partition transformation specifications

+
- -
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: integer
-

Max bi-directional links per node (pgvector default 16).

-
+ partitionSpec items
Type: object
+ -

Value must be greater or equal to 1

+ -
-
-
-
-
+
-
+

- +

-
+
Type: integer
-

Build-time dynamic candidate-list size (pgvector default 64).

+ partitionSpec items + + + + name
Type: string
+

Partition field name

-

Value must be greater or equal to 1

+
-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

IVFFlat index tuning (used when indexType is 'ivfflat').

-
No Additional Properties + partitionSpec + + + + partitionSpec items + + + + sourceColumn
Type: string
+

Source column for partitioning

+
-
+
+
+
+
+
-
+

- +

-
+
Type: integer
-

Number of inverted lists (pgvector guidance: rows/1000 for < 1M rows).

-
+ partitionSpec items + + + + transform
Type: enum (of string)
+

Partition transform function

+
+

Must be one of:

+
  • "identity"
  • "year"
  • "month"
  • "day"
  • "hour"
  • "bucket"
  • "truncate"
+
-

Value must be greater or equal to 1

+
-
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Provider-native addressing with tightened structure for better interop.

+ icebergConfig + + + + partitionSpec + + + + partitionSpec items + + + + numBuckets
Type: integer
+

Number of buckets (for bucket transform)

- - No Additional Properties -
+
+
+
+
+
-
+

- +

-
+
Type: string
-

Cloud account ID

+ partitionSpec + + + + partitionSpec items + + + + width
Type: integer
+

Truncate width (for truncate transform)

@@ -15393,18 +15759,24 @@

-
+
+
+
+
+
+
+
-
+

- +

-
+
Type: string
-

GCP project ID

+ sortOrder
Type: array of object
+

Sort order specifications for data files

- -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

BigQuery dataset or database name

-
+ sortOrder + + + + sortOrder items
Type: object
+ -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

Database name

+ sortOrder + + + + sortOrder items + + + + column
Type: string
+

Column to sort by

@@ -15579,18 +15953,18 @@

-
+
-
+

- +

-
+
Type: string
-

Schema name

-
+ sortOrder + + + + sortOrder items + + + + direction
Type: enum (of string)
+

Sort direction

+
+

Must be one of:

+
  • "asc"
  • "desc"
+
@@ -15641,18 +16032,18 @@

-
+
-
+

- +

-
+
Type: string
-

Table name

-
+ sortOrder + + + + sortOrder items + + + + nullOrder
Type: enum (of string)
+

NULL value ordering

+
+

Must be one of:

+
  • "nulls-first"
  • "nulls-last"
+
@@ -15703,18 +16111,24 @@

-
+
+
+
+
+
+
+
-
+

- +

-
+
Type: string
-

Storage bucket name

+ properties
Type: object
+

Iceberg table properties

@@ -15761,23 +16175,19 @@

-

-
-
-
-
+
-
+

- +

-
-
+
+

Each additional property must conform to the following schema

Type: string
-

File or object path

-
+ properties + + + + additionalProperties
Type: string
+ @@ -15827,18 +16243,26 @@

-
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: string
-

API gateway endpoint

-
+ vectorConfig
Type: object
+

NEW in v0.7.5: vector / embeddings output-port configuration (platform 'pgvector', format 'pgvectortable'). Consumed by fluidbuild.output_ports.vector to emit the pgvector embeddings-table + ANN-index DDL and a RAG provenance manifest for the schema's ai-embeddable columns.

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: stringFormat: uri
-

Base URL for API endpoints

+ dimensions
Type: integer
+

Embedding vector width (e.g. 1536 for OpenAI text-embedding-3-small, 1024 for Cohere embed-english-light). Maps to the pgvector column type vector(<dimensions>).

- +

Value must be greater or equal to 1 and lesser or equal to 16000

-
+
-
+

- +

-
+
Type: string
-

Pub/Sub or Kafka topic name

+ embeddingModel
Type: string
+

Provenance: the embedding model that produces the vectors (e.g. 'text-embedding-3-small'). Recorded in the RAG manifest and the embeddings-table's embedding_model column so a retriever can verify model parity.

@@ -16013,18 +16426,18 @@

-
+
-
+

- +

-
+
Type: string
-

Pub/Sub subscription name

-
+ vectorType
Type: enum (of string) Default: "vector"
+

pgvector column type: 'vector' (float32, up to 2000 dims) or 'halfvec' (float16, half the storage, up to 4000 dims).

+
+

Must be one of:

+
  • "vector"
  • "halfvec"
+
@@ -16075,18 +16491,18 @@

-
+
-
+

- +

-
+
Type: string
-

Cloud region

-
+ indexType
Type: enum (of string) Default: "hnsw"
+

ANN index built on the embedding column. 'hnsw' (higher recall / lower query latency, buildable before data lands), 'ivfflat' (lower build cost, needs data first), or 'none' (no index β€” exact search only).

+
+

Must be one of:

+
  • "hnsw"
  • "ivfflat"
  • "none"
+
@@ -16137,18 +16556,18 @@

-
+
-
+

- +

-
+
Type: string
-

Cloud zone

-
+ distanceMetric
Type: enum (of string) Default: "cosine"
+

Distance function the index optimises for → pgvector operator class: cosine→vectorcosineops, l2→vectorl2ops, innerproduct→vectoripops, l1→vectorl1_ops.

+
+

Must be one of:

+
  • "cosine"
  • "l2"
  • "inner_product"
  • "l1"
+
@@ -16199,18 +16621,18 @@

-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: Confluent Cloud environment id (env-xxxxx) for the Tableflow emitter.

+ table
Type: string
+

Target embeddings table name. Defaults to '<exposeId>_embeddings' when omitted.

@@ -16261,18 +16683,18 @@

-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: Confluent Cloud Kafka cluster id (lkc-xxxxx) the Tableflow topic belongs to.

+ sourceKeyColumn
Type: string
+

Provenance: the source primary-key column that ties each embedding row back to its origin row (written to the embeddings-table's source_key column). Enables update/delete propagation from the source product to the vector store.

@@ -16323,18 +16745,18 @@

-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: AWS Kinesis Data Stream name β€” maps to awskinesisstream in the streaming IaC emitter.

-
+ hnsw
Type: object
+

HNSW index tuning (used when indexType is 'hnsw').

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: Amazon Redshift Serverless namespace name β€” maps to awsredshiftserverlessnamespace.

+ hnsw + + + + m
Type: integer
+

Max bi-directional links per node (pgvector default 16).

- +

Value must be greater or equal to 1

-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: Amazon Redshift Serverless workgroup name β€” maps to awsredshiftserverlessworkgroup.

+ hnsw + + + + efConstruction
Type: integer
+

Build-time dynamic candidate-list size (pgvector default 64).

- +

Value must be greater or equal to 1

-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: ARN of the IAM role attached to the Redshift Serverless namespace / used by the redshift-data CREATE EXTERNAL SCHEMA bridge.

-
+ ivfflat
Type: object
+

IVFFlat index tuning (used when indexType is 'ivfflat').

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: Redshift external (Spectrum) schema name created over a Glue database via the redshift-data CREATE EXTERNAL SCHEMA bridge.

+ ivfflat + + + + lists
Type: integer
+

Number of inverted lists (pgvector guidance: rows/1000 for < 1M rows).

- +

Value must be greater or equal to 1

-
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: Glue Data Catalog database that backs the Redshift external schema.

+ location
Type: object
+

Provider-native addressing with tightened structure for better interop.

+ + No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: ARN of the pre-created AWS IAM role Confluent Tableflow assumes (byobaws). Its trust policy is updated post-apply with the provider integration's externalid (two-phase IAM).

+ account
Type: string
+

Cloud account ID

@@ -16757,18 +17195,18 @@

-
+
-
+

- +

-
+
Type: string
-
-

NEW in v0.7.5: The Iceberg catalog that owns this table: glue, rest, lakekeeper, polaris, unity, nessie, bigquery, hive, jdbc, hadoop, dynamodb or snowflake-managed. Spellings fold case and '-'/'', so icebergrest == rest and snowflake == snowflake-managed; fluid validate refuses any other value on an Iceberg expose. When absent: glue on platform aws, snowflake-managed on snowflake, rest elsewhere. The REST catalogs (rest, lakekeeper, polaris, unity) need location.uri plus location.warehouse as the catalog's warehouse NAME, not an object-store URI; Lakekeeper serves its Iceberg REST API under <host>/catalog (uri: http://lakekeeper:8181/catalog). On platform aws, a kind other than glue gets no Glue table and no Glue grant: that catalog creates and governs the table. Every emitter (streaming sinks, dbt catalogs.yml, the IaC, policy compile) reads one table, providers/icebergcatalog.py.

- -
-
- -
+ project
Type: string
+

GCP project ID

+
@@ -16826,18 +17257,18 @@

-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: Iceberg warehouse β€” the canonical Iceberg REST 'warehouse' property (maps to iceberg.catalog.warehouse): an object-store URI (s3://|gs://|abfss://) for object-store catalogs, or the catalog/warehouse name for a REST catalog.

+ dataset
Type: string
+

BigQuery dataset or database name

@@ -16888,18 +17319,18 @@

-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.5: REST Iceberg catalog endpoint URL β€” the canonical Iceberg REST 'uri' property (maps to iceberg.catalog.uri).

+ database
Type: string
+

Database name

@@ -16950,18 +17381,18 @@

-
+
-
+

- +

-
+
Type: array of string
-
-

NEW in v0.7.5: Iceberg partition columns β€” a FLUID abstraction over the Iceberg PARTITIONED BY / PartitionSpec (maps to iceberg.tables.default-partition-by). On a gcp BigQuery table with exposes[].lifecycle.expire: true (fluid-schema 0.7.6), one DATE, TIMESTAMP or DATETIME column to partition by (time_partitioning.field); without it the table is partitioned by ingestion time. With a column, retention counts from the date in that column, not from landing: BigQuery deletes a partition retention after its date, so a backfill of older rows is deleted at once. Leave it out for the S3 rule's semantics (age since written).

- -
-
- -
+ schema
Type: string
+

Schema name

+
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
- + table
Type: string
+

Table name

+
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: array of string
-

Binding configuration tags for infrastructure.

-
Same definition as tags + location + + + + bucket
Type: string
+

Storage bucket name

+
+ + + + + +
-
+
-
+

- +

-
+
Type: object
-

Binding configuration labels for automation.

-
Same definition as labels + location + + + + path
Type: string
+

File or object path

+
+ + + + + +
-
+
-
+

- +

-
+
Type: object
-

Provider-specific binding properties (e.g., cluster_by, comment for Snowflake).

+ location + + + + gateway
Type: string
+

API gateway endpoint

@@ -17228,19 +17687,23 @@

-
+
+

+
+
+
-
+

- +

-
-

Additional Properties of any type are allowed.

+
+
Type: object
- + host
Type: string
+

NEW in v0.7.6: Host or endpoint; for a Kafka binding, the bootstrap servers. The ODCS server mapper already reads host and falls back to account, so until now the only valid way to set the bootstrap servers was a field named for something else β€” and with neither set, the exported ODCS server is the placeholder ${KAFKA_BOOTSTRAP_SERVERS}. Usually set per environment in an overlay rather than in the base contract.

+
@@ -17289,22 +17753,18 @@

-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Per-resource governance: registerLocation, principal grants, tag associations, row/column filters. Account-wide governance (admins, tag definitions) lives at the contract top-level governance block.

+ location + + + + baseUrl
Type: stringFormat: uri
+

Base URL for API endpoints

- - No Additional Properties -
+
+
+
+
+
-
+

- +

-
+
Type: object
-

Per-resource LF settings: location registration, principal grants, LF-tag associations, row/column filters. Applies only to formats backed by S3 + Glue catalog (iceberg/parquet/csv/json/avro/orc/delta).

-
No Additional Properties + topic
Type: string
+

Pub/Sub or Kafka topic name

+
-
+
+
+
+
+
-
+

- +

-
+
Type: boolean Default: false
-

When true, emit awslakeformationresource that registers the binding's S3 path with Lake Formation. Required before LF can manage access to objects under that path.

+ subscription
Type: string
+

Pub/Sub subscription name

@@ -17473,18 +17939,18 @@

-
+
-
+

- +

-
+
Type: enum (of string) Default: "cross-account"
-
-

NEW in v0.7.6: Which grants[] principals get a statement in the awss3bucketpolicy emitted beside the grants (s3:ListBucket and s3:GetBucketLocation on the bucket, s3:GetObject under location.path). A statement lets its principal read the objects straight from S3, which skips Lake Formation's column, row and cell filters and its revocations. 'cross-account' (the default): only principals whose ARN names an AWS account other than the one running the apply, decided at plan time against the caller identity; same-account grantees get no statement, since on a registered location Lake Formation vends them credentials. 'none': no bucket policy at all; the tightest setting when the location is registered (registerLocation) and every reader, cross-account ones included, queries through a Lake Formation integrated engine, or when the bucket policy is managed elsewhere. 'all-grantees': every grantee, same-account included, which is the output before this field existed and reopens the direct S3 read path. An emitted awss3bucketpolicy is authoritative: it replaces every other statement on the bucket. When no grantee is left none is emitted.

- -
-
- -
-

Must be one of:

-
  • "cross-account"
  • "none"
  • "all-grantees"
-
+ region
Type: string
+

Cloud region

+
@@ -17552,18 +18001,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: array of object
-

Principal-based LF grants on this exposure's database/table. Each entry maps a principal to a permission set (and optional column projection). Emitted as awslakeformationpermissions resources. Field idiom adapted from ODCS v3 roles[] (principal + access-type), bound to IAM ARNs since LF is cloud-specific.

+ zone
Type: string
+

Cloud zone

- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
- No Additional Properties + environment_id
Type: string
+

NEW in v0.7.5: Confluent Cloud environment id (env-xxxxx) for the Tableflow emitter.

+
-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

IAM principal ARN (role or user) receiving the grant.

-
Must match regular expression: ^arn:aws[a-z0-9-]*:iam:: + kafka_cluster_id
Type: string
+

NEW in v0.7.5: Confluent Cloud Kafka cluster id (lkc-xxxxx) the Tableflow topic belongs to.

+
@@ -17761,18 +18187,18 @@

-
+
-
+

- +

-
+
Type: array of enum (of string)
-

LF permissions granted. Valid values per the hashicorp/aws awslakeformationpermissions resource.

+ stream
Type: string
+

NEW in v0.7.5: AWS Kinesis Data Stream name β€” maps to awskinesisstream in the streaming IaC emitter.

-

Must contain a minimum of 1 items

All items must be unique

No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: enum (of string)
-
-

Must be one of:

-
  • "ALL"
  • "SELECT"
  • "ALTER"
  • "DROP"
  • "DELETE"
  • "INSERT"
  • "DESCRIBE"
  • "CREATE_TABLE"
  • "CREATE_DATABASE"
  • "DATA_LOCATION_ACCESS"
-
+ namespace
Type: string
+

NEW in v0.7.5: Amazon Redshift Serverless namespace name β€” maps to awsredshiftserverlessnamespace.

+
-
-
-
+
-
+

- +

-
+
Type: array of string
-

Subset of permissions the principal can re-grant. Maps to awslakeformationpermissions.permissionswithgrant_option.

+ workgroup
Type: string
+

NEW in v0.7.5: Amazon Redshift Serverless workgroup name β€” maps to awsredshiftserverlessworkgroup.

-

All items must be unique

No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
+

NEW in v0.7.5: ARN of the IAM role attached to the Redshift Serverless namespace / used by the redshift-data CREATE EXTERNAL SCHEMA bridge.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+ Type: string
+

NEW in v0.7.5: Redshift external (Spectrum) schema name created over a Glue database via the redshift-data CREATE EXTERNAL SCHEMA bridge.

+
-
-
-
+
-
+

- +

-
+
Type: array of string
-

Optional column-level restriction. When set, the grant is emitted against awslakeformationpermissions.tablewithcolumns instead of .table, allowing the principal to read only the listed columns.

+ glue_database
Type: string
+

NEW in v0.7.5: Glue Data Catalog database that backs the Redshift external schema.

-

All items must be unique

No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
+

NEW in v0.7.5: ARN of the pre-created AWS IAM role Confluent Tableflow assumes (byobaws). Its trust policy is updated post-apply with the provider integration's externalid (two-phase IAM).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ location + + + + catalog
Type: string
+
+

NEW in v0.7.5: The Iceberg catalog that owns this table: glue, rest, lakekeeper, polaris, unity, nessie, bigquery, hive, jdbc, hadoop, dynamodb or snowflake-managed. Spellings fold case and '-'/'', so icebergrest == rest and snowflake == snowflake-managed; fluid validate refuses any other value on an Iceberg expose. When absent: glue on platform aws, snowflake-managed on snowflake, rest elsewhere. The REST catalogs (rest, lakekeeper, polaris, unity) need location.uri plus location.warehouse as the catalog's warehouse NAME, not an object-store URI; Lakekeeper serves its Iceberg REST API under <host>/catalog (uri: http://lakekeeper:8181/catalog). On platform aws, a kind other than glue gets no Glue table and no Glue grant: that catalog creates and governs the table. Every emitter (streaming sinks, dbt catalogs.yml, the IaC, policy compile) reads one table, providers/icebergcatalog.py.

+
+
+ +
-
-
-
+
-
+

- +

-
+
Type: string
+

NEW in v0.7.5: Iceberg warehouse β€” the canonical Iceberg REST 'warehouse' property (maps to iceberg.catalog.warehouse): an object-store URI (s3://|gs://|abfss://) for object-store catalogs, or the catalog/warehouse name for a REST catalog.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
-

Opposite of columns: every column EXCEPT these is granted. Maps to tablewithcolumns.excludedcolumnnames. Mutually exclusive with columns.

+ binding + + + + location + + + + uri
Type: string
+

NEW in v0.7.5: REST Iceberg catalog endpoint URL β€” the canonical Iceberg REST 'uri' property (maps to iceberg.catalog.uri).

-

All items must be unique

No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: array of string
+
+

NEW in v0.7.5: Iceberg partition columns β€” a FLUID abstraction over the Iceberg PARTITIONED BY / PartitionSpec (maps to iceberg.tables.default-partition-by). On a gcp BigQuery table with exposes[].lifecycle.expire: true (fluid-schema 0.7.6), one DATE, TIMESTAMP or DATETIME column to partition by (time_partitioning.field); without it the table is partitioned by ingestion time. With a column, retention counts from the date in that column, not from landing: BigQuery deletes a partition retention after its date, so a backfill of older rows is deleted at once. Leave it out for the S3 rule's semantics (age since written).

+ +
+
+ +
+ + + + + + No Additional Items

Each item of this array must be:

+
+ Type: string
@@ -18404,24 +18939,22 @@

-
-
-
+
-
+

- +

-
+
Type: array of string
+

Binding configuration tags for infrastructure.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
-
-

LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as awslakeformationresourcelftags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).

- -
-
- + exposes items + + + + binding + + + + labels
Type: object
+

Binding configuration labels for automation.

+
Same definition as labels +
- - - - - - -
+
+
+
-
+

- +

-
-

Each additional property must conform to the following schema

+
+
Type: object
+

Provider-specific binding properties (e.g., cluster_by, comment for Snowflake).

+
+ + + + + + +
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: string
+ binding + + + + properties + + + + additionalProperties
Type: object
@@ -18561,18 +19157,18 @@

-
+
-
+

- +

-
+
Type: object
-

Row-level (and optionally column-level) filter applied to all reads of this table via LF. Emitted as awslakeformationdatacellsfilter. The filter must then be granted to principals via grants[].permissions including SELECT β€” LF enforces both layers.

-
No Additional Properties + governance
Type: object
+

Per-resource governance: registerLocation, principal grants, tag associations, row/column filters. Account-wide governance (admins, tag definitions) lives at the contract top-level governance block.

+
+ + No Additional Properties -
+
-
+

- +

-
+
Type: string
-

Filter name (must be unique per table).

-
+ lakeFormation
Type: object
+

Per-resource LF settings: location registration, principal grants, LF-tag associations, row/column filters. Applies only to formats backed by S3 + Glue catalog (iceberg/parquet/csv/json/avro/orc/delta).

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

PartiQL-flavoured row predicate. ALL lets every row through (use when only column filtering is intended).

+ registerLocation
Type: boolean Default: false
+

When true, emit awslakeformationresource that registers the binding's S3 path with Lake Formation. Required before LF can manage access to objects under that path.

@@ -18778,18 +19337,18 @@

-
+
-
+

- +

-
+
Type: array of string
-

Columns visible through the filter. Mutually exclusive with allColumns / excludedColumnNames.

-
+ bucketPolicy
Type: enum (of string) Default: "cross-account"
+
+

NEW in v0.7.6: Which grants[] principals get a statement in the awss3bucketpolicy emitted beside the grants (s3:ListBucket and s3:GetBucketLocation on the bucket, s3:GetObject under location.path). A statement lets its principal read the objects straight from S3, which skips Lake Formation's column, row and cell filters and its revocations. 'cross-account' (the default): only principals whose ARN names an AWS account other than the one running the apply, decided at plan time against the caller identity; same-account grantees get no statement, since on a registered location Lake Formation vends them credentials. 'none': no bucket policy at all; the tightest setting when the location is registered (registerLocation) and every reader, cross-account ones included, queries through a Lake Formation integrated engine, or when the bucket policy is managed elsewhere. 'all-grantees': every grantee, same-account included, which is the output before this field existed and reopens the direct S3 read path. An emitted awss3bucketpolicy is authoritative: it replaces every other statement on the bucket. When no grantee is left none is emitted.

+ +
+
+ +
+

Must be one of:

+
  • "cross-account"
  • "none"
  • "all-grantees"
+
-

All items must be unique

No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
- + grants
Type: array of object
+

Principal-based LF grants on this exposure's database/table. Each entry maps a principal to a permission set (and optional column projection). Emitted as awslakeformationpermissions resources. Field idiom adapted from ODCS v3 roles[] (principal + access-type), bound to IAM ARNs since LF is cloud-specific.

+
- -
-
-
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: array of string
-

Columns hidden through the filter (everything else is visible).

-
+ grants items
Type: object
+ No Additional Properties -

All items must be unique

No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: string
- + principal
Type: string
+

IAM principal ARN (role or user) receiving the grant.

+
Must match regular expression: ^arn:aws[a-z0-9-]*:iam:: -
-
-
+
-
+

- +

-
+
Type: boolean Default: false
-

Convenience for the LF column-wildcard form (every column visible). When true, columnNames / excludedColumnNames must be omitted.

-
- - - - - - -
-
-
-
-
-
-
-
-
-
-
-

- -

-
+ grants items + + + + permissions
Type: array of enum (of string)
+

LF permissions granted. Valid values per the hashicorp/aws awslakeformationpermissions resource.

+
+ -
-
+ + + +

Must contain a minimum of 1 items

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ Type: object
-

Where the protected Glue Data Catalog views of the expose's masked columns come from (NEW in v0.7.6). Lake Formation cannot mask, so a principal an access: mask column restriction names reads a protected, SECURITY DEFINER multi-dialect view, <table>_masked, that returns the columns masked by the restriction's rule and applies its row filter; its grant on the table excludes the columns. The definer is granted SELECT with grant option on the table and DESCRIBE on the database; the principal SELECT and DESCRIBE on the view.

-
No Additional Properties + grants + + + + grants items + + + + permissions + + + + permissions items
Type: enum (of string)
+
+

Must be one of:

+
  • "ALL"
  • "SELECT"
  • "ALTER"
  • "DROP"
  • "DELETE"
  • "INSERT"
  • "DESCRIBE"
  • "CREATE_TABLE"
  • "CREATE_DATABASE"
  • "DATA_LOCATION_ACCESS"
+
-
+
+
+
+
+
+
+
-
+

- +

-
+
Type: string
-

ARN of the IAM role the views are defined as: Glue and Lake Formation must be allowed to assume it (sts:AssumeRole, sts:SetContext), and the applying principal to pass it (iam:PassRole).

+ grants items + + + + permissionsWithGrantOption
Type: array of string
+

Subset of permissions the principal can re-grant. Maps to awslakeformationpermissions.permissionswithgrant_option.

- -
-
-
-
-
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

The Glue connection of type VIEWVALIDATIONATHENA (WORKGROUP_NAME) Glue validates each view's SQL through; the applying principal and the definer need glue:PassConnection on it.

-
+ grants items + + + + permissionsWithGrantOption + + + + permissionsWithGrantOption items
Type: string
+ -
-
-
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

NEW in v0.7.6: Per-exposure packaging override β€” key-wise precedence over the contract-wide top-level packaging block (binding.packaging > packaging > absent-legacy).

+ governance + + + + lakeFormation + + + + grants + + + + grants items + + + + columns
Type: array of string
+

Optional column-level restriction. When set, the grant is emitted against awslakeformationpermissions.tablewithcolumns instead of .table, allowing the principal to read only the listed columns.

- - No Additional Properties - -
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: enum (of string)
-

NEW in v0.7.6: Blanket ownership mode for every container kind. Default when the block is present: 'isolated'. Per-kind exceptions go in containers.

-
-

Must be one of:

-
  • "isolated"
  • "shared"
-
+ lakeFormation + + + + grants + + + + grants items + + + + columns + + + + columns items
Type: string
+ +
+
-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.6: Pool id of the platform-owned tenant pool this product writes into. REQUIRED (enforced by the resolver) whenever any container resolves 'shared' β€” a pool must be addressable. Propagated as the fluid_pool label/tag for cost attribution.

+ lakeFormation + + + + grants + + + + grants items + + + + excludedColumns
Type: array of string
+

Opposite of columns: every column EXCEPT these is granted. Maps to tablewithcolumns.excludedcolumnnames. Mutually exclusive with columns.

-

Must be at least 1 characters long

- -
-
-
-
-
-
-
-

- -

-
-
-
+

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ Type: string
-

NEW in v0.7.6: Optional path to the platform team's pool manifest file; snapshotted into the bundle at stage 1 so bundleDigest covers it (any pool-file edit invalidates downstream plans).

-
- - - -

Must be at least 1 characters long

- - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

NEW in v0.7.6: Per-container-kind ownership overrides β€” each key wins over mode for that kind, yielding hybrid tiers. Kind↔platform mapping: bucket β†’ awss3bucket / googlestoragebucket; database β†’ snowflakedatabase AND awsgluecatalogdatabase; dataset β†’ googlebigquerydataset; schema β†’ snowflakeschema; warehouse β†’ snowflakewarehouse; cluster β†’ Confluent environment/cluster (v1: shared only).

-
No Additional Properties + excludedColumns items
Type: string
+ -
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-

NEW in v0.7.6: Ownership of the object-store bucket (awss3bucket / googlestoragebucket).

-
+ tags
Type: object
+
+

LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as awslakeformationresourcelftags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).

-
-

Must be one of:

-
  • "isolated"
  • "shared"
-
+
+
+ +
-
-
-
-
-
+
-
+

- +

-
-
+
+

Each additional property must conform to the following schema

Type: enum (of string)
-

NEW in v0.7.6: Ownership of the database container β€” covers Snowflake database AND AWS glue_database.

-
Same definition as bucket + tags + + + + additionalProperties
Type: string
+ + + + + + +
-
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-

NEW in v0.7.6: Ownership of the BigQuery dataset (googlebigquerydataset).

-
Same definition as bucket -
-
-
-
-Type: object
+

Row-level (and optionally column-level) filter applied to all reads of this table via LF. Emitted as awslakeformationdatacellsfilter. The filter must then be granted to principals via grants[].permissions including SELECT β€” LF enforces both layers.

+
No Additional Properties + + + + + + +
-
+

- +

-
+
Type: enum (of string)
-

NEW in v0.7.6: Ownership of the Snowflake schema (snowflake_schema).

-
Same definition as bucket + rowFilter + + + + name
Type: string
+

Filter name (must be unique per table).

+
+ + + + + +
-
+
-
+

- +

-
+
Type: enum (of string)
-

NEW in v0.7.6: Ownership of the Snowflake warehouse (snowflake_warehouse) β€” isolated gives per-product cost attribution.

-
Same definition as bucket + rowFilter + + + + rowExpression
Type: string
+

PartiQL-flavoured row predicate. ALL lets every row through (use when only column filtering is intended).

+
+ + + + + +
-
+
-
+

- +

-
+
Type: enum (of string)
-

NEW in v0.7.6: Ownership of the Confluent environment/cluster. v1 accepts 'shared' only β€” dedicated-cluster provisioning is not yet supported (the resolver rejects 'isolated').

-
Same definition as bucket -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: array of string
+

Columns visible through the filter. Mutually exclusive with allColumns / excludedColumnNames.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
-

NEW in v0.7.6: Encryption at rest for an AWS S3 binding or a GCP BigQuery binding. Absent: nothing is emitted, and the cloud's default applies: SSE-S3 for new S3 objects, Google-managed keys for BigQuery.

-
No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Default: "product"
-
-

NEW in v0.7.6: The KMS key the bucket's objects are encrypted with (SSE-KMS with an S3 Bucket Key). 'product' (the default): fluid apply creates a customer managed key for each bucket this product owns (awskmskey, rotation on, alias alias/fluid/<contract id>/<bucket>) and makes it the bucket's default encryption (awss3bucketserversideencryptionconfiguration). Its key policy lets the account's IAM policies decide who may use it (the KMS default statement for the account root) and, when the binding carries governance.lakeFormation, lets the Lake Formation service-linked role AWSServiceRoleForLakeFormationDataAccess encrypt and decrypt, matched by aws:PrincipalArn so the role need not exist yet. A principal querying through Athena with credentials Lake Formation vends then needs no KMS permission of its own; a grantee in another account that the bucket policy lets read the objects directly (governance.lakeFormation.bucketPolicy) is also allowed kms:Decrypt, through S3 only. The writer (the identity the build runs as) needs kms:GenerateDataKey and kms:Decrypt from IAM. tofu destroy deletes the alias and schedules the key for deletion after 7 days, the minimum KMS allows; kms:CancelKeyDeletion recovers it until then. Dropping the block, or naming another key, removes the product key while the bucket keeps the objects encrypted under it, which become unreadable once the deletion window ends unless they are rewritten or the deletion is cancelled; fluid apply refuses that plan without --allow-data-loss. On a shared (pool) bucket the bucket's default encryption is the pool owner's, so 'product' is refused there. 'alias/<name>' or a key or alias ARN: an existing key, looked up at plan time (data awskmskey, so the identity running fluid apply needs kms:DescribeKey on it) and made the default encryption of a bucket this product owns by its key ARN, since S3 resolves an alias in the account of whoever writes; the plan fails unless the key is Enabled and a symmetric encryption key (SYMMETRICDEFAULT). On a pool bucket it is only checked. Its key policy is its owner's, and must let the Lake Formation role use it when the location is registered: an AWS managed key cannot be used with that role, so alias/aws/s3 is refused with registerLocation, and the plan fails for any key that is not customer managed. 'none': nothing is emitted. fluid verify checks that the key is Enabled and that the objects under the binding's prefix are SSE-KMS with it. On a gcp binding (fluid-schema 0.7.6, BigQuery tables): 'product' creates a Cloud KMS key ring and crypto key for each dataset this product writes (googlekmskeyring fluid-<contract id>-<dataset> and googlekmscryptokey 'bigquery', in the dataset's location, EU and US mapping to the europe and us key locations, rotated every 90 days), grants the project's BigQuery service agent roles/cloudkms.cryptoKeyEncrypterDecrypter on it, and makes it the dataset's defaultencryptionconfiguration and the table's encryptionconfiguration. The project must have the Cloud KMS API (cloudkms.googleapis.com) enabled. A key ring and key cannot be deleted on GCP: tofu destroy removes them from state and schedules the key's versions for destruction (30 days by default), and a later apply adopts the same names. 'projects/<p>/locations/<l>/keyRings/<r>/cryptoKeys/<k>': an existing key in the dataset's location, used as given; its owner grants the BigQuery service agent. 'none': Google-managed keys. An alias/... or arn:... value is refused on a gcp binding, and a Cloud KMS name on an aws binding. Adding a key to a table that exists replaces the table (BigQuery cannot re-key it in place), so fluid apply refuses that plan without --allow-data-loss; the next build lands the data again. fluid verify checks kmsKeyName on the dataset and the table.

+ columnNames items
Type: string
+ + + + + + +
+
+
+
+
+
+
+
+
+

+ +

-
- -

- -

-
- + +
+
Type: enum (of string)
-
-

Must be one of:

-
  • "product"
  • "none"
-
+ excludedColumnNames
Type: array of string
+

Columns hidden through the filter (everything else is visible).

+
- -
+

All items must be unique

No Additional Items

Each item of this array must be:

+
+
Type: object
-Must match regular expression: ^alias/[A-Za-z0-9/_-]{1,250}$ + excludedColumnNames + + + + excludedColumnNames items
Type: string
+ -
- +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
-Must match regular expression: ^arn:aws[a-z-]*:kms:[a-z0-9-]+:[0-9]{12}:(key|alias)/[A-Za-z0-9/_-]{1,250}$ + allColumns
Type: boolean Default: false
+

Convenience for the LF column-wildcard form (every column visible). When true, columnNames / excludedColumnNames must be omitted.

+
-
- +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
-Must match regular expression: ^projects/[a-z0-9.:-]{1,100}/locations/[a-z0-9-]{1,63}/keyRings/[A-Za-z0-9_-]{1,63}/cryptoKeys/[A-Za-z0-9_-]{1,63}$ - - - - - - -
+ maskedViews
Type: object
+

Where the protected Glue Data Catalog views of the expose's masked columns come from (NEW in v0.7.6). Lake Formation cannot mask, so a principal an access: mask column restriction names reads a protected, SECURITY DEFINER multi-dialect view, <table>_masked, that returns the columns masked by the restriction's rule and applies its row filter; its grant on the table excludes the columns. The definer is granted SELECT with grant option on the table and DESCRIBE on the database; the principal SELECT and DESCRIBE on the view.

+
No Additional Properties -
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
-
-

NEW in v0.7.6: Maps the contract's LOGICAL principals (accessPolicy.grants[].principal and exposes[].policy.authz.columnRestrictions[].principal, keyed exactly as the contract writes them) to the identities they stand for on this binding's platform, so the base contract stays cloud-neutral and each environment's overlay binds it. On gcp an identity is an IAM member (user:, group:, serviceAccount: or domain:); on aws an IAM principal ARN. A value is one identity or a list; [] says the principal has no identity on this cloud, and nothing is granted to it there. When the block is present, every principal the contract names for this expose must be a key: an unmapped one is refused, never emitted as written. Without the block the principals are used as written, except that a GCP principal in a reserved top-level domain (.example, .test, .invalid, .localhost) is refused as a placeholder. Modelled on ODCS v3's roles[] bound per server (servers[].roles) and on dbt grants resolved per target.

- -
-
- -
+ governance + + + + lakeFormation + + + + maskedViews + + + + definer
Type: string
+

ARN of the IAM role the views are defined as: Glue and Lake Formation must be allowed to assume it (sts:AssumeRole, sts:SetContext), and the applying principal to pass it (iam:PassRole).

+
-
+
+
+
+
+
-
+

- +

-
-

Each additional property must conform to the following schema

+
+

-

- -

-
- - - Type: string
+

The Glue connection of type VIEWVALIDATIONATHENA (WORKGROUP_NAME) Glue validates each view's SQL through; the applying principal and the definer need glue:PassConnection on it.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+ Type: object
+

NEW in v0.7.6: Per-exposure packaging override β€” key-wise precedence over the contract-wide top-level packaging block (binding.packaging > packaging > absent-legacy).

+
+ No Additional Properties -

Must be at least 1 characters long

+ -
- +
+
+
+

+ +

+
+ +
+
Type: array of string
- + mode
Type: enum (of string)
+

NEW in v0.7.6: Blanket ownership mode for every container kind. Default when the block is present: 'isolated'. Per-kind exceptions go in containers.

+
+

Must be one of:

+
  • "isolated"
  • "shared"
+
-

All items must be unique

No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
- - - - -

Must be at least 1 characters long

- - -
-
-
+ pool
Type: string
+

NEW in v0.7.6: Pool id of the platform-owned tenant pool this product writes into. REQUIRED (enforced by the resolver) whenever any container resolves 'shared' β€” a pool must be addressable. Propagated as the fluid_pool label/tag for cost attribution.

+
- +

Must be at least 1 characters long

-
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

NEW in v0.7.6: An expose's lifecycle. The same fields as the contract-root lifecycle, plus expire, which is only meaningful where there is storage to expire.

+ binding + + + + packaging + + + + poolManifest
Type: string
+

NEW in v0.7.6: Optional path to the platform team's pool manifest file; snapshotted into the bundle at stage 1 so bundleDigest covers it (any pool-file edit invalidates downstream plans).

- - No Additional Properties - +

Must be at least 1 characters long

-
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-

Unified lifecycle state vocabulary.

-
- -
-

Must be one of:

-
  • "preview"
  • "active"
  • "deprecated"
  • "retired"
-
+ packaging + + + + containers
Type: object
+

NEW in v0.7.6: Per-container-kind ownership overrides β€” each key wins over mode for that kind, yielding hybrid tiers. Kind↔platform mapping: bucket β†’ awss3bucket / googlestoragebucket; database β†’ snowflakedatabase AND awsgluecatalogdatabase; dataset β†’ googlebigquerydataset; schema β†’ snowflakeschema; warehouse β†’ snowflakewarehouse; cluster β†’ Confluent environment/cluster (v1: shared only).

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

How long this expose's data is kept (ISO-8601 duration, e.g. P30D, P7Y). A declaration unless expire is true. With expire: true on an AWS S3 binding, fluid apply expires the objects under the binding's prefix this long after they are written; on a GCP BigQuery table it deletes each daily partition this long after its day ends. Years and months are counted as 365 and 30 days, and a part of a day rounds up to a whole one.

-
Same definition as window -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: boolean Default: false
-
-

NEW in v0.7.6: Delete stored data once it is older than retention. Default false, so a contract that declares a retention period does not start deleting data when forge-cli is upgraded. Honoured by the AWS emitter on an S3 binding and by the GCP emitter on a BigQuery table; on a gcp binding that is not a BigQuery table (GCS, Pub/Sub, Iceberg storage) it is refused rather than dropped. AWS: fluid apply writes one rule per expose into the bucket's awss3bucketlifecycleconfiguration, filtered to the binding's prefix (location.path, else {database}/{table}/; a binding with neither is refused, so no rule ever expires a whole bucket). The rule expires the current version retention after it is written, removes a noncurrent version (on a versioning-enabled bucket) one day after it stops being current, and aborts an incomplete multipart upload after 7 days or the retention period if shorter. The lifecycle configuration is AUTHORITATIVE for the whole bucket: S3 keeps one per bucket, so rules that fluid apply did not write are replaced. It is therefore written only for a bucket this product owns; on a shared (pool) bucket (packaging) nothing is written, the pool's owner must hold the rule, and fluid verify checks that an enabled rule covering the prefix expires objects after exactly this period. Objects already older than the period when the rule is first applied are expired on S3's next lifecycle run, typically within a day. Removing expire later deletes the lifecycle configuration, which fluid apply refuses without --allow-data-loss (it refuses every plan that removes a resource holding data or policy; removing an access grant or a policy tag is a revocation and is not gated); tofu destroy deletes it. GCP (fluid-schema 0.7.6): the BigQuery table is partitioned by day (timepartitioning type DAY, on binding.location.partitionBy when it names one date or timestamp column, else by ingestion time) with expirationms = retention in days, so BigQuery deletes each partition that long after its day ends. BigQuery counts from the partition's date, not from when its rows were written: by ingestion time (no partitionBy, the default and the S3 rule's semantics) no row is deleted sooner than retention after it landed; with partitionBy, retention is the age of the date in that column, so a backfill of rows whose date is already older than retention lands in expired partitions and is deleted at once. It is never a whole-table expiration, which would delete the product. BigQuery cannot partition a table that exists, so adding expire to a live table replaces it: fluid apply refuses that plan without --allow-data-loss, and the next build lands the data again. A later change of retention is an in-place update. Removing expire from a live table removes the partitioning's replacement trigger, which fluid apply also refuses without --allow-data-loss; BigQuery cannot un-partition a table, so to keep data longer set a longer retention instead. fluid verify checks the live table's partition expiration.

+ bucket
Type: enum (of string)
+

NEW in v0.7.6: Ownership of the object-store bucket (awss3bucket / googlestoragebucket).

+
-
-
- -
+
+

Must be one of:

+
  • "isolated"
  • "shared"
+
@@ -21019,18 +21617,18 @@

-
+
-
+

- +

-
+
Type: object
- No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: string
-

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

-
Same definition as window + database
Type: enum (of string)
+

NEW in v0.7.6: Ownership of the database container β€” covers Snowflake database AND AWS glue_database.

+
Same definition as bucket
-
+
-
+

- +

-
+
Type: string
- - - - - - - + containers + + + + dataset
Type: enum (of string)
+

NEW in v0.7.6: Ownership of the BigQuery dataset (googlebigquerydataset).

+
Same definition as bucket
-
+
-
+

- +

-
+
Type: string
-

Reference to replacement data product or expose.

-
- - - - - - + containers + + + + schema
Type: enum (of string)
+

NEW in v0.7.6: Ownership of the Snowflake schema (snowflake_schema).

+
Same definition as bucket
-
+
-
+

- +

-
+
Type: array of string
-

Deprecation policy tags.

-
Same definition as tags + containers + + + + warehouse
Type: enum (of string)
+

NEW in v0.7.6: Ownership of the Snowflake warehouse (snowflake_warehouse) β€” isolated gives per-product cost attribution.

+
Same definition as bucket
-
+
-
+

- +

-
+
Type: object
-

Deprecation policy labels.

-
Same definition as labels + containers + + + + cluster
Type: enum (of string)
+

NEW in v0.7.6: Ownership of the Confluent environment/cluster. v1 accepts 'shared' only β€” dedicated-cluster provisioning is not yet supported (the resolver rejects 'isolated').

+
Same definition as bucket
@@ -21364,18 +21936,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: array of string
-

Lifecycle tags for automation.

-
Same definition as tags -
-
-
-
-Type: object
+

NEW in v0.7.6: Encryption at rest for an AWS S3 binding or a GCP BigQuery binding. Absent: nothing is emitted, and the cloud's default applies: SSE-S3 for new S3 objects, Google-managed keys for BigQuery.

+
No Additional Properties + + + + + + +
-
+

- +

-
+
Type: object
-

Lifecycle labels for governance.

-
Same definition as labels -
-
-
-
-
+ encryption + + + + kms
Default: "product"
+
+

NEW in v0.7.6: The KMS key the bucket's objects are encrypted with (SSE-KMS with an S3 Bucket Key). 'product' (the default): fluid apply creates a customer managed key for each bucket this product owns (awskmskey, rotation on, alias alias/fluid/<contract id>/<bucket>) and makes it the bucket's default encryption (awss3bucketserversideencryptionconfiguration). Its key policy lets the account's IAM policies decide who may use it (the KMS default statement for the account root) and, when the binding carries governance.lakeFormation, lets the Lake Formation service-linked role AWSServiceRoleForLakeFormationDataAccess encrypt and decrypt, matched by aws:PrincipalArn so the role need not exist yet. A principal querying through Athena with credentials Lake Formation vends then needs no KMS permission of its own; a grantee in another account that the bucket policy lets read the objects directly (governance.lakeFormation.bucketPolicy) is also allowed kms:Decrypt, through S3 only. The writer (the identity the build runs as) needs kms:GenerateDataKey and kms:Decrypt from IAM. tofu destroy deletes the alias and schedules the key for deletion after 7 days, the minimum KMS allows; kms:CancelKeyDeletion recovers it until then. Dropping the block, or naming another key, removes the product key while the bucket keeps the objects encrypted under it, which become unreadable once the deletion window ends unless they are rewritten or the deletion is cancelled; fluid apply refuses that plan without --allow-data-loss. On a shared (pool) bucket the bucket's default encryption is the pool owner's, so 'product' is refused there. 'alias/<name>' or a key or alias ARN: an existing key, looked up at plan time (data awskmskey, so the identity running fluid apply needs kms:DescribeKey on it) and made the default encryption of a bucket this product owns by its key ARN, since S3 resolves an alias in the account of whoever writes; the plan fails unless the key is Enabled and a symmetric encryption key (SYMMETRICDEFAULT). On a pool bucket it is only checked. Its key policy is its owner's, and must let the Lake Formation role use it when the location is registered: an AWS managed key cannot be used with that role, so alias/aws/s3 is refused with registerLocation, and the plan fails for any key that is not customer managed. 'none': nothing is emitted. fluid verify checks that the key is Enabled and that the objects under the binding's prefix are SSE-KMS with it. On a gcp binding (fluid-schema 0.7.6, BigQuery tables): 'product' creates a Cloud KMS key ring and crypto key for each dataset this product writes (googlekmskeyring fluid-<contract id>-<dataset> and googlekmscryptokey 'bigquery', in the dataset's location, EU and US mapping to the europe and us key locations, rotated every 90 days), grants the project's BigQuery service agent roles/cloudkms.cryptoKeyEncrypterDecrypter on it, and makes it the dataset's defaultencryptionconfiguration and the table's encryptionconfiguration. The project must have the Cloud KMS API (cloudkms.googleapis.com) enabled. A key ring and key cannot be deleted on GCP: tofu destroy removes them from state and schedules the key's versions for destruction (30 days by default), and a later apply adopts the same names. 'projects/<p>/locations/<l>/keyRings/<r>/cryptoKeys/<k>': an existing key in the dataset's location, used as given; its owner grants the BigQuery service agent. 'none': Google-managed keys. An alias/... or arn:... value is refused on a gcp binding, and a Cloud KMS name on an aws binding. Adding a key to a table that exists replaces the table (BigQuery cannot re-key it in place), so fluid apply refuses that plan without --allow-data-loss; the next build lands the data again. fluid verify checks kmsKeyName on the dataset and the table.

+
-
-
-
-
-
-

- -

-
- -
-
+
+ +

+ +

+
+ Type: object
- - - No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: array of object
- + item 0
Type: enum (of string)
+
+

Must be one of:

+
  • "product"
  • "none"
+
- No Additional Items

Each item of this array must be:

-
-
+ +
Type: object
- No Additional Properties + kms + + + + anyOf + + + + item 1
Type: object
+Must match regular expression: ^alias/[A-Za-z0-9/_-]{1,250}$ -
-
-
-

- -

-
- -
-
+
+ Type: string
- + anyOf + + + + item 2
Type: object
+Must match regular expression: ^arn:aws[a-z-]*:kms:[a-z0-9-]+:[0-9]{12}:(key|alias)/[A-Za-z0-9/_-]{1,250}$ -
-
-
-
-
-
-
-

- -

-
- -
-
+
+ Type: string
+ anyOf + + + + item 3
Type: object
+Must match regular expression: ^projects/[a-z0-9.:-]{1,100}/locations/[a-z0-9-]{1,63}/keyRings/[A-Za-z0-9_-]{1,63}/cryptoKeys/[A-Za-z0-9_-]{1,63}$ + + + + +
+ @@ -21744,18 +22330,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

Human/machine-readable SLI check description

-
+ principals
Type: object
+
+

NEW in v0.7.6: Maps the contract's LOGICAL principals (accessPolicy.grants[].principal and exposes[].policy.authz.columnRestrictions[].principal, keyed exactly as the contract writes them) to the identities they stand for on this binding's platform, so the base contract stays cloud-neutral and each environment's overlay binds it. On gcp an identity is an IAM member (user:, group:, serviceAccount: or domain:); on aws an IAM principal ARN. A value is one identity or a list; [] says the principal has no identity on this cloud, and nothing is granted to it there. When the block is present, every principal the contract names for this expose must be a key: an unmapped one is refused, never emitted as written. Without the block the principals are used as written, except that a GCP principal in a reserved top-level domain (.example, .test, .invalid, .localhost) is refused as a placeholder. Modelled on ODCS v3's roles[] bound per server (servers[].roles) and on dbt grants resolved per target.

+ +
+
+ +
-
-
-
-
-
-
-
-
-
-
-
+
-
+

- +

-
-
+
+

Each additional property must conform to the following schema

Type: array of object
- - - - - - - No Additional Items

Each item of this array must be:

-
-
+

+ +

+
Type: object
- No Additional Properties - + additionalProperties + + + + anyOf + + + + item 0
Type: string
- -
-
-
-

- -

-
+

Must be at least 1 characters long

-
-
+ +
+ Type: enum (of string)
-
-

Must be one of:

-
  • "slack"
  • "email"
  • "webhook"
  • "pagerduty"
  • "custom"
-
+ anyOf + + + + item 1
Type: array of string
+ - -
-
-
-
-
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
+ anyOf + + + + item 1 + + + + item 1 items
Type: string
+ + + + +

Must be at least 1 characters long

+
+
+
+ @@ -22055,18 +22658,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: string
- + serialization
Type: object
+

NEW in v0.7.6: How message VALUES (and keys) are serialized on the wire. A Kafka binding today declares only format: kafka_topic, which tells a consumer nothing about whether the values are Avro or JSON β€” the one thing it has to know to read the topic at all. The first four values are the message formats ODCS v3.1.0 documents for its own Kafka server format field; the two -confluent values name those same wire formats carried through a Confluent schema registry, spelled as Apache Flink spells them.

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: stringFormat: uri
- + value
Type: enum (of string)
+

How message values are serialized.

+
+

Must be one of:

+
  • "json"
  • "avro"
  • "protobuf"
  • "xml"
  • "avro-confluent"
  • "json-confluent"
+
@@ -22191,18 +22778,18 @@

-
+
-
+

- +

-
+
Type: enum (of string)
-
+ key
Type: enum (of string)
+

How message keys are serialized, for a topic whose keys are not opaque bytes.

+

Must be one of:

-
  • "info"
  • "warning"
  • "critical"
+
  • "json"
  • "avro"
  • "protobuf"
  • "xml"
  • "avro-confluent"
  • "json-confluent"
@@ -22262,18 +22843,26 @@

Must be one of:

-
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: object
+

NEW in v0.7.6: An expose's lifecycle. The same fields as the contract-root lifecycle, plus expire, which is only meaningful where there is storage to expire.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of string
-

Alert configuration tags.

-
Same definition as tags + exposes items + + + + lifecycle + + + + state
Type: enum (of string)
+

Unified lifecycle state vocabulary.

+
+ +
+

Must be one of:

+
  • "preview"
  • "active"
  • "deprecated"
  • "retired"
+
+ + + + + +
-
+
-
+

- +

-
+
Type: string
+

How long this expose's data is kept (ISO-8601 duration, e.g. P30D, P7Y). A declaration unless expire is true. With expire: true on an AWS S3 binding, fluid apply expires the objects under the binding's prefix this long after they are written; on a GCP BigQuery table it deletes each daily partition this long after its day ends. Years and months are counted as 365 and 30 days, and a part of a day rounds up to a whole one.

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
-

Alert configuration labels.

-
Same definition as labels -
+ exposes items + + + + lifecycle + + + + expire
Type: boolean Default: false
+
+

NEW in v0.7.6: Delete stored data once it is older than retention. Default false, so a contract that declares a retention period does not start deleting data when forge-cli is upgraded. Honoured by the AWS emitter on an S3 binding and by the GCP emitter on a BigQuery table; on a gcp binding that is not a BigQuery table (GCS, Pub/Sub, Iceberg storage) it is refused rather than dropped. AWS: fluid apply writes one rule per expose into the bucket's awss3bucketlifecycleconfiguration, filtered to the binding's prefix (location.path, else {database}/{table}/; a binding with neither is refused, so no rule ever expires a whole bucket). The rule expires the current version retention after it is written, removes a noncurrent version (on a versioning-enabled bucket) one day after it stops being current, and aborts an incomplete multipart upload after 7 days or the retention period if shorter. The lifecycle configuration is AUTHORITATIVE for the whole bucket: S3 keeps one per bucket, so rules that fluid apply did not write are replaced. It is therefore written only for a bucket this product owns; on a shared (pool) bucket (packaging) nothing is written, the pool's owner must hold the rule, and fluid verify checks that an enabled rule covering the prefix expires objects after exactly this period. Objects already older than the period when the rule is first applied are expired on S3's next lifecycle run, typically within a day. Removing expire later deletes the lifecycle configuration, which fluid apply refuses without --allow-data-loss (it refuses every plan that removes a resource holding data or policy; removing an access grant or a policy tag is a revocation and is not gated); tofu destroy deletes it. GCP (fluid-schema 0.7.6): the BigQuery table is partitioned by day (timepartitioning type DAY, on binding.location.partitionBy when it names one date or timestamp column, else by ingestion time) with expirationms = retention in days, so BigQuery deletes each partition that long after its day ends. BigQuery counts from the partition's date, not from when its rows were written: by ingestion time (no partitionBy, the default and the S3 rule's semantics) no row is deleted sooner than retention after it landed; with partitionBy, retention is the age of the date in that column, so a backfill of rows whose date is already older than retention lands in expired partitions and is deleted at once. It is never a whole-table expiration, which would delete the product. BigQuery cannot partition a table that exists, so adding expire to a live table replaces it: fluid apply refuses that plan without --allow-data-loss, and the next build lands the data again. A later change of retention is an in-place update. Removing expire from a live table removes the partitioning's replacement trigger, which fluid apply also refuses without --allow-data-loss; BigQuery cannot un-partition a table, so to keep data longer set a longer retention instead. fluid verify checks the live table's partition expiration.

+
-
-
+
+
-
+ + + + + +
-
+
-
+

- +

-
+
Type: object
-

Default SLI trio (freshness, completeness, latency) by kind.

-
- - No Additional Properties + deprecationPolicy
Type: object
+ No Additional Properties -
+
-
+

- +

-
+
Type: boolean Default: true
-

Enable auto-generation of default SLIs.

-
- - - - - - + noticePeriod
Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window
-
+
-
+

- +

-
+
Type: object
-

Default freshness SLI configuration.

-
No Additional Properties + contact
Type: string
+ -
+
+
+
+
+
-
+

- +

-
+
Type: boolean Default: true
- + replacement
Type: string
+

Reference to replacement data product or expose.

+
@@ -22635,18 +23297,18 @@

-
+
-
+

- +

-
+
Type: string
-

Freshness threshold (auto-set by kind: table=PT6H, stream=PT1M, api=PT1S)

-
Same definition as window + tags
Type: array of string
+

Deprecation policy tags.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: enum (of string) Default: "warning"
-
-

Must be one of:

-
  • "info"
  • "warning"
  • "critical"
+ labels
Type: object
+

Deprecation policy labels.

+
Same definition as labels
- - - - - - +
+
+
-
+
-
+

- +

-
+
Type: array of string
-

Freshness SLI tags.

+ tags
Type: array of string
+

Lifecycle tags for automation.

Same definition as tags
-
+
-
+

- +

-
+
Type: object
-

Freshness SLI labels.

+ labels
Type: object
+

Lifecycle labels for governance.

Same definition as labels
@@ -22899,18 +23515,18 @@

-
+
-
+

- +

-
+
Type: object
-

Default completeness SLI configuration.

-
No Additional Properties + observability
Type: object
+ + + No Additional Properties -
+
-
+

- +

-
+
Type: boolean Default: true
+ metrics
Type: array of object
- -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: number Default: 0.95
-

Completeness threshold (auto-set by kind: table=0.95, stream=0.99)

-
+ metrics items
Type: object
+ No Additional Properties -

Value must be greater or equal to 0 and lesser or equal to 1

+ -
-
-
-
-
+
-
+

- +

-
+
Type: enum (of string) Default: "warning"
-
-

Must be one of:

-
  • "info"
  • "warning"
  • "critical"
-
+ name
Type: string
+ @@ -23165,18 +23725,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: array of string
-

Completeness SLI tags.

-
Same definition as tags + source
Type: string
+ + + + + + +
-
+
-
+

- +

-
+
Type: object
-

Completeness SLI labels.

-
Same definition as labels + sli
Type: string
+

Human/machine-readable SLI check description

+
+ + + + + +
+
+
-
+
-
+

- +

-
+
Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
-

Default latency SLI configuration.

-
No Additional Properties + exposes + + + + exposes items + + + + observability + + + + onBreach + + + + onBreach items
Type: object
+ No Additional Properties -
+
-
+

- +

-
+
Type: boolean Default: true
- + type
Type: enum (of string)
+
+

Must be one of:

+
  • "slack"
  • "email"
  • "webhook"
  • "pagerduty"
  • "custom"
+
@@ -23421,18 +24036,18 @@

-
+
-
+

- +

-
+
Type: string
-

P95 latency threshold (auto-set by kind: api=PT200MS, table=PT5S, stream=PT100MS)

-
Same definition as window + channel
Type: string
+ + + + + + +
-
+
-
+

- +

-
+
Type: number Default: 95
-

Latency percentile to measure.

-
+ target
Type: string
+ -

Value must be greater or equal to 0 and lesser or equal to 100

+
-
+
-
+

- +

-
+
Type: enum (of string) Default: "warning"
-
-

Must be one of:

-
  • "info"
  • "warning"
  • "critical"
-
+ url
Type: stringFormat: uri
+ @@ -23624,18 +24240,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: array of string
-

Latency SLI tags.

-
Same definition as tags + severity
Type: enum (of string)
+
+

Must be one of:

+
  • "info"
  • "warning"
  • "critical"
+
+ + + + + +
-
+
-
+

- +

-
+
Type: object
-

Latency SLI labels.

-
Same definition as labels -
-
-
-
+ tags
Type: array of string
+

Alert configuration tags.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: array of string
-

Default SLI configuration tags.

-
Same definition as tags -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

Default SLI configuration labels.

+ labels
Type: object
+

Alert configuration labels.

Same definition as labels
+
+
-
+
-
+

- +

-
+
Type: object
-

Alert dispatcher config consumed by the acquisition Alerter (DLQ overflow, schema-fingerprint changes, anomaly signals).

-
No Additional Properties + defaultSLIs
Type: object
+

Default SLI trio (freshness, completeness, latency) by kind.

+
+ + No Additional Properties -
+
-
+

- +

-
+
Type: array of object
- + enabled
Type: boolean Default: true
+

Enable auto-generation of default SLIs.

+
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
- No Additional Properties + freshness
Type: object
+

Default freshness SLI configuration.

+
No Additional Properties -
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "log"
  • "file"
  • "webhook"
-
+ enabled
Type: boolean Default: true
+ @@ -24110,18 +24684,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: string
-

For kind=file: NDJSON sink path (env-vars expanded).

-
- - - - - - + threshold
Type: string
+

Freshness threshold (auto-set by kind: table=PT6H, stream=PT1M, api=PT1S)

+
Same definition as window
-
+
-
+

- +

-
+
Type: stringFormat: uri
-

For kind=webhook: Slack-compatible incoming webhook URL. Validated against the SSRF guard at construction time.

-
+ severity
Type: enum (of string) Default: "warning"
+
+

Must be one of:

+
  • "info"
  • "warning"
  • "critical"
+
@@ -24262,28 +24818,18 @@

-
-
-
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: array of string
-

Observability configuration tags.

+ defaultSLIs + + + + freshness + + + + tags
Type: array of string
+

Freshness SLI tags.

Same definition as tags
-
+
-
+

- +

-
+
Type: object
-

Observability configuration labels.

+ defaultSLIs + + + + freshness + + + + labels
Type: object
+

Freshness SLI labels.

Same definition as labels
@@ -24374,18 +24948,18 @@

-
+
-
+

- +

-
+
Type: object
- - - No Additional Properties + observability + + + + defaultSLIs + + + + completeness
Type: object
+

Default completeness SLI configuration.

+
No Additional Properties -
+
-
+

- +

-
+
Type: stringFormat: uri
+ defaultSLIs + + + + completeness + + + + enabled
Type: boolean Default: true
@@ -24473,18 +25074,18 @@

-
+
-
+

- +

-
+
Type: stringFormat: uri
- + defaultSLIs + + + + completeness + + + + threshold
Type: number Default: 0.95
+

Completeness threshold (auto-set by kind: table=0.95, stream=0.99)

+
- +

Value must be greater or equal to 0 and lesser or equal to 1

-
+
-
+

- +

-
+
Type: stringFormat: uri
- + defaultSLIs + + + + completeness + + + + severity
Type: enum (of string) Default: "warning"
+
+

Must be one of:

+
  • "info"
  • "warning"
  • "critical"
+
@@ -24581,18 +25214,18 @@

-
+
-
+

- +

-
+
Type: stringFormat: uri
- - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: array of string
-

Documentation tags for organization.

+ tags
Type: array of string
+

Completeness SLI tags.

Same definition as tags
-
+
-
+

- +

-
+
Type: object
-

Documentation labels for automation.

+ defaultSLIs + + + + completeness + + + + labels
Type: object
+

Completeness SLI labels.

Same definition as labels
@@ -24737,18 +25344,18 @@

-
+
-
+

- +

-
+
Type: object
-

NEW in v0.7.2: Semantic model definition mapping physical columns to business concepts. Defines entities, measures, dimensions, and metrics for agent-consumable business logic. Eliminates semantic hallucination by providing machine-readable KPI definitions. Compatible with dbt MetricFlow, Snowflake Semantic Views, and OSI-aligned formats.

-
- - No Additional Properties + observability + + + + defaultSLIs + + + + latency
Type: object
+

Default latency SLI configuration.

+
No Additional Properties -
-
Example:
-
{
-    "name": "orders_revenue_model",
-    "description": "Canonical semantic model for revenue analytics",
-    "defaultAggTimeDimension": "order_date",
-    "entities": [
-        {
-            "name": "order",
-            "type": "primary",
-            "expr": "order_id"
-        },
-        {
-            "name": "customer",
-            "type": "foreign",
-            "expr": "customer_id"
-        }
-    ],
-    "measures": [
-        {
-            "name": "order_amount",
-            "agg": "sum",
-            "expr": "amount",
-            "description": "Sum of all order amounts"
-        },
-        {
-            "name": "order_count",
-            "agg": "count_distinct",
-            "expr": "order_id",
-            "createMetric": true
-        }
-    ],
-    "dimensions": [
-        {
-            "name": "order_date",
-            "type": "time",
-            "expr": "completed_at",
-            "typeParams": {
-                "timeGranularity": "day"
-            }
-        },
-        {
-            "name": "region",
-            "type": "categorical"
-        }
-    ],
-    "metrics": [
-        {
-            "name": "net_revenue",
-            "description": "GAAP-compliant net revenue: completed orders minus discounts, excluding refunds",
-            "type": "derived",
-            "inputMetrics": [
-                "gross_revenue",
-                "total_discounts"
-            ],
-            "expr": "gross_revenue - total_discounts",
-            "owner": "finance-data@company.com"
-        },
-        {
-            "name": "gross_revenue",
-            "description": "Total value of completed orders",
-            "type": "simple",
-            "measure": "order_amount",
-            "filter": "status = 'completed'"
-        }
-    ]
-}
-
-
-
+ +
-
+

- +

-
+
Type: string
-

Human-readable name for this semantic model.

-
+ defaultSLIs + + + + latency + + + + enabled
Type: boolean Default: true
+ @@ -24906,18 +25470,18 @@

-
+
-
+

- +

-
+
Type: string
-

Business context explaining what this semantic model represents.

-
- - - - - - + defaultSLIs + + + + latency + + + + threshold
Type: string
+

P95 latency threshold (auto-set by kind: api=PT200MS, table=PT5S, stream=PT100MS)

+
Same definition as window
-
+
-
+

- +

-
+
Type: string
-

Default time dimension used for measure aggregation. Can be overridden per-measure.

+ defaultSLIs + + + + latency + + + + percentile
Type: number Default: 95
+

Latency percentile to measure.

- +

Value must be greater or equal to 0 and lesser or equal to 100

-
+
-
+

- +

-
+
Type: array of object
-

Join keys with type annotations. Entities connect this table to others in the semantic graph.

-
- - - - - - No Additional Items

Each item of this array must be:

-
-
- - - Type: object
- No Additional Properties + severity
Type: enum (of string) Default: "warning"
+
+

Must be one of:

+
  • "info"
  • "warning"
  • "critical"
+
-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

Entity name (e.g., 'order', 'customer').

-
- - - - - - + tags
Type: array of string
+

Latency SLI tags.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: enum (of string)
-

Entity key type: primary (unique, complete), foreign (references another table), unique (unique, possibly partial), natural (real-world identifier).

-
-

Must be one of:

-
  • "primary"
  • "foreign"
  • "unique"
  • "natural"
+ labels
Type: object
+

Latency SLI labels.

+
Same definition as labels
- - - - - - +
+
+
-
+
-
+

- +

-
+
Type: string
-

Column expression. Defaults to entity name if omitted.

-
- - - - - - + tags
Type: array of string
+

Default SLI configuration tags.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: string
- - - - - - - + labels
Type: object
+

Default SLI configuration labels.

+
Same definition as labels
-
-
-
+
-
+

- +

-
+
Type: array of object
-

Aggregatable column expressions. Measures are the atomic building blocks from which metrics are composed.

-
+ alert
Type: object
+

Alert dispatcher config consumed by the acquisition Alerter (DLQ overflow, schema-fingerprint changes, anomaly signals).

+
No Additional Properties - No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: object
- No Additional Properties + channels
Type: array of object
+ - -
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

Measure name (e.g., 'totalamount', 'ordercount').

-
+ channels items
Type: object
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
- + channels items + + + + kind
Type: enum (of string)
+
+

Must be one of:

+
  • "log"
  • "file"
  • "webhook"
+
@@ -25633,18 +26159,18 @@

-
+
-
+

- +

-
+
Type: enum (of string)
-

Aggregation function applied to the expression.

-
-

Must be one of:

-
  • "sum"
  • "avg"
  • "count"
  • "count_distinct"
  • "min"
  • "max"
  • "median"
  • "percentile"
-
+ channels items + + + + path
Type: string
+

For kind=file: NDJSON sink path (env-vars expanded).

+
@@ -25705,18 +26235,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: string
-

SQL expression to aggregate. Defaults to measure name if omitted.

+ channels items + + + + url
Type: stringFormat: uri
+

For kind=webhook: Slack-compatible incoming webhook URL. Validated against the SSRF guard at construction time.

@@ -25774,18 +26311,28 @@

-
+
+
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: object
-

Aggregation parameters (mirrors dbt-semantic-interfaces agg_params). Currently used by agg=percentile.

-
No Additional Properties - - - - - - -
+ tags
Type: array of string
+

Observability configuration tags.

+
Same definition as tags +
+
+
+
+
-
+

- +

-
+
Type: object
+

Observability configuration labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
-

Percentile in [0, 1] for agg=percentile (0.5 = median). Consumers default to 0.5 when omitted.

-
+ docs
Type: object
+ + + No Additional Properties -

Value must be greater or equal to 0 and lesser or equal to 1

+ -
-
-
-
-
+
-
+

- +

-
+
Type: boolean
-

Use the discrete percentile (PERCENTILEDISC / usediscrete_percentile) instead of continuous interpolation.

-
+ homepage
Type: stringFormat: uri
+ @@ -25991,22 +26522,18 @@

-
-
-
-
-
+
-
+

- +

-
+
Type: string
-

Override the model-level default time dimension for this measure.

-
+ runbook
Type: stringFormat: uri
+ @@ -26064,18 +26576,18 @@

-
+
-
+

- +

-
+
Type: object
-

Dimension over which this measure cannot be naively aggregated (e.g., account balances).

-
No Additional Properties + dictionary
Type: stringFormat: uri
+ -
+
+
+
+
+
-
+

- +

-
+
Type: string
+ changeLog
Type: stringFormat: uri
@@ -26204,18 +26684,18 @@

-
+
-
+

- +

-
+
Type: enum (of string)
-

Which value to pick when the non-additive dimension is present.

-
-

Must be one of:

-
  • "min"
  • "max"
-
- - - - - - -
-
-
-
+ tags
Type: array of string
+

Documentation tags for organization.

+
Same definition as tags
-
+
-
+

- +

-
+
Type: object
+

Documentation labels for automation.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: false
-

If true, automatically create a simple metric from this measure.

+ exposes items + + + + semantics
Type: object
+

NEW in v0.7.2: Semantic model definition mapping physical columns to business concepts. Defines entities, measures, dimensions, and metrics for agent-consumable business logic. Eliminates semantic hallucination by providing machine-readable KPI definitions. Compatible with dbt MetricFlow, Snowflake Semantic Views, and OSI-aligned formats.

+ + No Additional Properties - -
-
-
-
-
-
-
-
-
+
+
Example:
+
{
+    "name": "orders_revenue_model",
+    "description": "Canonical semantic model for revenue analytics",
+    "defaultAggTimeDimension": "order_date",
+    "entities": [
+        {
+            "name": "order",
+            "type": "primary",
+            "expr": "order_id"
+        },
+        {
+            "name": "customer",
+            "type": "foreign",
+            "expr": "customer_id"
+        }
+    ],
+    "measures": [
+        {
+            "name": "order_amount",
+            "agg": "sum",
+            "expr": "amount",
+            "description": "Sum of all order amounts"
+        },
+        {
+            "name": "order_count",
+            "agg": "count_distinct",
+            "expr": "order_id",
+            "createMetric": true
+        }
+    ],
+    "dimensions": [
+        {
+            "name": "order_date",
+            "type": "time",
+            "expr": "completed_at",
+            "typeParams": {
+                "timeGranularity": "day"
+            }
+        },
+        {
+            "name": "region",
+            "type": "categorical"
+        }
+    ],
+    "metrics": [
+        {
+            "name": "net_revenue",
+            "description": "GAAP-compliant net revenue: completed orders minus discounts, excluding refunds",
+            "type": "derived",
+            "inputMetrics": [
+                "gross_revenue",
+                "total_discounts"
+            ],
+            "expr": "gross_revenue - total_discounts",
+            "owner": "finance-data@company.com"
+        },
+        {
+            "name": "gross_revenue",
+            "description": "Total value of completed orders",
+            "type": "simple",
+            "measure": "order_amount",
+            "filter": "status = 'completed'"
+        }
+    ]
+}
+
-
+
-
+

- +

-
+
Type: array of object
-

Categorical and time-based grouping axes. Dimensions define how data can be sliced in queries.

+ name
Type: string
+

Human-readable name for this semantic model.

- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
- No Additional Properties + description
Type: string
+

Business context explaining what this semantic model represents.

+
-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

Dimension name (e.g., 'orderdate', 'region', 'productcategory').

+ defaultAggTimeDimension
Type: string
+

Default time dimension used for measure aggregation. Can be overridden per-measure.

@@ -26529,18 +27065,18 @@

-
+
-
+

- +

-
+
Type: array of object
+

Join keys with type annotations. Entities connect this table to others in the semantic graph.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: enum (of string)
-

Dimension type: categorical (text/enum grouping) or time (temporal axis).

-
-

Must be one of:

-
  • "categorical"
  • "time"
-
+ exposes items + + + + semantics + + + + entities + + + + entities items
Type: object
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

SQL expression. Defaults to dimension name if omitted.

+ name
Type: string
+

Entity name (e.g., 'order', 'customer').

@@ -26670,18 +27232,18 @@

-
+
-
+

- +

-
+
Type: string
- + type
Type: enum (of string)
+

Entity key type: primary (unique, complete), foreign (references another table), unique (unique, possibly partial), natural (real-world identifier).

+
+

Must be one of:

+
  • "primary"
  • "foreign"
  • "unique"
  • "natural"
+
@@ -26738,18 +27304,18 @@

-
+
-
+

- +

-
+
Type: object
-

Type-specific parameters.

-
No Additional Properties + expr
Type: string
+

Column expression. Defaults to entity name if omitted.

+
-
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-

Default time granularity for this dimension.

-
-

Must be one of:

-
  • "day"
  • "week"
  • "month"
  • "quarter"
  • "year"
  • "hour"
  • "minute"
-
+ description
Type: string
+ -
-
-
-
@@ -26892,18 +27447,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: array of object
-

Composable KPI definitions built from measures. Metrics are the named business calculations that agents and tools resolve.

+ measures
Type: array of object
+

Aggregatable column expressions. Measures are the atomic building blocks from which metrics are composed.

@@ -26944,7 +27499,7 @@

No Additional Items

Each item of this array must be:

-
+
Type: object
+ measures items
Type: object
No Additional Properties @@ -26990,18 +27545,18 @@

-
+
-
+

- +

-
+
Type: string
-

Metric name (e.g., 'netrevenue', 'customerchurn_rate').

+ name
Type: string
+

Measure name (e.g., 'totalamount', 'ordercount').

@@ -27059,18 +27614,18 @@

-
+
-
+

- +

-
+
Type: string
-

Business definition of this metric.

-
+ description
Type: string
+ @@ -27128,18 +27682,18 @@

-
+
-
+

- +

-
+
Type: enum (of string)
-

Metric type: simple (single measure + filter), derived (expression over other metrics), ratio (numerator/denominator).

-
+ agg
Type: enum (of string)
+

Aggregation function applied to the expression.

+

Must be one of:

-
  • "simple"
  • "derived"
  • "ratio"
+
  • "sum"
  • "avg"
  • "count"
  • "count_distinct"
  • "min"
  • "max"
  • "median"
  • "percentile"
@@ -27200,18 +27754,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: string
-

Reference to a measure name (for simple metrics).

+ expr
Type: string
+

SQL expression to aggregate. Defaults to measure name if omitted.

@@ -27269,18 +27823,18 @@

-
+
-
+

- +

-
+
Type: string
-

SQL WHERE clause filter (e.g., "status = 'completed' AND is_refunded = FALSE").

-
+ aggParams
Type: object
+

Aggregation parameters (mirrors dbt-semantic-interfaces agg_params). Currently used by agg=percentile.

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: array of string
-

References to other metric names (for derived/ratio metrics).

+ aggParams + + + + percentile
Type: number
+

Percentile in [0, 1] for agg=percentile (0.5 = median). Consumers default to 0.5 when omitted.

- +

Value must be greater or equal to 0 and lesser or equal to 1

- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
- + useDiscretePercentile
Type: boolean
+

Use the discrete percentile (PERCENTILEDISC / usediscrete_percentile) instead of continuous interpolation.

+
+
+
-
+
-
+

- +

-
+
Type: string
-

Mathematical expression combining input metrics (for derived metrics).

+ aggTimeDimension
Type: string
+

Override the model-level default time dimension for this measure.

@@ -27539,18 +28113,18 @@

-
+
-
+

- +

-
+
Type: string
-

Numerator metric name (for ratio metrics).

-
+ nonAdditiveDimension
Type: object
+

Dimension over which this measure cannot be naively aggregated (e.g., account balances).

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
-

Denominator metric name (for ratio metrics).

-
+ nonAdditiveDimension + + + + name
Type: string
+ @@ -27677,18 +28253,18 @@

-
+
-
+

- +

-
+
Type: string
-

Business owner of this metric definition.

-
+ nonAdditiveDimension + + + + windowChoice
Type: enum (of string)
+

Which value to pick when the non-additive dimension is present.

+
+

Must be one of:

+
  • "min"
  • "max"
+
@@ -27746,24 +28332,22 @@

-
-
-
+
-
+

- +

-
+
Type: array of string
-

Semantic model tags for discovery and categorization.

-
Same definition as tags + measures + + + + measures items + + + + createMetric
Type: boolean Default: false
+

If true, automatically create a simple metric from this measure.

+
+ + + + + +
-
+
+
+
+
+
+
+
-
+

- +

-
+
Type: object
-

Semantic model labels for automation and metadata.

-
Same definition as labels -
-
-
-
-
-
-
-
-
-
-
-

- -

-
+ dimensions
Type: array of object
+

Categorical and time-based grouping axes. Dimensions define how data can be sliced in queries.

+
+ -
-
+ + + + No Additional Items

Each item of this array must be:

+
+
+ Type: object
-

AWS Glue crawler configuration for auto-discovery of schema from S3 paths.

-
+ semantics + + + + dimensions + + + + dimensions items
Type: object
+ No Additional Properties -
+
-
+

- +

-
+
Type: string
-

Glue crawler name.

+ dimensions + + + + dimensions items + + + + name
Type: string
+

Dimension name (e.g., 'orderdate', 'region', 'productcategory').

@@ -27953,18 +28578,18 @@

-
+
-
+

- +

-
+
Type: string
-

IAM role ARN for the crawler.

-
+ dimensions + + + + dimensions items + + + + type
Type: enum (of string)
+

Dimension type: categorical (text/enum grouping) or time (temporal axis).

+
+

Must be one of:

+
  • "categorical"
  • "time"
+
@@ -28008,18 +28650,18 @@

-
+
-
+

- +

-
+
Type: string
-

Cron schedule expression (e.g., 'cron(0 6 * * ? *)').

+ dimensions + + + + dimensions items + + + + expr
Type: string
+

SQL expression. Defaults to dimension name if omitted.

@@ -28063,18 +28719,18 @@

-
+
-
+

- +

-
+
Type: array of string
-

Custom classifier names to apply.

-
- - - - - - No Additional Items

Each item of this array must be:

-
- Type: string
@@ -28161,24 +28783,22 @@

-

-
-
+
-
+

- +

-
+
Type: object
-

Policy for schema changes detected by the crawler.

-
+ dimensions + + + + dimensions items + + + + typeParams
Type: object
+

Type-specific parameters.

+
No Additional Properties -
+
-
+

- +

-
+
Type: enum (of string)
-Type: enum (of string)
+

Default time granularity for this dimension.

+

Must be one of:

-
  • "UPDATE_IN_DATABASE"
  • "LOG"
+
  • "day"
  • "week"
  • "month"
  • "quarter"
  • "year"
  • "hour"
  • "minute"
@@ -28282,18 +28931,28 @@

Must be one of:

-
+
+
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "LOG"
  • "DELETE_FROM_DATABASE"
  • "DEPRECATE_IN_DATABASE"
-
+ metrics
Type: array of object
+

Composable KPI definitions built from measures. Metrics are the named business calculations that agents and tools resolve.

+
- -
-
-
-
-
-
-
-
-
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: object
-

AWS Glue Iceberg table management configuration (snapshots, compaction).

-
+ semantics + + + + metrics + + + + metrics items
Type: object
+ No Additional Properties -
+
-
+

- +

-
+
Type: enum (of string)
-

Underlying file format for Iceberg data files.

-
-

Must be one of:

-
  • "parquet"
  • "orc"
  • "avro"
-
+ metrics + + + + metrics items + + + + name
Type: string
+

Metric name (e.g., 'netrevenue', 'customerchurn_rate').

+
@@ -28456,18 +29108,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: object
-

Snapshot retention policy.

+ metrics + + + + metrics items + + + + description
Type: string
+

Business definition of this metric.

@@ -28507,18 +29173,22 @@

-
+
+

+
+
+
-
+

- +

-
+
Type: integer
-

Maximum age of snapshots in milliseconds.

-
+ metrics items + + + + type
Type: enum (of string)
+

Metric type: simple (single measure + filter), derived (expression over other metrics), ratio (numerator/denominator).

+
+

Must be one of:

+
  • "simple"
  • "derived"
  • "ratio"
+
@@ -28569,18 +29249,18 @@

-
+
-
+

- +

-
+
Type: integer
-

Minimum number of snapshots to retain.

+ metrics items + + + + measure
Type: string
+

Reference to a measure name (for simple metrics).

@@ -28631,22 +29318,18 @@

-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Iceberg compaction settings.

+ metrics + + + + metrics items + + + + filter
Type: string
+

SQL WHERE clause filter (e.g., "status = 'completed' AND is_refunded = FALSE").

@@ -28686,18 +29383,22 @@

-
+
+

+
+
+
-
+

- +

-
+
Type: boolean
- + metrics items + + + + inputMetrics
Type: array of string
+

References to other metric names (for derived/ratio metrics).

+
- -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: integer
-

Target file size in MB after compaction.

-
+ metrics items + + + + inputMetrics + + + + inputMetrics items
Type: string
+ -
-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

NEW in v0.7.4: Declares this expose as agent-consumable over MCP, with per-expose overrides for the consumer-side output-port server (fluid mcp output-port serve). Its presence opts the expose into the Fluid MCP gateway, where policy.agentPolicy.allowedModels / allowedUseCases are enforced at every read.

-
- - No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: object
- No Additional Properties + expr
Type: string
+

Mathematical expression combining input metrics (for derived metrics).

+
-
+
+
+
+
+
-
+

- +

-
+
Type: integer
-

Hard cap for the sample MCP tool. Defends against an over-curious agent asking for petabyte-scale row counts. Default: 100.

+ metrics items + + + + numerator
Type: string
+

Numerator metric name (for ratio metrics).

-

Value must be greater or equal to 1

+
-
-
-
-
-
+
-
+

- +

-
+
Type: object
- No Additional Properties + metrics + + + + metrics items + + + + denominator
Type: string
+

Denominator metric name (for ratio metrics).

+
-
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-

Surfaced on the describe tool so consumer agents can declare downstream handling. Advisory; the real gate is policy.agentPolicy.

-
-

Must be one of:

-
  • "public"
  • "internal"
  • "confidential"
  • "restricted"
-
+ metrics items + + + + owner
Type: string
+

Business owner of this metric definition.

+
-
-
-
-
-
-
-
-
@@ -29108,18 +29801,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: array
- - - - - - - No Additional Items

Each item of this array must be:

-
-
- - - Type: object
- - - No Additional Properties - - - - - - -
+ semantics + + + + tags
Type: array of string
+

Semantic model tags for discovery and categorization.

+
Same definition as tags +
+
+
+
+
-
+

- +

-
+
Type: string
-

Upstream data product identifier (e.g., 'silver.hr.people_v2'). Renamed from 'provider' for clarity.

-
Same definition as id + semantics + + + + labels
Type: object
+

Semantic model labels for automation and metadata.

+
Same definition as labels
-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

The upstream exposeId being consumed.

-
Same definition as id -
-
-
-
-Type: object
+

AWS Glue crawler configuration for auto-discovery of schema from S3 paths.

+
+ + + + + + +
-
+

- +

-
+
Type: string
-

Semver range (e.g., ^1.2, ~1.4, >=1.0.0). Not strictly validated by regex; tooling enforces.

+ crawler + + + + name
Type: string
+

Glue crawler name.

- - @@ -29299,18 +30002,18 @@

-
+
-
+

- +

-
+
Type: object
- No Additional Properties + crawler + + + + role
Type: string
+

IAM role ARN for the crawler.

+
-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

-
Same definition as window + schedule
Type: string
+

Cron schedule expression (e.g., 'cron(0 6 * * ? *)').

+
+ + + + + +
-
+
-
+

- +

-
+
Type: string
-

Maximum acceptable staleness for consumed data.

-
Same definition as window -
-
-
-
-
-
-
-

- -

-
+ classifiers
Type: array of string
+

Custom classifier names to apply.

+
+ -
-
+ + + + No Additional Items

Each item of this array must be:

+
+
+ Type: number
-

Minimum completeness expectation.

-
+ classifiers + + + + classifiers items
Type: string
+ -

Value must be greater or equal to 0 and lesser or equal to 1

+ +
+
-
+
-
+

- +

-
+
Type: array of string
-

QoS expectation tags.

-
Same definition as tags -
-
-
-
-Type: object
+

Policy for schema changes detected by the crawler.

+
+ + + + + + +
-
+

- +

-
+
Type: object
-

QoS expectation labels.

-
Same definition as labels + schemaChangePolicy + + + + updateBehavior
Type: enum (of string)
+
+

Must be one of:

+
  • "UPDATE_IN_DATABASE"
  • "LOG"
-
-
-
+ + + + + +
-
+
-
+

- +

-
+
Type: array of string
- - - - - - - No Additional Items

Each item of this array must be:

-
- Type: enum (of string)
+
+

Must be one of:

+
  • "LOG"
  • "DELETE_FROM_DATABASE"
  • "DEPRECATE_IN_DATABASE"
+
+
+
-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

Business purpose for consuming this data.

+ iceberg
Type: object
+

AWS Glue Iceberg table management configuration (snapshots, compaction).

@@ -29730,22 +30447,18 @@

-

-
-
-
-
+
-
+

- +

-
+
Type: array of string
-

Consumption relationship tags.

-
Same definition as tags + iceberg + + + + writeFormat
Type: enum (of string)
+

Underlying file format for Iceberg data files.

+
+

Must be one of:

+
  • "parquet"
  • "orc"
  • "avro"
+
+ + + + + +
-
+
-
+

- +

-
+
Type: object
-

Consumption relationship labels.

-
Same definition as labels -
-
-
-
-Type: object
+

Snapshot retention policy.

+
+ + + + + + +
-
+

- +

-
+
Type: string
-

Cross-mesh only. Id of the federated workspace that owns this upstream, as declared in federation/upstreams.yaml. Set it when the upstream lives in another mesh rather than this workspace; omit it for a local upstream, which is resolved by walking the workspace.

+ iceberg + + + + snapshotRetention + + + + maxSnapshotAgeMs
Type: integer
+

Maximum age of snapshots in milliseconds.

-

Must be at least 1 characters long

+
-
+
-
+

- +

-
+
Type: string
-

Cross-mesh only. The contract digest this product was composed against, as sha256:<64 hex>. fluid apply re-fetches the upstream's live digest and compares: a mismatch means the upstream changed since composition. Required whenever upstreamWorkspace is set.

-
Must match regular expression: ^sha256:[0-9a-f]{64}$ + iceberg + + + + snapshotRetention + + + + minSnapshotsToKeep
Type: integer
+

Minimum number of snapshots to retain.

+
@@ -29914,24 +30680,22 @@

-
-
-
+
-
+

- +

-
+
Type: array
-

NEW in v0.5.5: Multiple build configurations for multi-modal data products (e.g., batch + streaming, SQL + ML).

-
- - - - - - No Additional Items

Each item of this array must be:

-
-
- - - Type: object
-

Enhanced build configuration supporting v0.4.0 patterns with v0.5.5 improvements.

+ iceberg + + + + compaction
Type: object
+

Iceberg compaction settings.

+ - No Additional Properties

- -

-
- + + + + +
+
+
+

+ +

+
+ +
+
Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

-
Type: boolean
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Target file size in MB after compaction.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.4: Declares this expose as agent-consumable over MCP, with per-expose overrides for the consumer-side output-port server (fluid mcp output-port serve). Its presence opts the expose into the Fluid MCP gateway, where policy.agentPolicy.allowedModels / allowedUseCases are enforced at every read.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Hard cap for the sample MCP tool. Defends against an over-curious agent asking for petabyte-scale row counts. Default: 100.

+
+ + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Surfaced on the describe tool so consumer agents can declare downstream handling. Advisory; the real gate is policy.agentPolicy.

+
+

Must be one of:

+
  • "public"
  • "internal"
  • "confidential"
  • "restricted"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Upstream data product identifier (e.g., 'silver.hr.people_v2'). Renamed from 'provider' for clarity.

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

The upstream exposeId being consumed.

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.6: The upstream contract's OWN id, for an upstream that has one of its own β€” a product not built with FLUID, registered in a catalog under an id FLUID did not derive. The canonical-port builder already prefers an explicit contractId over the derived <productId>.<exposeId>; until now consumes[] was closed, so there was no way to supply it and the derived id was exported as if the upstream answered to it.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Semver range (e.g., ^1.2, ~1.4, >=1.0.0). Not strictly validated by regex; tooling enforces.

+
+ + + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+
+

NEW in v0.7.6: The exact upstream contract version this product was built and tested against, within versionConstraint. Exported as the ODPS inputPorts[].version, and as the version of the matching outputPorts[].inputContracts[] entry. Without it the export derives the range's lower bound; for a range it cannot read it falls back to 1.0.0, because ODPS v1.0.0 requires a version on every input port and the field cannot simply be omitted. versionConstraint stays the range this product accepts; this is the single version its exported lineage claims. fluid validate does not check that it falls inside versionConstraint.

+ +
+
+ +
+ + Must match regular expression: ^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$ + + + + + +
+
Example:
+
"2.1.0"
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Maximum acceptable staleness for consumed data.

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
+

Minimum completeness expectation.

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

QoS expectation tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

QoS expectation labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business purpose for consuming this data.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Consumption relationship tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Consumption relationship labels.

+
Same definition as labels +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Cross-mesh only. Id of the federated workspace that owns this upstream, as declared in federation/upstreams.yaml. Set it when the upstream lives in another mesh rather than this workspace; omit it for a local upstream, which is resolved by walking the workspace.

+
+ + + +

Must be at least 1 characters long

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Cross-mesh only. The contract digest this product was composed against, as sha256:<64 hex>. fluid apply re-fetches the upstream's live digest and compares: a mismatch means the upstream changed since composition. Required whenever upstreamWorkspace is set.

+
Must match regular expression: ^sha256:[0-9a-f]{64}$ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

NEW in v0.5.5: Multiple build configurations for multi-modal data products (e.g., batch + streaming, SQL + ML).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+

Enhanced build configuration supporting v0.4.0 patterns with v0.5.5 improvements.

+
+ + No Additional Properties

+ +

+
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "hybrid-reference" + + + + + + +
+
+
+
+
+ + + +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Hybrid-reference pattern from v0.4.0 (dbt-style).

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

dbt profile target to run against. Forwarded to dbt as --target <name> by the build runner (allowlisted against the profile's real targets; falls back to the profile default when the requested target is unavailable).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+

dbt node selector(s) forwarded to dbt --select. Accepts a single selector string or an array of selector strings; the build runner drops flag-like values (those starting with '-').

+

+ +

+
+ + + Type: string
+ -
+ + + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+

Legacy alias for select (dbt's older --models flag). Same string-or-array shape; select takes precedence when both are set.

+

+ +

+
+ + + Type: string
+ + + + + + + +
+ + + Type: array of string
+ + -
- -
+ + + No Additional Items

Each item of this array must be:

+
+
+ Type: object
+ item 1 items
Type: string
@@ -30100,18 +33228,31 @@

-
+
+

+
+ + + + + + +
+
+
+
+
-
+

- +

-
+
Type: const
-Specific value: "hybrid-reference" + properties + + + + vars
Type: object
+ -
+
+
+
+

+ +

-
-
-
- - -
- +
+

Additional Properties of any type are allowed.

Type: object
+ then + + + + properties + + + + vars + + + + additionalProperties
Type: object
@@ -30219,18 +33390,26 @@

-
+
+

+
+
+
+
+
+
+
-
+

- +

-
+
Type: object
-

Hybrid-reference pattern from v0.4.0 (dbt-style).

-
+ properties + + + + materializations
Type: object
- No Additional Properties -
+
-
+

- +

-
-
+
+

Each additional property must conform to the following schema

Type: string
- + materializations + + + + additionalProperties
Type: enum (of string)
+
+

Must be one of:

+
  • "table"
  • "view"
  • "incremental"
  • "ephemeral"
+
@@ -30361,18 +33554,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: string
-

dbt profile target to run against. Forwarded to dbt as --target <name> by the build runner (allowlisted against the profile's real targets; falls back to the profile default when the requested target is unavailable).

-
- - - - - - + tags
Type: array of string
+

Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as tags
-
+
-
+

- +

-
+

-

dbt node selector(s) forwarded to dbt --select. Accepts a single selector string or an array of selector strings; the build runner drops flag-like values (those starting with '-').

-

- -

-
Type: object
+

Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+ + + + + + +
Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: string
+ if
Type: object
@@ -30590,9 +33814,19 @@

-

- +
+
+
+

+ +

+
+ +
+
Type: array of string
- + pattern
Type: const
+Specific value: "embedded-logic" - No Additional Items

Each item of this array must be:

-
-
+ +
+
+
+
+
+ + + +
Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
- - - - - + then + + + + properties
Type: object
+

Embedded logic pattern from v0.4.0.

+
- -
-
-
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+

-

Legacy alias for select (dbt's older --models flag). Same string-or-array shape; select takes precedence when both are set.

-

- -

-
- + sql
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: string
- + language
Type: enum (of string)
+
+

Must be one of:

+
  • "sql"
  • "flink_sql"
  • "pyspark"
  • "scala"
  • "python"
  • "r"
+
-
- +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: array of string
+ parameters
Type: object
- No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

Type: string
+ additionalProperties
Type: object
@@ -31070,31 +34302,26 @@

+

-
- - - - - - +
-
+
-
+

- +

-
+
Type: object
- - - - - - - -
+ tags
Type: array of string
+

Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as tags +
+
+
+
+
-
+

- +

-
-

Additional Properties of any type are allowed.

+
+
Type: object
- - - - - - - + labels
Type: object
+

Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as labels
@@ -31240,19 +34454,16 @@

-
-
-
-

- -

-
+
+ -
-
+ + + + +
+ Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ allOf + + + + item 2 + + + + if
Type: object
@@ -31311,19 +34566,19 @@

-
+
-
+

- +

-
-

Each additional property must conform to the following schema

+
+
Type: const
+Specific value: "multi-stage" + + + + + + +
+
+
+
+
+ + + +
+ + + Type: enum (of string)
-
-

Must be one of:

-
  • "table"
  • "view"
  • "incremental"
  • "ephemeral"
-
+ builds items + + + + allOf + + + + item 2 + + + + then
Type: object
+ -
-
-
-

-
-
-
-
-
+
-
+

- +

-
+
Type: array of string
-

Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

-
Same definition as tags -
-
-
-
-Type: object
+

NEW in v0.5.5: Multi-stage orchestration pattern.

+
+ + No Additional Properties + + + + + + +
-
+

- +

-
+
Type: object
-

Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

-
Same definition as labels -
-
-
-
-
-
-
-
-
+ stages
Type: array of object
+ - -
+ No Additional Items

Each item of this array must be:

+
+
Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

- - -
- -
- - - Type: object
- + stages items
Type: object
+ No Additional Properties -
+
-
+

- +

-
+
Type: const
-Specific value: "embedded-logic" + properties + + + + stages + + + + stages items + + + + name
Type: string
+ @@ -31724,13 +34980,19 @@

-
+
+
+
+

+ +

+
- - -
- +
+
Type: object
- + stages items + + + + pattern
Type: enum (of string) Default: "hybrid-reference"
+
+

Must be one of:

+
  • "hybrid-reference"
  • "embedded-logic"
+
-
+
+
+
+
+
-
+

- +

-
+
Type: object
-

Embedded logic pattern from v0.4.0.

-
- - No Additional Properties - - - - - - -
+ properties + + + + stages + + + + stages items + + + + properties
Type: object
+

Base properties for all build patterns.

+
Same definition as properties +
+
+
+
+
-
+

- +

-
+
Type: string
+ stages + + + + stages items + + + + dependsOn
Type: array of string
- -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: enum (of string)
-
-

Must be one of:

-
  • "sql"
  • "flink_sql"
  • "pyspark"
  • "scala"
  • "python"
  • "r"
-
+ stages + + + + stages items + + + + dependsOn + + + + dependsOn items
Type: string
+ +
+
-
+
-
+

- +

-
+
Type: object
+ stages + + + + stages items + + + + outputs
Type: array
- -
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-

Additional Properties of any type are allowed.

+
+ Type: object
- - - - - - - -
+ stages items + + + + outputs + + + + identifier
Type: string
+

Unified identifier pattern: starts/ends with alphanumeric or underscore, allows dots and hyphens in middle. Both casing conventions accepted (snake_case / dotted lowercase / PascalCase / UPPER) so existing catalog ids round-trip unchanged.

+
Same definition as id
-
-
+
-
+

- +

-
+
Type: array of string
+ stages + + + + stages items + + + + tags
Type: array of string

Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

Same definition as tags
-
+
-
+

- +

-
+
Type: object
+ stages + + + + stages items + + + + labels
Type: object

Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

Same definition as labels
+
+
-
- - - - +
+
+
+

+ +

+
- -
- +
+
Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

- - -
- -
- - - Type: object
+ orchestration
Type: object
@@ -32408,18 +35742,18 @@

-
+
-
+

- +

-
+
Type: const
-Specific value: "multi-stage" - + properties + + + + orchestration + + + + parallelism
Type: integer
+ +

Value must be greater or equal to 1

-
+
+
+
+

+ +

+
- - -
- +
+
Type: object
- - - - - - - -
+ then + + + + properties + + + + orchestration + + + + retryPolicy
Type: object
+Same definition as retries +
+
+
+
+
+
+

+
+
-
+

- +

-
+
Type: object
-

NEW in v0.5.5: Multi-stage orchestration pattern.

-
- - No Additional Properties - - - - - - -
+ properties + + + + tags
Type: array of string
+

Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as tags +
+
+
+
+
-
+

- +

-
+
Type: array of object
- + labels
Type: object
+

Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as labels +
+
+
+
+
+
+
+
+
- No Additional Items

Each item of this array must be:

-
-
+ +
Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
- No Additional Properties + item 3 + + + + if
Type: object
+ -
+
-
+

- +

-
+
Type: string
- + pattern
Type: const
+Specific value: "acquisition" @@ -32822,19 +36228,13 @@

-
-
-
-

- -

-
+
-
-
+ + +
+ Type: enum (of string) Default: "hybrid-reference"
-
-

Must be one of:

-
  • "hybrid-reference"
  • "embedded-logic"
-
+ then
Type: object
+ -
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Base properties for all build patterns.

-
Same definition as properties -
-
-
-
-Type: object
+

NEW in v0.7.3: Source-aligned acquisition pattern. The build ingests data from an external system (no transformation). Engine ∈ {duckdb, airbyte, meltano, dlt, kafka-connect, debezium}.

+
+ + No Additional Properties + + + + + + +
-
+

- +

-
+
Type: array of string
+ source
Type: object
+

External source system specification.

+
+ No Additional Properties - No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: string
- + kind
Type: string
+

Source system kind: filesystem, postgres, mysql, sqlite, http, salesforce, stripe, github, kafka, etc. Engine-specific catalog of supported kinds.

+
-
-
-
+
-
+

- +

-
+
Type: array
- + connection
Type: object
+

Connection details. Use ${VAR} placeholders for env values; use secretRef for credentials. Inline secrets are forbidden by validator.

+
- No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: string
-

Unified identifier pattern: starts/ends with alphanumeric or underscore, allows dots and hyphens in middle. Both casing conventions accepted (snake_case / dotted lowercase / PascalCase / UPPER) so existing catalog ids round-trip unchanged.

-
Same definition as id -
-
+ uri
Type: string
+ + + + + + +
-
+
-
+

- +

-
+
Type: array of string
-

Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

-
Same definition as tags + host
Type: string
+ + + + + + +
-
+
-
+

- +

-
+

+

+ +

+
+ + + Type: object
-

Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

-
Same definition as labels -
-
-
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
- Type: integer
@@ -33584,19 +36940,9 @@

-
-
-
-

- -

-
- -
-
+
+ Type: integer
+ connection + + + + port + + + + oneOf + + + + item 1
Type: string
+

Allows ${ENV_VAR} placeholders that resolve to an integer at runtime.

+
+ + -

Value must be greater or equal to 1

+
+ + + +

-
+
-
+

- +

-
+
Type: object
-Same definition as retries -
-
-
-
+ connection + + + + database
Type: string
+ + + + + + +
-
+
-
+

- +

-
+
Type: array of string
-

Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

-
Same definition as tags + source + + + + connection + + + + instance_url
Type: string
+ + + + + + +
-
+
-
+

- +

-
+
Type: object
-

Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

-
Same definition as labels -
-
-
-
-
-
-
-
-
+ source + + + + connection + + + + secretRef
Type: string
+

Reference to a secret in the configured backend (vault://, aws://, gcp://, azure://, env://).

+
Must match regular expression: ^[a-z][a-z0-9_+.-]*://.+ -
- +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

- - -
- -
- - - Type: object
- + user
Type: string
+

NEW in v0.7.3: Username/principal for SQL- or REST-style sources. Required by most database drivers (Postgres/MySQL/Oracle/etc.) β€” without it the underlying client falls back to the OS-user identity and typically fails authentication. Goes alongside secretRef (or an inline password) for the credential pair.

+
-
+
+
+
+
+
-
+

- +

-
+
Type: const
-Specific value: "acquisition" - - - - - - -
-
-
-
-
- - - -
- - - Type: object
- + schema
Type: string
+

NEW in v0.7.3: Default schema/namespace within the source database to read from. Maps to engine-specific config: dlt sqldatabase schema=, Meltano tap-postgres filterschemas=[], Airbyte source-postgres schemas=[], Debezium schema.include.list, Kafka Connect JDBC schema.pattern. Use schemas (array) to scope to multiple schemas at once.

+
-
+
+
+
+
+
-
+

- +

-
+
Type: object
-

NEW in v0.7.3: Source-aligned acquisition pattern. The build ingests data from an external system (no transformation). Engine ∈ {duckdb, airbyte, meltano, dlt, kafka-connect, debezium}.

+ properties + + + + source + + + + connection + + + + schemas
Type: array of string
+

NEW in v0.7.3: Multi-schema variant of schema. When both are set, schemas wins and schema is treated as a deprecated alias. Engines that don't natively support multi-schema reads will iterate the list.

- - No Additional Properties - -
+

All items must be unique

No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: object
-

External source system specification.

-
+ source + + + + connection + + + + schemas + + + + schemas items
Type: string
- No Additional Properties -
+
+
+
+
+
+
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: string
-

Source system kind: filesystem, postgres, mysql, sqlite, http, salesforce, stripe, github, kafka, etc. Engine-specific catalog of supported kinds.

-
+ connection + + + + additionalProperties
Type: object
+ @@ -34345,18 +37752,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: object
-

Connection details. Use ${VAR} placeholders for env values; use secretRef for credentials. Inline secrets are forbidden by validator.

-
+ mode
Type: enum (of string)
+
+

Must be one of:

+
  • "full_refresh"
  • "incremental_append"
  • "incremental_dedup"
  • "incremental_merge"
  • "cdc"
  • "streaming"
+
-
+
+
+
+
+
-
+

- +

-
+
Type: string
- + cursor_field
Type: string
+

Column used for incremental cursor (e.g., updated_at, SystemModstamp).

+
@@ -34513,18 +37924,18 @@

-
+
-
+

- +

-
+
Type: string
- + watermark
Type: object
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+

-

- -

-
Type: enum (of string)
+
+

Must be one of:

+
  • "high_water_mark"
  • "log_position"
  • "lsn"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: integer
- - - - - + allowedLateness
Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
- -
- +
+
Type: string
-

Allows ${ENV_VAR} placeholders that resolve to an integer at runtime.

+ streams
Type: array of string
+

List of streams/tables/objects to ingest.

- -
- - - - - - -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
+ streams items
Type: string
@@ -34968,22 +38336,24 @@

+

+
-
+
-
+

- +

-
+
Type: string
+ reader
Type: object
@@ -35057,22 +38420,18 @@

-

-
-
-
-
+
-
+

- +

-
+
Type: string
-

Reference to a secret in the configured backend (vault://, aws://, gcp://, azure://, env://).

-
Must match regular expression: ^[a-z][a-z0-9_+.-]*://.+ + format
Type: enum (of string)
+
+

Must be one of:

+
  • "csv"
  • "parquet"
  • "json"
  • "ndjson"
  • "avro"
  • "orc"
+
@@ -35151,18 +38512,18 @@

-
+
-
+

- +

-
+
Type: string
-

NEW in v0.7.3: Username/principal for SQL- or REST-style sources. Required by most database drivers (Postgres/MySQL/Oracle/etc.) β€” without it the underlying client falls back to the OS-user identity and typically fails authentication. Goes alongside secretRef (or an inline password) for the credential pair.

-
+ options
Type: object
+ -
-
-
-
-
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: string
-

NEW in v0.7.3: Default schema/namespace within the source database to read from. Maps to engine-specific config: dlt sqldatabase schema=, Meltano tap-postgres filterschemas=[], Airbyte source-postgres schemas=[], Debezium schema.include.list, Kafka Connect JDBC schema.pattern. Use schemas (array) to scope to multiple schemas at once.

-
+ options + + + + additionalProperties
Type: object
+ @@ -35331,19 +38693,23 @@

-
+
+
+
+
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: array of string
-

NEW in v0.7.3: Multi-schema variant of schema. When both are set, schemas wins and schema is treated as a deprecated alias. Engines that don't natively support multi-schema reads will iterate the list.

-
+ additionalProperties
Type: object
+ -

All items must be unique

No Additional Items

Each item of this array must be:

+ +
+
+
+
+
+
+
+
+
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
+

Destination format for ingested data. binding.platform on the expose remains the source of truth for where; sink.format describes how.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
- + allOf + + + + item 3 + + + + then + + + + properties + + + + sink + + + + format
Type: enum (of string)
+
+

Must be one of:

+
  • "iceberg"
  • "delta"
  • "parquet"
  • "csv"
  • "json"
  • "snowflake_table"
  • "bigquery_table"
  • "redshift_table"
  • "duckdb_table"
+
-
-
-
+
-
+

- +

-
-

Additional Properties of any type are allowed.

+
+
Type: object
- + catalog
Type: enum (of string)
+
+

Must be one of:

+
  • "rest"
  • "glue"
  • "nessie"
  • "unity"
  • "snowflake-managed"
  • "hive"
  • "lakekeeper"
  • "polaris"
+
@@ -35594,22 +39038,18 @@

-
-
-
-
-
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "full_refresh"
  • "incremental_append"
  • "incremental_dedup"
  • "incremental_merge"
  • "cdc"
  • "streaming"
-
+ partitionBy
Type: array of string
+ - -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
-

Column used for incremental cursor (e.g., updated_at, SystemModstamp).

-
+ partitionBy + + + + partitionBy items
Type: string
+ +
+
-
+
-
+

- +

-
+
Type: object
- No Additional Properties - - - - + catalogAuth

+
+

NEW in v0.7.6: How the Iceberg sink config forge-cli DERIVES (kafka-connect with sink.format iceberg, embedded debezium) authenticates to an Iceberg REST catalog (rest, lakekeeper, polaris, unity, snowflake-managed). Names an environment variable, never a secret: forge-cli emits the secret as the catalog property value env:<NAME>, which Apache Iceberg's REST client resolves from the environment of the process that runs the sink (the Kafka Connect worker, or the Debezium Server forge-cli launches). Iceberg drops a property whose variable is unset, so the embedded Debezium run refuses to start without it; forge-cli cannot see a Connect worker's environment. type oauth2 (client credentials): credentialEnv names a variable holding clientid:clientsecret, oauth2ServerUri is the identity provider's token endpoint, scope is required (lakekeeper on a Keycloak-fronted Lakekeeper; PRINCIPAL_ROLE:ALL on Polaris with internal auth, whose oauth2ServerUri is <polaris>/api/catalog/v1/oauth/tokens). type bearer: tokenEnv names a variable holding a bearer token (Unity Catalog OSS). oauth2ServerUri, scope, audience and resource are written as they are and may not hold a ${...} expression or start with env: (Debezium Server would expand it, Kafka Connect would not). fluid validate refuses it on a hand-written sink config, on a catalog not reached over REST, and next to an override that sets rest.auth.type, credential, token, oauth2-server-uri, scope, audience, resource, rest.sigv4-enabled or a header.Authorization.

- -
-
-
-

- -

- -
-
+
+ +
No Additional Properties

+ +

+
+ Type: enum (of string)
-
-

Must be one of:

-
  • "high_water_mark"
  • "log_position"
  • "lsn"
-
- + allOf + + + + item 0
Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ +
+ +
-
-
-
-
-
-
-
-

- -

-
- -
-
Type: string
-

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

-
Same definition as window -
-
-
-
-
-
-
-
-
+ allOf + + + + item 0 + + + + if
Type: object
+ + + + + + + +
-
+

- +

-
+
Type: array of string
-

List of streams/tables/objects to ingest.

-
+ catalogAuth + + + + allOf + + + + item 0 + + + + if + + + + type
Type: const
+Specific value: "oauth2" - No Additional Items

Each item of this array must be:

-
-
+ +
+
+
+
+
+ + + +
Type: string
- - - - - - - -
-
-
-
-
-
-
-
-
-

- -

-
- -
-
+ allOf + + + + item 0 + + + + then

+
+

Must not be:

+
+
+ Type: object
+ catalogAuth + + + + allOf + + + + item 0 + + + + then + + + + not
Type: object
+ +
+

The following properties are required:

+
  • tokenEnv
+
+ + + +
+
+
+
+

The following properties are required:

+
  • credentialEnv
  • oauth2ServerUri
  • scope
+
-
-
-
-

- -

-
+
+ -
-
+ + + + +
+ Type: enum (of string)
-
-

Must be one of:

-
  • "csv"
  • "parquet"
  • "json"
  • "ndjson"
  • "avro"
  • "orc"
-
- + allOf + + + + item 1
Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ +
+ +
-
-
-
-
-
-
-
-

- -

-
- -
-
Type: object
+ allOf + + + + item 1 + + + + if
Type: object
@@ -36439,19 +39983,19 @@

-
+
-
+

- -

-
- -
-

Additional Properties of any type are allowed.

+ +

+
+ +
+
Type: object
- + item 1 + + + + if + + + + type
Type: const
+Specific value: "bearer" @@ -36535,23 +40093,13 @@

-
-
-
-
-
-
-

- -

-
-
-

Additional Properties of any type are allowed.

+ + +
+ Type: object
- - - - - - - -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- -
+

+ +

+
+ + + Type: object
-

Destination format for ingested data. binding.platform on the expose remains the source of truth for where; sink.format describes how.

-
- - No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: enum (of string)
-
-

Must be one of:

-
  • "iceberg"
  • "delta"
  • "parquet"
  • "csv"
  • "json"
  • "snowflake_table"
  • "bigquery_table"
  • "redshift_table"
  • "duckdb_table"
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
+
+

Must not be:

+
+
+ Type: enum (of string)
-
-

Must be one of:

-
  • "rest"
  • "glue"
  • "nessie"
  • "unity"
  • "snowflake-managed"
  • "hive"
  • "lakekeeper"
  • "polaris"
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: array of string
+ not
Type: object
+ +
+

The following properties are required:

+
  • credentialEnv
+
+ +
+
+
- No Additional Items

Each item of this array must be:

-
-
+ + + + +
Type: string
- - - - - - - -
-
-
-
-
-
-
-
-
-

- -

-
- -
-
- -
+
+

Must not be:

+
+
+ + +
-
-

NEW in v0.7.6: How the Iceberg sink config forge-cli DERIVES (kafka-connect with sink.format iceberg, embedded debezium) authenticates to an Iceberg REST catalog (rest, lakekeeper, polaris, unity, snowflake-managed). Names an environment variable, never a secret: forge-cli emits the secret as the catalog property value env:<NAME>, which Apache Iceberg's REST client resolves from the environment of the process that runs the sink (the Kafka Connect worker, or the Debezium Server forge-cli launches). Iceberg drops a property whose variable is unset, so the embedded Debezium run refuses to start without it; forge-cli cannot see a Connect worker's environment. type oauth2 (client credentials): credentialEnv names a variable holding clientid:clientsecret, oauth2ServerUri is the identity provider's token endpoint, scope is required (lakekeeper on a Keycloak-fronted Lakekeeper; PRINCIPAL_ROLE:ALL on Polaris with internal auth, whose oauth2ServerUri is <polaris>/api/catalog/v1/oauth/tokens). type bearer: tokenEnv names a variable holding a bearer token (Unity Catalog OSS). oauth2ServerUri, scope, audience and resource are written as they are and may not hold a ${...} expression or start with env: (Debezium Server would expand it, Kafka Connect would not). fluid validate refuses it on a hand-written sink config, on a catalog not reached over REST, and next to an override that sets rest.auth.type, credential, token, oauth2-server-uri, scope, audience, resource, rest.sigv4-enabled or a header.Authorization.

- -
-
- -
No Additional Properties

- -

-
- - - Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

-
Type: object
+ +
+

The following properties are required:

+
  • oauth2ServerUri
+
- - + -
- -
+ +
+
+
+ + + + +
+ + Type: object
- - - - - - - -
-
-
-

- -

-
- -
-
+ then + + + + allOf + + + + item 2

+
+

Must not be:

+
+
+ Type: const
-Specific value: "oauth2" - + allOf + + + + item 2 + + + + not
Type: object
+ +
+

The following properties are required:

+
  • scope
+
-
-
-
+
+ - -
+ + + +

+ then + + + + allOf + + + + item 3

Must not be:

@@ -37592,25 +41039,39 @@

Must not be:

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - item 0 + item 1 - then + then - not
Type: object
+ allOf + + + + item 3 + + + + not
Type: object

The following properties are required:

-
  • tokenEnv
+
  • audience
@@ -37620,16 +41081,6 @@

The following properties are required:

-
-

The following properties are required:

-
  • credentialEnv
  • oauth2ServerUri
  • scope
-
- - - - - -
@@ -37637,7 +41088,7 @@

The following properties are required:

+ id="tab-pane_builds_items_allOf_i3_then_properties_sink_catalogAuth_allOf_i1_then_allOf_i4" role="tabpanel"> Type: object
-

- -

-

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. - Otherwise, the conditions in the "Else" tab should be respected.

- - -
+ item 1 + + + + then + + + + allOf + + + + item 4

+
+

Must not be:

+
+
-
- Type: object
+ then + + + + allOf + + + + item 4 + + + + not
Type: object
+ +
+

The following properties are required:

+
  • resource
+
+ +
+
+
-
+ + + +
+
+

The following properties are required:

+
  • tokenEnv
+
+ + + + + +
+ + + + + + +
+ + + + + + +
-
+

- +

-
+
Type: const
-Specific value: "bearer" + type
Type: enum (of string)
+

oauth2: OAuth2 client credentials against oauth2ServerUri. bearer: a static bearer token.

+
+

Must be one of:

+
  • "oauth2"
  • "bearer"
+
@@ -37935,13 +41424,19 @@

-
+
+
+
+

+ +

+
- - -
- +
+

+

type oauth2: the NAME of the environment variable holding clientid:clientsecret (Iceberg splits it on the first ':'). Emitted as credential=env:<NAME>. A value shaped like a token (a run of 20 or more letters and digits, with a digit, between underscores) is refused.

+
+

Must not be:

+
+
+ + +
-

- -

-
- - - Type: object
+Must match regular expression: (?:^|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_|$) + + + + + + +
+
+
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?![\s\S]) + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
-
-

Must not be:

-
-
- - -
+

type oauth2: absolute URL starting with a lowercase http:// or https:// of the identity provider's token endpoint, with no user:password@ part (the client id and secret go in credentialEnv's variable), e.g. a Keycloak realm's <keycloak>/realms/<realm>/protocol/openid-connect/token. Emitted as oauth2-server-uri; without it Iceberg would send the credential to the catalog's own v1/oauth/tokens.

+
+

Must not be:

+
+
+ + + Type: object
- -
-

The following properties are required:

-
  • credentialEnv
-
+ not
Type: object
+Must match regular expression: \$\{|[\x00-\x1f\x7f]|^\s*env: + @@ -38262,16 +41765,30 @@

The following properties are required:

-
+
Must match regular expression: ^https?://[^\s/?#@]+(?:[/?#]\S*)?$ -
- +
+
+
+
+
+
+
+

+ +

+
+ +
+

-
+

type oauth2: the OAuth2 scope to request (Iceberg's default, catalog, is not one most identity providers know). Emitted as scope.

+

Must not be:

- - Type: object
+Must match regular expression: \$\{|[\x00-\x1f\x7f]|^\s*env: + @@ -38486,12 +41945,26 @@

The following properties are required:

- +

Must be at least 1 characters long

-
- +
+
+
+
+
+
+
+

+ +

+
+ +
+

-
+

type oauth2, optional: sent as the audience parameter on the token request.

+

Must not be:

@@ -38654,47 +42100,16 @@

Must not be:

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - allOf - - - - item 1 - - - - then - - - - allOf - - - - item 2 + audience - not
Type: object
- -
-

The following properties are required:

-
  • scope
-
+ not
Type: object
+Must match regular expression: \$\{|[\x00-\x1f\x7f]|^\s*env: + @@ -38706,12 +42121,26 @@

The following properties are required:

- +

Must be at least 1 characters long

-
- +
+
+
+
+
+
+
+

+ +

+
+ +
+

-
+

type oauth2, optional: sent as the resource parameter on the token request.

+

Must not be:

@@ -38874,47 +42276,16 @@

Must not be:

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - allOf - - - - item 1 - - - - then - - - - allOf - - - - item 3 + resource - not
Type: object
- -
-

The following properties are required:

-
  • audience
-
+ not
Type: object
+Must match regular expression: \$\{|[\x00-\x1f\x7f]|^\s*env: + @@ -38926,12 +42297,26 @@

The following properties are required:

- +

Must be at least 1 characters long

-
- +
+
+
+
+
+
+
+

+ +

+
+ +
+

+

type bearer: the NAME of the environment variable holding the bearer token. Emitted as token=env:<NAME>. A value shaped like a token (a run of 20 or more letters and digits, with a digit, between underscores, such as a dapi... personal access token) is refused.

+
+

Must not be:

+
+
+ + +
-
-

Must not be:

-
-
- - - Type: object
+Must match regular expression: (?:^|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_|$) + + + + + + +
+
+
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?![\s\S]) + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
- -
-

The following properties are required:

-
  • resource
-
- - - - - -
-
-
- - - - - - -
-
-

The following properties are required:

-
  • tokenEnv
-
- - - - - -
- - - - + delivery
Type: object
+

Delivery semantics + idempotency + DLQ configuration.

+
- -
+ No Additional Properties -
+
-
+

- +

-
+
Type: enum (of string)
-

oauth2: OAuth2 client credentials against oauth2ServerUri. bearer: a static bearer token.

-
Type: enum (of string) Default: "at_least_once"
+

Must be one of:

-
  • "oauth2"
  • "bearer"
+
  • "at_most_once"
  • "at_least_once"
  • "exactly_once"
@@ -39266,18 +42647,18 @@

Must be one of:

-
+
-
+

- +

-
+

-

type oauth2: the NAME of the environment variable holding clientid:clientsecret (Iceberg splits it on the first ':'). Emitted as credential=env:<NAME>. A value shaped like a token (a run of 20 or more letters and digits, with a digit, between underscores) is refused.

-
-

Must not be:

-
- Type: string Default: "{run_id}:{stream}:{record_pk}"
+

Template for the idempotency key. Default: {runid}:{stream}:{recordpk}

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
-Must match regular expression: (?:^|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_|$) - - - - - - -
-
-
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?![\s\S]) + dlq
Type: object
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+

-

type oauth2: absolute URL starting with a lowercase http:// or https:// of the identity provider's token endpoint, with no user:password@ part (the client id and secret go in credentialEnv's variable), e.g. a Keycloak realm's <keycloak>/realms/<realm>/protocol/openid-connect/token. Emitted as oauth2-server-uri; without it Iceberg would send the credential to the catalog's own v1/oauth/tokens.

-
-

Must not be:

-
-
- + enabled
Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
-Must match regular expression: \$\{|[\x00-\x1f\x7f]|^\s*env: - - - - - - -
-
-
Must match regular expression: ^https?://[^\s/?#@]+(?:[/?#]\S*)?$ + sink
Type: object
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+

-

type oauth2: the OAuth2 scope to request (Iceberg's default, catalog, is not one most identity providers know). Emitted as scope.

-
-

Must not be:

-
-
- + sink + + + + format
Type: enum (of string)
+
+

Must be one of:

+
  • "parquet"
  • "json"
  • "ndjson"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
-Must match regular expression: \$\{|[\x00-\x1f\x7f]|^\s*env: + location
Type: string
+ +
+
-
- - - -

Must be at least 1 characters long

- -
-
+
-
+

- +

-
+

-

type oauth2, optional: sent as the audience parameter on the token request.

-
-

Must not be:

-
- Type: integer Default: 10000
+ + + + +

Value must be greater or equal to 0

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
-Must match regular expression: \$\{|[\x00-\x1f\x7f]|^\s*env: - + alertOn
Type: array of enum (of string)
- -
-
-
- - -

Must be at least 1 characters long

- -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+
-

type oauth2, optional: sent as the resource parameter on the token request.

-
-

Must not be:

-
-
- + alertOn + + + + alertOn items
Type: enum (of string)
+
+

Must be one of:

+
  • "pii_classification_failed"
  • "schema_violation"
  • "destination_write_failed"
  • "quality_gate_failed"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
-Must match regular expression: \$\{|[\x00-\x1f\x7f]|^\s*env: - + schemaEvolution
Type: object
+

Schema evolution behavior. Stricter-wins between policy and per-change overrides.

+
- + No Additional Properties -
-
-
- - -

Must be at least 1 characters long

-
-
-
-
-
+
-
+

- +

-
+

-

type bearer: the NAME of the environment variable holding the bearer token. Emitted as token=env:<NAME>. A value shaped like a token (a run of 20 or more letters and digits, with a digit, between underscores, such as a dapi... personal access token) is refused.

-
-

Must not be:

-
-
- + policy
Type: enum (of string) Default: "strict"
+
+

Must be one of:

+
  • "strict"
  • "discover_and_freeze"
  • "evolve_safe"
  • "evolve_all"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
Type: object
-Must match regular expression: (?:^|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_|$) - - - - - - -
-
-
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?![\s\S]) + onAddedColumn
Type: enum (of string)
+
+

Must be one of:

+
  • "include"
  • "warn"
  • "fail"
+
@@ -40322,26 +43698,18 @@

Must not be:

-
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Delivery semantics + idempotency + DLQ configuration.

-
- - No Additional Properties + schemaEvolution + + + + onRemovedColumn
Type: enum (of string)
+
+

Must be one of:

+
  • "drop"
  • "warn"
  • "fail"
+
-
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string) Default: "at_least_once"
-Type: enum (of string)
+

Must be one of:

-
  • "at_most_once"
  • "at_least_once"
  • "exactly_once"
+
  • "cast"
  • "warn"
  • "fail"
@@ -40489,18 +43868,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: string Default: "{run_id}:{stream}:{record_pk}"
-

Template for the idempotency key. Default: {runid}:{stream}:{recordpk}

-
+ sourceFingerprint
Type: enum (of string) Default: "required"
+
+

Must be one of:

+
  • "required"
  • "optional"
  • "disabled"
+
@@ -40572,18 +43953,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: object
- No Additional Properties + preLand
Type: array of enum (of string)
+

Pre-land hook chain - runs on each batch before destination write.

+
- -
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: boolean Default: true
- + preLand items
Type: enum (of string)
+
+

Must be one of:

+
  • "dlp_scan"
  • "tokenize_pii"
  • "quality_gate"
  • "emit_lineage_input"
+
+
+
-
+
-
+

- +

-
+
Type: object
- No Additional Properties + quality
Type: object
+

Pre-land quality gates + anomaly signals.

+
+ + No Additional Properties -
+
-
+

- +

-
+
Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: enum (of string)
-
-

Must be one of:

-
  • "parquet"
  • "json"
  • "ndjson"
-
+ builds items + + + + allOf + + + + item 3 + + + + then + + + + properties + + + + quality + + + + gates + + + + gates items
Type: object
+ -
-
-
-
-
+
-
+

- +

-
+
Type: string
- + rule
Type: enum (of string)
+
+

Must be one of:

+
  • "not_null"
  • "unique"
  • "regex"
  • "range"
  • "row_count_anomaly"
  • "freshness"
+
@@ -41019,22 +44432,18 @@

-
-
-
-
-
+
-
+

- +

-
+
Type: integer Default: 10000
+ gates items + + + + columns
Type: array of string
-

Value must be greater or equal to 0

- -
-
-
-
-
-
-
-

- -

-
-
-
+ No Additional Items

Each item of this array must be:

+
+
+ Type: array of enum (of string)
+ gates items + + + + columns + + + + columns items
Type: string
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: enum (of string)
-
-

Must be one of:

-
  • "pii_classification_failed"
  • "schema_violation"
  • "destination_write_failed"
  • "quality_gate_failed"
-
+ column
Type: string
+ -
-
-
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Schema evolution behavior. Stricter-wins between policy and per-change overrides.

-
+ quality + + + + gates + + + + gates items + + + + pattern
Type: string
- No Additional Properties -
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string) Default: "strict"
-
-

Must be one of:

-
  • "strict"
  • "discover_and_freeze"
  • "evolve_safe"
  • "evolve_all"
-
+ gates + + + + gates items + + + + min
Type: number
+ @@ -41455,18 +44907,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "include"
  • "warn"
  • "fail"
-
+ gates + + + + gates items + + + + max
Type: number
+ @@ -41540,18 +45003,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: enum (of string)
-
+ gates + + + + gates items + + + + algorithm
Type: enum (of string)
+

Must be one of:

-
  • "drop"
  • "warn"
  • "fail"
+
  • "ewma"
  • "iqr"
  • "exact"
@@ -41625,18 +45102,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "cast"
  • "warn"
  • "fail"
-
+ gates + + + + gates items + + + + sensitivity
Type: number
+ @@ -41710,18 +45198,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: enum (of string) Default: "required"
-
+ gates + + + + gates items + + + + severity
Type: enum (of string)
+

Must be one of:

-
  • "required"
  • "optional"
  • "disabled"
+
  • "info"
  • "warn"
  • "error"
@@ -41795,23 +45297,19 @@

Must be one of:

-
-
-
-
-
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: array of enum (of string)
-

Pre-land hook chain - runs on each batch before destination write.

-
- - - - - - No Additional Items

Each item of this array must be:

-
-
- - - Type: enum (of string)
-
-

Must be one of:

-
  • "dlp_scan"
  • "tokenize_pii"
  • "quality_gate"
  • "emit_lineage_input"
-
+ additionalProperties
Type: object
+ +
+
+
+
-
+
-
+

- +

-
+
Type: object
-

Pre-land quality gates + anomaly signals.

-
- - No Additional Properties + quality + + + + onError
Type: enum (of string) Default: "route_to_dlq"
+
+

Must be one of:

+
  • "route_to_dlq"
  • "abort_run"
  • "best_effort"
+
-
+
+
+
+
+
-
+

- +

-
+
Type: array of object
+ anomalies
Type: array of object
@@ -42101,7 +45563,7 @@

No Additional Items

Each item of this array must be:

-
+
Type: object
- + anomalies items
Type: object
+ No Additional Properties -
+
-
+

- +

-
+
Type: enum (of string)
-Type: enum (of string)
+

Must be one of:

-
  • "not_null"
  • "unique"
  • "regex"
  • "range"
  • "row_count_anomaly"
  • "freshness"
+
  • "record_count_drop"
  • "record_count_surge"
  • "bytes_per_record_outlier"
  • "latency_outlier"
  • "dlq_rate_spike"
  • "cursor_stalled"
  • "schema_fingerprint_changed"
@@ -42274,18 +45736,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: array of string
- + algorithm
Type: enum (of string)
+
+

Must be one of:

+
  • "ewma"
  • "iqr"
  • "exact"
+
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
+ sensitivity
Type: number
@@ -42455,24 +45927,22 @@

-

-
-
+
-
+

- +

-
+
Type: string
- + severity
Type: enum (of string)
+
+

Must be one of:

+
  • "info"
  • "warn"
  • "error"
+
@@ -42557,18 +46030,28 @@

-
+
+
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: string
+ cost
Type: object
+

Cost tracking + budget enforcement.

+
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: number
- + budget
Type: object
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: number
- + monthly
Type: object
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "ewma"
  • "iqr"
  • "exact"
-
- + rows
Type: integer
+ +

Value must be greater or equal to 0

-
+
-
+

- +

-
+
Type: number
- + bytes
Type: string
+

Human-readable size with unit suffix (e.g., '50GB', '500MB').

+
@@ -43040,18 +46470,18 @@

-
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "info"
  • "warn"
  • "error"
-
- + computeMinutes
Type: integer
+ +

Value must be greater or equal to 0

-
+
+
+
+
+
-
+

- +

-
-

Additional Properties of any type are allowed.

+
+
Type: object
- + onExceed
Type: enum (of string) Default: "warn"
+
+

Must be one of:

+
  • "warn"
  • "abort"
+
@@ -43235,24 +46662,22 @@

-
-
-
+
-
+

- +

-
+
Type: enum (of string) Default: "route_to_dlq"
-
-

Must be one of:

-
  • "route_to_dlq"
  • "abort_run"
  • "best_effort"
-
+ chargeback
Type: object
+ -
-
-
-
-
+
-
+

- +

-
+
Type: array of object
+ chargeback + + + + team
Type: string
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: object
- No Additional Properties + project
Type: string
+ -
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "record_count_drop"
  • "record_count_surge"
  • "bytes_per_record_outlier"
  • "latency_outlier"
  • "dlq_rate_spike"
  • "cursor_stalled"
  • "schema_fingerprint_changed"
-
+ costCenter
Type: string
+ @@ -43578,19 +47011,19 @@

Must be one of:

-
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: enum (of string)
-
-

Must be one of:

-
  • "ewma"
  • "iqr"
  • "exact"
-
+ additionalProperties
Type: object
+ @@ -43677,18 +47100,26 @@

Must be one of:

-
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: number
+ catalog
Type: object
+

Catalog auto-registration on first apply.

+
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "info"
  • "warn"
  • "error"
-
+ register
Type: array of enum (of string)
+ - -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: object
-

Cost tracking + budget enforcement.

-
- - No Additional Properties + catalog + + + + register + + + + register items
Type: enum (of string)
+
+

Must be one of:

+
  • "datahub"
  • "openmetadata"
  • "datamesh_manager"
  • "unity"
  • "glue"
  • "snowflake_horizon"
+
-
+
+
+
+
+
+
+
-
+

- +

-
+
Type: object
- No Additional Properties + documentation
Type: enum (of string) Default: "auto"
+
+

Must be one of:

+
  • "auto"
  • "manual"
  • "none"
+
-
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: object
+

Single-flight concurrency control.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ builds items + + + + allOf + + + + item 3 + + + + then + + + + properties + + + + concurrency + + + + lock
Type: object
No Additional Properties @@ -44119,18 +47585,18 @@

-
+
-
+

- +

-
+
Type: integer
- + scope
Type: enum (of string) Default: "product"
+
+

Must be one of:

+
  • "product"
  • "build"
+
-

Value must be greater or equal to 0

+
-
+
-
+

- +

-
+
Type: string
-

Human-readable size with unit suffix (e.g., '50GB', '500MB').

-
- - - - - - + timeout
Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window
-
+
-
+

- +

-
+
Type: integer
- + onContended
Type: enum (of string) Default: "abort"
+
+

Must be one of:

+
  • "abort"
  • "queue"
  • "replace"
+
-

Value must be greater or equal to 0

+
@@ -44412,18 +47857,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string) Default: "warn"
-
-

Must be one of:

-
  • "warn"
  • "abort"
-
+ lineage
Type: object
+ No Additional Properties -
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
+ emit
Type: boolean Default: true
@@ -44586,18 +48010,26 @@

-
+
+

+
+
+
+
+
+

+
-
+

- +

-
+
Type: string
+ duckdb
Type: object
@@ -44671,22 +48089,18 @@

-

-
-
-
-
+
-
+

- +

-
+
Type: string
+ extensions
Type: array of string
- -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
+ extensions items
Type: string
@@ -44849,22 +48242,24 @@

+

+
-
+
-
+

- +

-
+

Additional Properties of any type are allowed.

Type: object
+ additionalProperties
Type: object
@@ -44946,22 +48334,18 @@

-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Catalog auto-registration on first apply.

-
+ airbyte
Type: object
- No Additional Properties -
+
-
+

- +

-
+
Type: array of enum (of string)
+ connector_image
Type: string
- No Additional Items

Each item of this array must be:

-
-
- - - Type: enum (of string)
-
-

Must be one of:

-
  • "datahub"
  • "openmetadata"
  • "datamesh_manager"
  • "unity"
  • "glue"
  • "snowflake_horizon"
-
- - - - - -
-
-
+
-
+

- +

-
+
Type: enum (of string) Default: "auto"
-
-

Must be one of:

-
  • "auto"
  • "manual"
  • "none"
-
+ version
Type: string
+ @@ -45271,22 +48569,18 @@

Must be one of:

-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Single-flight concurrency control.

-
+ airbyte + + + + normalization
Type: boolean
- No Additional Properties -
+
+
+
+
+
-
+

- +

-
+
Type: object
- No Additional Properties + image_signature
Type: object
+

Connector image supply-chain verification.

+
+ + No Additional Properties -
+
-
+

- +

-
+
Type: enum (of string) Default: "product"
-Type: enum (of string) Default: "cosign"
+

Must be one of:

-
  • "product"
  • "build"
+
  • "cosign"
@@ -45519,18 +48824,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: string
-

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

-
Same definition as window + publicKey
Type: string
+ + + + + + +
-
+
-
+

- +

-
+
Type: enum (of string) Default: "abort"
-Type: enum (of string) Default: "optional"
+

Must be one of:

-
  • "abort"
  • "queue"
  • "replace"
+
  • "required"
  • "optional"
  • "disabled"
@@ -45699,93 +49009,18 @@

Must be one of:

-
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
- No Additional Properties - - - - - - -
+
-
+

- +

-
+
Type: boolean Default: true
+ deployment
Type: object
+

Engine deployment mode and target.

+
+ No Additional Properties -
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
- + airbyte + + + + deployment + + + + mode
Type: enum (of string) Default: "embedded"
+
+

Must be one of:

+
  • "embedded"
  • "bring-your-own"
  • "managed"
+
-
+
+
+
+
+
-
+

- +

-
+
Type: array of string
- + deployment + + + + server_url
Type: string
+

URL for bring-your-own mode.

+
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
+ auth
Type: object
@@ -46084,25 +49357,19 @@

-

-
-
-
-
-
-
+
-
+

- +

-
-

Additional Properties of any type are allowed.

+
+
Type: object
+ deployment + + + + auth + + + + secretRef
Type: string
@@ -46172,23 +49453,19 @@

-
-
-
-
-
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: object
+ airbyte + + + + deployment + + + + auth + + + + additionalProperties
Type: object
@@ -46247,18 +49545,26 @@

-
+
+

+
+
+
+
+
+
+
-
+

- +

-
+
Type: string
- + deployment + + + + managed
Type: object
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: string
- + deployment + + + + managed + + + + target
Type: enum (of string)
+
+

Must be one of:

+
  • "docker"
  • "kubernetes"
  • "terraform"
  • "opentofu"
+
@@ -46411,18 +49737,18 @@

-
+
-
+

- +

-
+
Type: boolean
- + deployment + + + + managed + + + + profile
Type: enum (of string) Default: "small"
+
+

Must be one of:

+
  • "small"
  • "medium"
  • "large"
+
@@ -46493,18 +49836,18 @@

-
+
-
+

- +

-
+
Type: object
-

Connector image supply-chain verification.

-
+ deployment + + + + managed + + + + chart
Type: object
- No Additional Properties -
+
-
+

- +

-
+
Type: enum (of string) Default: "cosign"
-
-

Must be one of:

-
  • "cosign"
-
+ managed + + + + chart + + + + repo
Type: string
+ @@ -46666,18 +50031,18 @@

Must be one of:

-
+
-
+

- +

-
+
Type: string
+ managed + + + + chart + + + + name
Type: string
@@ -46755,18 +50134,18 @@

-
+
-
+

- +

-
+
Type: enum (of string) Default: "optional"
-
-

Must be one of:

-
  • "required"
  • "optional"
  • "disabled"
-
+ managed + + + + chart + + + + version
Type: string
+ @@ -46847,23 +50237,19 @@

Must be one of:

-
-
-
-
-
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: object
-

Engine deployment mode and target.

-
+ deployment + + + + managed + + + + chart + + + + additionalProperties
Type: object
- No Additional Properties -
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string) Default: "embedded"
-
-

Must be one of:

-
  • "embedded"
  • "bring-your-own"
  • "managed"
-
+ managed + + + + values_overlay
Type: object
+ -
-
-
-
-
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: string
-

URL for bring-your-own mode.

-
+ managed + + + + values_overlay + + + + additionalProperties
Type: object
+ @@ -47114,18 +50539,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: object
+ managed + + + + secrets
Type: array of object
- -
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
- + secrets + + + + secrets items
Type: object
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
-

Additional Properties of any type are allowed.

+
+
Type: object
+ secrets + + + + secrets items + + + + name
Type: string
@@ -47391,22 +50834,18 @@

-
-
-
-
-
+
-
+

- +

-
+
Type: object
- No Additional Properties + managed + + + + secrets + + + + secrets items + + + + ref
Type: string
+ -
+
+
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: enum (of string)
-
-

Must be one of:

-
  • "docker"
  • "kubernetes"
  • "terraform"
  • "opentofu"
-
+ network
Type: object
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: enum (of string) Default: "small"
-
-

Must be one of:

-
  • "small"
  • "medium"
  • "large"
-
+ network + + + + egressAllowList
Type: array of string
+ - -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: object
+ network + + + + egressAllowList + + + + egressAllowList items
Type: string
@@ -47770,19 +51237,37 @@

-
+
+

+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: string
+ additionalProperties
Type: object
@@ -47873,18 +51337,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: string
+ meltano
Type: object
@@ -47972,22 +51412,18 @@

-

-
-
-
-
+
-
+

- +

-
+
Type: string
+ tap
Type: string
@@ -48079,19 +51494,19 @@

-
+
-
+

- +

-
-

Additional Properties of any type are allowed.

+
+
Type: object
+ project_dir
Type: string
@@ -48182,22 +51576,18 @@

-
-
-
-
-
+
-
+

- +

-
+
Type: object
- - - - - - - -Type: object
+

Engine deployment mode and target.

+
Same definition as deployment +
+
+
+
+
-
+

- +

-
+

Additional Properties of any type are allowed.

Type: object
+ additionalProperties
Type: object
@@ -48385,18 +51739,18 @@

-
+
-
+

- +

-
+
Type: array of object
+ dlt
Type: object
- No Additional Items

Each item of this array must be:

+ +
-
- +
+

+ +

+
+ +
+
Type: object
- No Additional Properties + source_module
Type: string
+ -
+
+
+
+
+
-
+

- +

-
+
Type: string
+ pipeline_name
Type: string
@@ -48676,19 +51974,19 @@

-
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: string
+ additionalProperties
Type: object
@@ -48786,24 +52056,22 @@

-
-
-
+
-
+

- +

-
+
Type: object
- No Additional Properties + kafka-connect
Type: object
+ -
+
-
+

- +

-
+
Type: array of string
+ connector_class
Type: string
- No Additional Items

Each item of this array must be:

+ +
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
+ connector_name
Type: string
@@ -49079,37 +52291,23 @@

-

-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
+

- +

-
-

Additional Properties of any type are allowed.

+
+
Type: object
+ tasks_max
Type: integer Default: 1
- +

Value must be greater or equal to 1

-
-
-
-
-
+
-
+

- +

-
+
Type: object
- + kafka-connect + + + + schema_registry
Type: object
+

Confluent-compatible Schema Registry config. When url is set, the runner injects key.converter/value.converter for Avro into the emitted connector config.

+
No Additional Properties -
+
-
+

- +

-
+
Type: string
- + schema_registry + + + + url
Type: stringFormat: uri
+

Schema Registry HTTP endpoint, e.g. http://schema-registry:8081

+
@@ -49336,18 +52546,18 @@

-
+
-
+

- +

-
+
Type: string
- + schema_registry + + + + auth
Type: object
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Engine deployment mode and target.

-
Same definition as deployment + schema_registry + + + + auth + + + + secretRef
Type: string
+ + + + + + +
-
+
+
+
+
+
+
+
+
+
-
+

- +

-
-

Additional Properties of any type are allowed.

+
+
Type: object
+ sink_connector_name
Type: string
@@ -49577,22 +52817,18 @@

-
-
-
-
-
+
-
+

- +

-
+
Type: object
+ kafka-connect + + + + sink_connector_config
Type: object
@@ -49652,19 +52895,19 @@

-
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: string
+ sink_connector_config + + + + additionalProperties
Type: object
@@ -49734,18 +52984,22 @@

-
+
+
+
+
+
-
+

- +

-
+
Type: string
- - - - - - - + image_signature
Type: object
+

Connector image supply-chain verification.

+
Same definition as image_signature
-
+
-
+

- +

-
-

Additional Properties of any type are allowed.

+
+
Type: object
- - - - - - - -
-
-
-
+ deployment
Type: object
+

Engine deployment mode and target.

+
Same definition as deployment
-
+
-
+

- +

-
+
Type: object
- + kafka-connect + + + + iceberg_sink_enabled
Type: boolean
+

Opt-in: derive the Iceberg sink connector config. Defaults OFF when a hand-written sinkconnectorconfig is present (RFC Β§6.2).

+
-
+
+
+
+
+
-
+

- +

-
+
Type: string
- + sink_topics
Type: array of string
+

Explicit topics the derived Iceberg sink consumes (else derived from source streams).

+
- -
-
-
-
-
+ No Additional Items

Each item of this array must be:

-
-

- -

-
- -
-
+
+ Type: string
+ sink_topics + + + + sink_topics items
Type: string
@@ -50133,22 +53379,24 @@

+

+
-
+
-
+

- +

-
+
Type: integer Default: 1
- + iceberg_catalog_overrides
Type: object
+

Operator escape hatch: connector config keys merged LAST over the derived Iceberg sink config.

+
-

Value must be greater or equal to 1

+ -
-
-
-
-
+
-
+

- +

-
-
+
+

Each additional property must conform to the following schema

Type: object
-

Confluent-compatible Schema Registry config. When url is set, the runner injects key.converter/value.converter for Avro into the emitted connector config.

-
No Additional Properties + iceberg_catalog_overrides + + + + additionalProperties
Type: string
+ -
+
+
+
+
+
+
+
+
+
-
+

- +

-
+
Type: stringFormat: uri
-

Schema Registry HTTP endpoint, e.g. http://schema-registry:8081

-
+ streamingSink
Type: object
+

Optional Iceberg streaming-sink tuning (RFC Β§6.2). All keys optional.

+
No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: object
- No Additional Properties - + commitIntervalMs
Type: integer
+ +

Value must be greater or equal to 1

-
+
+
+
+
+
-
+

- +

-
+
Type: string
+ routeField
Type: string
@@ -50569,26 +53814,18 @@

-
-
-
-
-
-
-
-
-
+
-
+

- +

-
+
Type: string
+ streamingSink + + + + dynamicEnabled
Type: boolean
@@ -50659,18 +53903,18 @@

-
+
-
+

- +

-
+
Type: object
+ streamingSink + + + + autoCreate
Type: boolean
@@ -50737,19 +53988,23 @@

-
+
+

+
+
+
-
+

- +

-
-

Additional Properties of any type are allowed.

+
+
Type: object
+ evolveSchema
Type: boolean
@@ -50826,22 +54081,18 @@

-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Connector image supply-chain verification.

-
Same definition as image_signature -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

Engine deployment mode and target.

-
Same definition as deployment + upsertMode
Type: boolean
+ + + + + + +
-
+
-
+

- +

-
+
Type: boolean
-

Opt-in: derive the Iceberg sink connector config. Defaults OFF when a hand-written sinkconnectorconfig is present (RFC Β§6.2).

-
+ streamingSink + + + + controlTopic
Type: string
+ @@ -51067,19 +54259,23 @@

-
+
+
+
+
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: array of string
-

Explicit topics the derived Iceberg sink consumes (else derived from source streams).

-
+ additionalProperties
Type: object
+ - No Additional Items

Each item of this array must be:

+ +
+
+
+
+
+
+
+
+
-
- +
+

+ +

+
+ +
+
Type: string
+ debezium
Type: object
@@ -51221,24 +54420,18 @@

-

-
-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Operator escape hatch: connector config keys merged LAST over the derived Iceberg sink config.

-
+ connector_class
Type: string
+ -
+
+
+
+
+
-
+

- +

-
-

Each additional property must conform to the following schema

+
+
Type: string
- + snapshot_mode
Type: enum (of string)
+
+

Must be one of:

+
  • "initial"
  • "schema_only"
  • "never"
  • "when_needed"
  • "always"
+
@@ -51395,22 +54587,18 @@

-
-
-
-
-
+
-
+

- +

-
+
Type: object
-

Optional Iceberg streaming-sink tuning (RFC Β§6.2). All keys optional.

-
No Additional Properties - - - - - - -Type: object
+

Engine deployment mode and target.

+
Same definition as deployment +
+
+
+
+
-
+

- +

-
-
+
+

Additional Properties of any type are allowed.

Type: integer
+ additionalProperties
Type: object
-

Value must be greater or equal to 1

+
-
+
+
+
+
+
+
+
+
+
+ + + + + + +
+ + + + + + +
-
+

- +

-
+
Type: string
+

Build identifier for multi-build scenarios (e.g., 'batch-processing', 'ml-training').

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Human-readable description of this build configuration.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
- + pattern
Type: enum (of string) Default: "hybrid-reference"
+

Build pattern: hybrid-reference (dbt-style), embedded-logic (raw SQL/code), multi-stage (complex pipelines), or acquisition (source-aligned ingestion from external systems).

+
+

Must be one of:

+
  • "hybrid-reference"
  • "embedded-logic"
  • "multi-stage"
  • "acquisition"
+
@@ -51656,18 +54909,18 @@

-
+
-
+

- +

-
+
Default: "dbt"
+
+

Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-qualified dbt engines (dbt-<adapter>, e.g. dbt-glue, dbt-snowflake, dbt-bigquery) are accepted too and route through the dbt build runner, which derives the adapter from the suffix. Ingestion engines (acquisition pattern only): duckdb (zero-infra files/JDBC), airbyte (350+ SaaS), meltano (Singer 600+ taps), dlt (Python-native), kafka-connect (streaming), debezium (CDC). AWS Glue: glue (Spark ETL jobs managed by Glue). NEW in v0.7.6: flink (Apache Flink jobs, usually long-running streaming SQL with properties.language: flink_sql). forge-cli validates and plans a flink build but ships no runner for it: the platform that runs the job, such as the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, executes it. Put deployment settings under properties.parameters, for example parallelism and upgradeMode, as the Flink Kubernetes Operator's job spec names them, and flinkConfiguration.

+ +
+
+ +

+ +

+
Type: enum (of string)
+
+

Must be one of:

+
  • "dbt"
  • "sql"
  • "python"
  • "spark"
  • "glue"
  • "custom"
  • "duckdb"
  • "airbyte"
  • "meltano"
  • "dlt"
  • "kafka-connect"
  • "debezium"
  • "flink"
+
-
-
-
-
-
-
-
-

- -

-
- -
-
+
+ Type: boolean
+ item 1
Type: object
+Must match regular expression: ^dbt-[a-z0-9]+([_-][a-z0-9]+)*$ + + + + +
+ @@ -51834,18 +55074,18 @@

-
+
-
+

- +

-
+
Type: array of enum (of string)
+

Capabilities the build asks for (acquisition pattern). Validation enforces capabilities βŠ† runner.declared_capabilities.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+ Type: enum (of string)
+
+

Must be one of:

+
  • "full_refresh"
  • "incremental_append"
  • "incremental_dedup"
  • "incremental_merge"
  • "cdc"
  • "streaming"
  • "schema_discovery"
  • "schema_evolution"
  • "dlp_scan"
  • "at_most_once"
  • "at_least_once"
  • "exactly_once"
+
+
+
-
+
-
+

- +

-
+
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Pattern-specific build properties with conditional validation.

+
+ + + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+ Type: object
+

Build execution configuration (triggers, runtime, retries).

+
+ No Additional Properties -
-
-
-
-
+
-
+

- +

-
+
Type: object
+ + +

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ + Type: object
@@ -52097,27 +55427,19 @@

-

-
-
-
-
-
-
-
-
+
-
+

- +

-
-

Additional Properties of any type are allowed.

+
+
Type: object
- + type
Type: const
+Specific value: "continuous" @@ -52187,23 +55495,13 @@

-
-
-
-
-
-
-

- -

-
-
-
+ + +
+ Type: object
- - - - - - - -
-
-
-

- -

-
- -
-
+ then

+

+ +

+
+ Type: string
- - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
+

cron is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove cron, or set type: schedule.

+
+

Must not be:

+
+
+ Type: enum (of string)
-
-

Must be one of:

-
  • "initial"
  • "schema_only"
  • "never"
  • "when_needed"
  • "always"
-
- + fill-rule="evenodd" + d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" + /> + + not
Type: object
+ +
+

The following properties are required:

+
  • cron
+
-
-
-
-
-
-

- -

-
+
+ -
-
+ + + + +
+ Type: object
-

Engine deployment mode and target.

-
Same definition as deployment -
-
-
-
-
-
-
-

- -

-
- -
-

Additional Properties of any type are allowed.

+ item 1

+

schedule is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove schedule, or set type: schedule.

+
+

Must not be:

+
+
+ Type: object
+ not
Type: object
- +
+

The following properties are required:

+
  • schedule
+
-
-
-
-
-
-
-
-
-
-
- - - - - - -
-
-
-
-

- -

-
- -
-
+
+ Type: string
-

Build identifier for multi-build scenarios (e.g., 'batch-processing', 'ml-training').

-
Same definition as id -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: string
-

Human-readable description of this build configuration.

-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: enum (of string) Default: "hybrid-reference"
-

Build pattern: hybrid-reference (dbt-style), embedded-logic (raw SQL/code), multi-stage (complex pipelines), or acquisition (source-aligned ingestion from external systems).

-
-

Must be one of:

-
  • "hybrid-reference"
  • "embedded-logic"
  • "multi-stage"
  • "acquisition"
-
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
+

datasets is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove datasets, or set type: dataset.

+
+

Must not be:

+
+
+ Default: "dbt"
-
-

Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-qualified dbt engines (dbt-<adapter>, e.g. dbt-glue, dbt-snowflake, dbt-bigquery) are accepted too and route through the dbt build runner, which derives the adapter from the suffix. Ingestion engines (acquisition pattern only): duckdb (zero-infra files/JDBC), airbyte (350+ SaaS), meltano (Singer 600+ taps), dlt (Python-native), kafka-connect (streaming), debezium (CDC). AWS Glue: glue (Spark ETL jobs managed by Glue). NEW in v0.7.6: flink (Apache Flink jobs, usually long-running streaming SQL with properties.language: flink_sql). forge-cli validates and plans a flink build but ships no runner for it: the platform that runs the job, such as the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, executes it. Put deployment settings under properties.parameters, for example parallelism and upgradeMode, as the Flink Kubernetes Operator's job spec names them, and flinkConfiguration.

- -
-
- -

- -

-
- - - Type: enum (of string)
-
-

Must be one of:

-
  • "dbt"
  • "sql"
  • "python"
  • "spark"
  • "glue"
  • "custom"
  • "duckdb"
  • "airbyte"
  • "meltano"
  • "dlt"
  • "kafka-connect"
  • "debezium"
  • "flink"
-
+ not
Type: object
+ +
+

The following properties are required:

+
  • datasets
+
+ + + + + +
+
+
@@ -52859,7 +55979,7 @@

Must be one of:

+ id="tab-pane_builds_items_execution_trigger_then_allOf_i3" role="tabpanel"> Type: object
-Must match regular expression: ^dbt-[a-z0-9]+([_-][a-z0-9]+)*$ - - - - - - -
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
+ then + + + + allOf + + + + item 3

+

event is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove event, or set type: event.

+
+

Must not be:

+
+
+ Type: array of enum (of string)
-

Capabilities the build asks for (acquisition pattern). Validation enforces capabilities βŠ† runner.declared_capabilities.

-
- - - - - -

All items must be unique

No Additional Items

Each item of this array must be:

-
-
- - - Type: enum (of string)
-
-

Must be one of:

-
  • "full_refresh"
  • "incremental_append"
  • "incremental_dedup"
  • "incremental_merge"
  • "cdc"
  • "streaming"
  • "schema_discovery"
  • "schema_evolution"
  • "dlp_scan"
  • "at_most_once"
  • "at_least_once"
  • "exactly_once"
-
- + item 3 + + + + not
Type: object
+ +
+

The following properties are required:

+
  • event
+
-
-
-
-
-
-
-
-

- -

-
+
+ -
-
+ + + + +
+ Type: string
- - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- - Type: object
-

Pattern-specific build properties with conditional validation.

-
- + allOf + + + + item 4

+

timetable is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove timetable, or set type: timetable.

+
+

Must not be:

+
+
- - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
Type: object
-

Build execution configuration (triggers, runtime, retries).

-
- - No Additional Properties - - - - - - -
-
-
-

- -

-
- -
-
- - Type: object
+ item 4 + + + + not
Type: object
+
+

The following properties are required:

+
  • timetable
+
+ + + + + +
+
+
+ - + + + + +
+ + + + + + +
@@ -53254,9 +56316,17 @@

/> type

Type: enum (of string)
-
+
+

NEW in v0.7.6: continuous, for a build that runs as a long-lived job on its own platform, such as an engine: flink job on the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, or a kafka-connect or debezium connector. The platform keeps the job running, so no scheduler starts runs: fluid generate schedule, fluid schedule-sync and the fluid apply DAGs leave the build out. A continuous trigger takes no other trigger, so cron, schedule, datasets, event and timetable are refused next to it, as Databricks Jobs refuses a continuous trigger next to any other.

+ +
+
+ +

Must be one of:

-
  • "schedule"
  • "event"
  • "manual"
  • "dependency"
  • "dataset"
  • "schedule_and_dataset"
  • "timetable"
+
  • "schedule"
  • "event"
  • "manual"
  • "dependency"
  • "dataset"
  • "schedule_and_dataset"
  • "timetable"
  • "continuous"
diff --git a/specs/0.7.7/fluid-spec.html b/specs/0.7.7/fluid-spec.html new file mode 100644 index 0000000..26f4b0e --- /dev/null +++ b/specs/0.7.7/fluid-spec.html @@ -0,0 +1,84060 @@ + + + + + + + + + + + + + + + + FLUID 0.7.7 β€” Preview: recursive streaming types + + + +

FLUID 0.7.7 β€” Preview: recursive streaming types

Type: object
+

Opt-in preview. Adds recursive struct fields, array elements and map key/value types. Stable 0.7.6 is unchanged.

+
No Additional Properties + + + + + +
+
Example:
+
{
+    "fluidVersion": "0.5.7",
+    "kind": "DataProduct",
+    "id": "gold.customer.analytics_360_v1",
+    "name": "Customer 360 Analytics",
+    "description": "Unified customer profiles with ML-driven insights",
+    "domain": "Customer Experience",
+    "tags": [
+        "customer-data",
+        "analytics",
+        "gold-layer"
+    ],
+    "labels": {
+        "team": "customer-analytics",
+        "criticality": "high",
+        "cost-center": "engineering"
+    },
+    "metadata": {
+        "layer": "Gold",
+        "owner": {
+            "team": "customer-analytics",
+            "email": "customer-analytics@company.com"
+        },
+        "businessContext": {
+            "domain": "Customer Experience",
+            "subdomain": "Customer Intelligence"
+        }
+    },
+    "exposes": [
+        {
+            "exposeId": "customer_profiles",
+            "title": "Unified Customer Profiles",
+            "version": "2.1.0",
+            "kind": "table",
+            "tags": [
+                "pii",
+                "customer-facing",
+                "real-time"
+            ],
+            "labels": {
+                "sensitivity": "high",
+                "retention": "7-years",
+                "region": "us-central1"
+            },
+            "contract": {
+                "schema": [
+                    {
+                        "name": "customer_id",
+                        "type": "STRING",
+                        "required": true,
+                        "description": "Unique customer identifier",
+                        "sensitivity": "cleartext",
+                        "tags": [
+                            "identifier",
+                            "primary-key"
+                        ],
+                        "labels": {
+                            "business-name": "Customer ID",
+                            "data-category": "identifier"
+                        }
+                    }
+                ]
+            },
+            "binding": {
+                "platform": "gcp",
+                "format": "bigquery_table",
+                "location": {
+                    "project": "company-data",
+                    "dataset": "gold_customer",
+                    "table": "profiles_v1"
+                }
+            }
+        }
+    ]
+}
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Vendor / plugin-namespaced configuration. Each plugin claims a single sub-key (e.g. customScaffold, customCatalog) and validates its own schema there via the fluidbuild.extensionvalidators entry-point.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Contract schema version. Accepts '0.7.3' (source-aligned data products + acquisition pattern + ingestion engines) or '0.7.4' (adds the MCP output-port gateway + runtime agentPolicy enforcement). 0.7.4 is fully backward-compatible with 0.7.3 β€” every 0.7.3 contract validates as 0.7.4 unchanged. '0.7.7' is the current preview.

+
+

Must be one of:

+
  • "0.7.3"
  • "0.7.4"
  • "0.7.5"
  • "0.7.6"
  • "0.7.7"
+
Must match regular expression: ^\d+\.\d+(\.\d+)?$ + + + + + +
+
Examples:
+
"0.7.4"
+
+
"0.7.3"
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Kind of contract. 'MLPipeline' added for basic ML support.

+
+

Must be one of:

+
  • "DataProduct"
  • "MLPipeline"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Stable product identifier, e.g. 'gold.hr.employee360v1'.

+
+ + Must match regular expression: ^[A-Za-z0-9_][A-Za-z0-9_.-]*[A-Za-z0-9_]$|^[A-Za-z0-9_]$ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + +

Must be at least 1 characters long

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+
+

NEW in v0.7.6: Either a plain string β€” unchanged meaning, it becomes purpose on export β€” or the structured object. While this was a string, an export of a valid contract could only ever carry description.purpose: usage and limitations had no home a contract could declare, so every ODCS and ODPS export left them empty. fluid odps import restores the object to the root description when the schema version the imported contract declares accepts an object, and also keeps a verbatim copy in extensions.odps.metadata.description so the ODPS leg stays an exact fixed point. All three fields therefore survive a FLUID β†’ ODPS β†’ FLUID cycle and reach both the ODCS and the ODPS export afterwards. Importing against a version whose root description is a string still roots purpose alone, with the object recoverable from that bucket.

+ +
+
+ +

+ +

+
+ + + Type: string
+

Prose description. Exported as description.purpose.

+
+ + + + + + +
+ + + Type: object
+

NEW in v0.7.6: The structured form of the root description. The three field names are Bitol's β€” ODPS v1.1.0 $defs.Description and the ODCS v3.1.0 root description both use exactly purpose/usage/limitations β€” so import and export are a pass-through rather than a translation.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

What this data product is for.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

How a consumer is expected to use it.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

What it must not be used for, and what it does not cover.

+
+ + + + + + +
+
+
+
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business domain/mesh domain (e.g., 'HR', 'Finance').

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Product-level tags for categorization and discovery.

+
+ + + + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+

Tag format: lowercase alphanumeric with hyphens, e.g., 'customer-data', 'pii', 'real-time'

+
Must match regular expression: ^[a-z0-9][a-z0-9-]*[a-z0-9]$|^[a-z0-9]$ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Product-level labels for metadata and automation.

+
+ + + + + + + +
+
Example:
+
{
+    "team": "customer-analytics",
+    "criticality": "high",
+    "cost-center": "engineering",
+    "retention": "7-years",
+    "region": "us-central1",
+    "business-name": "Customer Profiles",
+    "data-category": "analytical"
+}
+
+
+
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ + Type: string
+

Label values are always strings for consistency and tooling compatibility.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Generation envelope injected by fluid forge / fluid init β€” records how, when and by which tool/command the contract was generated. Machine-written (not user-authored); additive metadata that does not affect the data product's semantic contract.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Envelope schema version.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Envelope discriminator, e.g. 'ContractMetadata'.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: date-time
+

ISO 8601 UTC timestamp of generation.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Tool, version and command that generated the contract.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Data layer label (e.g., Bronze/Silver/Gold). Free-form. When set together with productType, the two MUST be consistent under the canonical mapping (Bronze↔SDP, Silver↔ADP, Gold↔CDP).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

NEW in v0.7.3: Data Mesh data-product type. SDP = Source-Aligned Data Product (raw ingestion from external systems; corresponds to Bronze). ADP = Aggregated Data Product (cross-source joined / domain-modelled; corresponds to Silver). CDP = Consumption-Aligned Data Product (analytics- or product-shaped; corresponds to Gold). Either metadata.layer or metadata.productType is sufficient β€” the validator infers the missing one. When both are present they MUST agree.

+
+

Must be one of:

+
  • "SDP"
  • "ADP"
  • "CDP"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

NEW in v0.7.3: Data classification label propagated to catalog + access policy enforcement.

+
+

Must be one of:

+
  • "public"
  • "internal"
  • "confidential"
  • "restricted"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

NEW in v0.7.3: Feature gates the contract opts into (e.g., 'acquisition' while Bronze acquisition is in beta).

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: email
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Oncall rotation or contact for operational issues.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.6: The team behind this product, as a roster rather than a single contact. metadata.owner holds one team name and one mailbox and carries no role, so a product with a data owner, a steward and two engineers had nowhere to say so and the exported ODCS/ODPS team block came out empty. The field names are Bitol's, so an imported team round-trips without a translation. Optional: a contract without it exports exactly as it does today.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Team name, e.g. 'Sales Analytics'.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

The people on the team, each with the role they hold on this product.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+

NEW in v0.7.6: One member of metadata.team.members. The field names and the username-only required list are Bitol's ODCS v3.1.0 / ODPS v1.1.0 $defs.TeamMember, copied verbatim so an imported team exports back unchanged.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

The member's identifier in the organisation's directory β€” the one field Bitol requires.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Display name.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

The role this member holds on this product, e.g. 'owner', 'steward', 'data engineer'. Free-form: Bitol does not constrain it.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: email
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: date-time
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Business alignment information for data mesh organization.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business domain this product belongs to.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business subdomain for more granular organization.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business capability this product supports.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Value stream this product participates in.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Product-level tags on metadata for discovery and categorization.

+
Same definition as tags +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+ + + + + + +

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + No Additional Properties + + + + + +
+
Example:
+
{
+    "exposeId": "customer_profiles",
+    "title": "Customer Profiles Table",
+    "kind": "table",
+    "tags": [
+        "pii",
+        "customer-facing",
+        "analytical"
+    ],
+    "labels": {
+        "sensitivity": "high",
+        "retention": "7-years",
+        "team": "customer-analytics"
+    },
+    "contract": {
+        "schema": [
+            {
+                "name": "customer_id",
+                "type": "STRING",
+                "required": true,
+                "sensitivity": "cleartext",
+                "tags": [
+                    "identifier",
+                    "primary-key"
+                ],
+                "labels": {
+                    "business-name": "Customer ID",
+                    "data-category": "identifier"
+                }
+            }
+        ]
+    },
+    "binding": {
+        "platform": "gcp",
+        "format": "bigquery_table",
+        "location": {
+            "project": "company-data",
+            "dataset": "customer",
+            "table": "profiles"
+        }
+    }
+}
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Stable interface handle (used by consumers & build.outputs).

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Detailed description of the exposed resource.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Semantic version (semver.org).

+
+ + Must match regular expression: ^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "table"
  • "view"
  • "api"
  • "file"
  • "stream"
  • "topic"
  • "feature_store"
  • "model"
  • "vector"
  • "graph"
  • "time_series"
  • "other"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Expose-level tags for categorization and routing.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Expose-level labels for metadata and automation.

+
Same definition as labels +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + No Additional Properties

+ +

+
+ + + Type: object
+ +
+

The following properties are required:

+
  • schema
+
+ + + + + +
+ + + Type: object
+ +
+

The following properties are required:

+
  • openapiRef
+
+ + + + + +
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.6: An explicit, stable contract id, used verbatim as the ODCS id and as the ODPS output port contractId. Without it the id is derived as <product id>.<exposeId>, so renaming the product or the expose silently changes the contract id and breaks every consumer reference to it, and a catalog that requires an id format of its own (a UUID, say) cannot be served at all. Each expose needs its own: two exposes sharing one id is refused by fluid validate.

+
+ + + +

Must be at least 1 characters long

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Hash of canonical schema/OpenAPI. Used by contract-tests gates.

+
Must match regular expression: ^sha256:[0-9a-fA-F]{64}$ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "strict"
+

NEW in v0.7.3: Schema evolution policy. strict = fail on any schema change; discoverandfreeze = capture from source on first run, fail thereafter; evolvesafe = accept additive changes; evolveall = accept all changes (cast on type narrow, route failures to DLQ).

+
+

Must be one of:

+
  • "strict"
  • "discover_and_freeze"
  • "evolve_safe"
  • "evolve_all"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

Tabular schema (for table/view/file).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array
+

Typed child columns of a struct/row/record.

+
+ + + + + +

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+Same definition as exposes_items_contract_schema_items +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Column data type. Accepts a canonical FLUID/SQL type name (case-insensitive) or a parameterized form such as decimal(18,4) or varchar(255). The enum lists the canonical bare types; the pattern additionally permits case variants and parameter suffixes. Intentionally generous β€” it rejects obvious garbage (e.g. MYSTERY_TYPE) while admitting the cross-dialect superset.

+
+ +

+ +

+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "string"
  • "text"
  • "varchar"
  • "varchar2"
  • "nvarchar"
  • "char"
  • "nchar"
  • "character"
  • "clob"
  • "int"
  • "integer"
  • "int2"
  • "int4"
  • "int8"
  • "int16"
  • "int32"
  • "int64"
  • "tinyint"
  • "smallint"
  • "mediumint"
  • "bigint"
  • "long"
  • "longint"
  • "serial"
  • "bigserial"
  • "float"
  • "float4"
  • "float8"
  • "float32"
  • "float64"
  • "double"
  • "real"
  • "decimal"
  • "dec"
  • "numeric"
  • "number"
  • "bignumeric"
  • "money"
  • "boolean"
  • "bool"
  • "bit"
  • "date"
  • "time"
  • "datetime"
  • "datetime2"
  • "smalldatetime"
  • "timestamp"
  • "timestamptz"
  • "timestamp_tz"
  • "timestamp_ntz"
  • "timestamp_ltz"
  • "timestampntz"
  • "interval"
  • "year"
  • "variant"
  • "object"
  • "array"
  • "struct"
  • "map"
  • "record"
  • "row"
  • "json"
  • "jsonb"
  • "super"
  • "binary"
  • "varbinary"
  • "bytes"
  • "blob"
  • "bytea"
  • "raw"
  • "geography"
  • "geometry"
  • "geom"
  • "point"
  • "uuid"
  • "uniqueidentifier"
  • "guid"
  • "enum"
  • "hll"
+
+ + + + + + +
+ + + Type: string
+Must match regular expression: (?i)^\s*(string|text|varchar|varchar2|nvarchar|char|nchar|character|clob|int|integer|int2|int4|int8|int16|int32|int64|tinyint|smallint|mediumint|bigint|long|longint|serial|bigserial|float|float4|float8|float32|float64|double|real|double\s+precision|decimal|dec|numeric|number|bignumeric|money|boolean|bool|bit|date|time|datetime|datetime2|smalldatetime|timestamp|timestamptz|timestamp_tz|timestamp_ntz|timestamp_ltz|timestampntz|timestamp\s+with(out)?\s+time\s+zone|interval|year|variant|object|array|struct|map|record|row|json|jsonb|super|binary|varbinary|bytes|blob|bytea|raw|geography|geometry|geom|point|uuid|uniqueidentifier|guid|enum|hll)\s*(\(\s*[0-9A-Za-z_,\s'\"-]*\s*\))?\s*$ + + + + + + +
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+ + + + + + +

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+Same definition as exposes_items_contract_schema_items +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+Same definition as element +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+Same definition as element +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+Same definition as element +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+Same definition as element +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+Same definition as element +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + +

Must be at least 1 characters long

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+

Column data type. Accepts a canonical FLUID/SQL type name (case-insensitive) or a parameterized form such as decimal(18,4) or varchar(255). The enum lists the canonical bare types; the pattern additionally permits case variants and parameter suffixes. Intentionally generous β€” it rejects obvious garbage (e.g. MYSTERY_TYPE) while admitting the cross-dialect superset.

+

+ +

+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "string"
  • "text"
  • "varchar"
  • "varchar2"
  • "nvarchar"
  • "char"
  • "nchar"
  • "character"
  • "clob"
  • "int"
  • "integer"
  • "int2"
  • "int4"
  • "int8"
  • "int16"
  • "int32"
  • "int64"
  • "tinyint"
  • "smallint"
  • "mediumint"
  • "bigint"
  • "long"
  • "longint"
  • "serial"
  • "bigserial"
  • "float"
  • "float4"
  • "float8"
  • "float32"
  • "float64"
  • "double"
  • "real"
  • "decimal"
  • "dec"
  • "numeric"
  • "number"
  • "bignumeric"
  • "money"
  • "boolean"
  • "bool"
  • "bit"
  • "date"
  • "time"
  • "datetime"
  • "datetime2"
  • "smalldatetime"
  • "timestamp"
  • "timestamptz"
  • "timestamp_tz"
  • "timestamp_ntz"
  • "timestamp_ltz"
  • "timestampntz"
  • "interval"
  • "year"
  • "variant"
  • "object"
  • "array"
  • "struct"
  • "map"
  • "record"
  • "row"
  • "json"
  • "jsonb"
  • "super"
  • "binary"
  • "varbinary"
  • "bytes"
  • "blob"
  • "bytea"
  • "raw"
  • "geography"
  • "geometry"
  • "geom"
  • "point"
  • "uuid"
  • "uniqueidentifier"
  • "guid"
  • "enum"
  • "hll"
+
+ + + + + + +
+ + + Type: string
+Must match regular expression: (?i)^\s*(string|text|varchar|varchar2|nvarchar|char|nchar|character|clob|int|integer|int2|int4|int8|int16|int32|int64|tinyint|smallint|mediumint|bigint|long|longint|serial|bigserial|float|float4|float8|float32|float64|double|real|double\s+precision|decimal|dec|numeric|number|bignumeric|money|boolean|bool|bit|date|time|datetime|datetime2|smalldatetime|timestamp|timestamptz|timestamp_tz|timestamp_ntz|timestamp_ltz|timestampntz|timestamp\s+with(out)?\s+time\s+zone|interval|year|variant|object|array|struct|map|record|row|json|jsonb|super|binary|varbinary|bytes|blob|bytea|raw|geography|geometry|geom|point|uuid|uniqueidentifier|guid|enum|hll)\s*(\(\s*[0-9A-Za-z_,\s'\"-]*\s*\))?\s*$ + + + + + + +
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Field-level sensitivity classification for improved policy ergonomics.

+
+ +
+

Must be one of:

+
  • "none"
  • "internal"
  • "confidential"
  • "restricted"
  • "pii"
  • "phi"
  • "cleartext"
  • "treated"
  • "anonymized"
  • "pseudonymized"
  • "tokenized"
  • "encrypted"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Semantic meaning (e.g., 'email', 'phone', 'identifier').

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business-friendly name for this field.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business definition and meaning of this field.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Field-level validation constraints.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "range"
  • "regex"
  • "enum"
  • "custom"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+
+

NEW in v0.7.6: Column-level data quality rules, in ODCS's own shape β€” the array that is written straight into schema[].properties[].quality[] of the exported ODCS v3.1.0 contract, and the array an imported ODCS column's rules are read back into, inline on the column rather than in the importer's pass-through bucket: the ODPS importer merges a resolved port's contract block into its output-port stub and discards the sub-import's root extensions, so rules left in that bucket were dropped outright by a FLUID β†’ ODPS β†’ FLUID cycle. The ODCS exporter has always replayed this key verbatim and always read it back on import, but $defs.column is additionalProperties: false and did not define it, so a column written the way the exporter supports was REFUSED by fluid validate: the only way to get a hand-written quality rule into a published contract was to write an invalid contract. Optional and additive β€” a column without it exports exactly as it does today, with the rules forge derives from required, the primary-key tag and validationRules. Setting it REPLACES those derived rules for that column rather than adding to them, which is what makes an imported ODCS column round-trip byte-for-byte. The authority for an item's contents is ODCS v3.1.0 $defs.DataQuality, deliberately not re-declared here (a copy would drift from the vendored schema, and fluid odcs validate already checks an emitted document against it); the keys below are the common ones. Every rule should carry an id: it is optional in ODCS, but a DQ engine that labels its results by rule id rejects a contract whose rules have none.

+ +
+
+ +
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Stable rule identifier, unique within this column's rules. The pattern is ODCS $defs.StableId. Rules forge derives itself are named <column>-<metric>.

+
Must match regular expression: ^[A-Za-z0-9_-]+$ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

ODCS quality check type. text = prose; library = one of ODCS's predefined metrics; sql = a query whose result is compared; custom = a named engine's own check.

+
+

Must be one of:

+
  • "text"
  • "library"
  • "sql"
  • "custom"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

type: library only. The five predefined metrics ODCS v3.1.0 defines; a name outside this list is not a library rule.

+
+

Must be one of:

+
  • "nullValues"
  • "missingValues"
  • "invalidValues"
  • "duplicateValues"
  • "rowCount"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

type: library only. Extra arguments for the metric β€” pattern (a regex) or validValues (a list) for invalidValues, as DataHub's ODCS ingestion source reads them.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

type: sql only. The query whose result the comparison operator is applied to.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

type: custom only. The engine that runs the check, e.g. soda, great-expectations, dbt.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

type: custom only. The check as that engine expresses it.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

The data quality dimension the rule measures.

+
+

Must be one of:

+
  • "accuracy"
  • "completeness"
  • "conformity"
  • "consistency"
  • "coverage"
  • "timeliness"
  • "uniqueness"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Severity of a failure. ODCS types this as a free string and documents info / warning / error by example.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Unit the comparison is in, usually rows (the default) or percent.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Field-level tags for categorization and data governance.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Field-level labels for metadata and automation.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+

OpenAPI document URL (for APIs).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Compatibility promise for non-breaking evolution.

+
+

Must be one of:

+
  • "none"
  • "patch"
  • "minor"
  • "backward"
  • "forward"
  • "strict"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of enum (of string)
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "additive_columns"
  • "additive_fields"
  • "additive_endpoints"
  • "nullable_to_required_disallowed"
  • "field_removal_with_notice"
  • "type_widening_allowed"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Enhanced data quality rules including anomaly detection.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "freshness"
  • "completeness"
  • "uniqueness"
  • "valid_values"
  • "accuracy"
  • "schema"
  • "anomaly_detection"
  • "drift_detection"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • ">="
  • ">"
  • "<="
  • "<"
  • "=="
  • "!="
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
+ + Must match regular expression: ^P(?!$)(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T(\d+H)?(\d+M)?(\d+S)?)?$ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "info"
  • "warn"
  • "error"
  • "critical"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

DQ rule tags for categorization and automation.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

DQ rule labels for metadata and routing.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Data quality specification tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Data quality specification labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Simplified SQL-expression data quality rules (alternate form of dq.rules for inline assertions).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Rule identifier.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

SQL or boolean expression that must evaluate to true.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Severity when the rule fails.

+
+

Must be one of:

+
  • "error"
  • "warning"
  • "info"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Contract-level tags for data quality and governance.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Contract-level labels for automation and metadata.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Availability percentage (e.g., 99.9%).

+
+ + Must match regular expression: ^(100(\.0+)?|\d{2}(\.\d+)?|\d{1}\d(\.\d+)?)%$ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

e.g., '0 rows'

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
+

Target completeness ratio (0.0 to 1.0).

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
+

Acceptable error rate (0.0 to 1.0).

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

QoS-level tags for SLA management.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

QoS-level labels for automation and reporting.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "oidc"
  • "oauth2"
  • "api_key"
  • "none"
  • "custom"
  • "iam"
  • "jwt"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+
+

Column-level access control. A column named in any restriction is restricted. 'deny': the principal may not read the columns. 'allow': the columns are readable only by the principals an allow names. A deny beats an allow, and a restriction never grants access: the readers are the expose's readers. Principals are logical and resolve through binding.principals. NEW in v0.7.6 (the field itself is older): fluid apply enforces it. GCP BigQuery: a Data Catalog taxonomy per product and dataset with fine-grained access control, a policy tag per set of restricted columns that share their readers, attached through the table schema's policyTags, and roles/datacatalog.categoryFineGrainedReader for exactly the expose's readers allowed to read them: the accessPolicy read grantees and the expose's own policy.authz.readers (a denied principal gets an access error on the columns). A restriction on an expose with no reader on the binding is refused, as it would lock the columns for everyone. A restriction's tags and labels are written into the policy tag's description; AWS has no place for them. AWS: each governance.lakeFormation grant with SELECT excludes the restricted columns its principal may not read (tablewithcolumns.excludedcolumnnames); a grant's hand-written excludedColumns must agree, and a binding with no Lake Formation grants is refused. fluid verify checks the policy tags and their readers on GCP, and the principals' Lake Formation permissions on AWS. NEW in v0.7.6: access: mask lets the principal read the columns masked by the platform (see mask): a deny beats a mask, and a mask beats an allow for the principal it names.

+ +
+
+ +
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + +
+ No Additional Properties

+ +

+
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "mask" + + + + + + +
+
+
+
+
+ + + +
+ + + Type: object
+ +
+

The following properties are required:

+
  • mask
+
+ + + + + +
+ + + + + + +
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ +
+

The following properties are required:

+
  • mask
+
+ + + + + +
+ + + +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "mask" + + + + + + +
+
+
+
+
+ + + + + + +
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Logical principal the restriction applies to, written as in accessPolicy (e.g. group:analysts@company.example); binding.principals maps it per cloud.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Columns of this expose's schema.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

deny: the principal may not read the columns. allow: only the principals an allow names may read them. mask: the principal reads the columns masked by the platform, by the rule in mask, and never their raw values (NEW in v0.7.6).

+
+

Must be one of:

+
  • "allow"
  • "deny"
  • "mask"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Access control rule tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Access control rule labels.

+
Same definition as labels +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

The masking rule an access: mask restriction applies, required with it and refused without it. lastfour: 'XXXXX' and the value's last four characters; firstfour: its first four characters and 'XXXXX' (a value of four characters or fewer is hashed instead, by each platform); nullify: NULL. GCP BigQuery: a data policy (DATAMASKINGPOLICY, LASTFOURCHARACTERS, FIRSTFOURCHARACTERS or ALWAYS_NULL) on the columns' policy tag, and roles/bigquerydatapolicy.maskedReader for the masked principals, who read the same column masked. AWS Glue: Lake Formation cannot mask, so the masked principals' grant on the table excludes the columns and they read a protected Glue Data Catalog view (SECURITY DEFINER, binding.governance.lakeFormation.maskedViews) that returns them masked by the same rule (NEW in v0.7.6).

+ +
+
+ +
+

Must be one of:

+
  • "last_four"
  • "first_four"
  • "nullify"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Authorization policy tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Authorization policy labels.

+
Same definition as labels +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+
+

Row-level access control (NEW in v0.7.6). Each filter lets its principal read only the rows its where predicate selects; a reader no filter names reads every row, and a principal may have one filter per expose. The predicate is one SQL boolean expression over this expose's columns, with no statement separator or comment. Principals are logical and resolve through binding.principals. GCP BigQuery: a row access policy (filterpredicate, grantees) per filter, plus one that selects every row for every other reader and writer, since a principal no row access policy names reads no row at all. AWS Glue: a Lake Formation data cells filter (rowfilter and the principal's excluded columns), and the principal's SELECT is granted on the filter instead of the table.

+ +
+
+ +
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Logical principal the filter applies to, written as in accessPolicy; binding.principals maps it per cloud.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

The filter's name on the platform (the row access policy or data cells filter); derived from the principal when omitted.

+
Must match regular expression: ^[a-z][a-z0-9_]{0,62}$ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

One SQL boolean expression over the expose's columns, e.g. consentdataanalytics = true AND is_suppressed = false.

+
+ + + +

Must be at least 1 characters long

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+
+

Columns to treat before they land. The DuckDB acquisition runner rewrites each named column inside the write, so the file, S3 object or BigQuery load and the DLQ receive the treated value; quality gates judge the source value first. A treated column lands as a string, so declare it with a string type (string, or VARCHAR under a schemaPolicy that compares source type names). A rule the build cannot apply (an unset salt or key, k_anonymity, a column the stream lacks, a non-string declared type, the incremental cursor) refuses the build rather than landing the column untreated. fluid verify checks every non-null value of a masked column for its strategy's shape, on a local file and, through Athena, on an S3+Glue table. The MCP output-port gateway drops masked columns from what it serves.

+ +
+
+ +
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

hash: lowercase hex SHA-256 of salt || value, deterministic so joins work (64 hex characters). mask: every character but the first params.keepFirst and the last params.keepLast replaced by '*', length preserved. tokenize: HMAC-SHA256 token, 32 hex characters, the token the tokenizepii preLand hook makes. encrypt: AES-GCM, written as 'aesgcm:v1:' + base64url(nonce || ciphertext || tag) with the column name as associated data; reversible with the key, not deterministic. kanonymity: accepted by the schema but refused at landing: it is a property of a whole table over its quasi-identifiers, not of one value.

+ +
+
+ +
+

Must be one of:

+
  • "mask"
  • "hash"
  • "tokenize"
  • "encrypt"
  • "k_anonymity"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Strategy parameters. The runner accepts exactly these keys and refuses any other, including a literal salt or key: secrets come only from the environment, never the contract, which is published with the plan, the bundle and the Glue table's fluid_contract parameter. Contracts on 0.7.5 and earlier may use the same keys; their schemas leave params untyped, and the runner checks them at build time.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string Default: "FLUID_PII_HASH_SECRET"
+

NEW in v0.7.6: hash only. The environment variable holding the salt, at least 16 bytes. Unset or empty, the build is refused: an unsalted SHA-256 of a phone number is reversed by enumerating the numbering plan.

+
Must match regular expression: ^[A-Z_][A-Z0-9_]{0,127}$ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.6: tokenize and encrypt. The environment variable holding the key. tokenize defaults to FLUIDPIITOKENIZATIONKEY and needs at least 32 bytes; encrypt defaults to FLUIDPIIENCRYPTIONSECRET_KEY and needs the base64 of a 16, 24 or 32-byte AES key (openssl rand -base64 32). Unset or empty, the build is refused.

+
Must match regular expression: ^[A-Z_][A-Z0-9_]{0,127}$ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer Default: 0
+

NEW in v0.7.6: mask only. Leading characters left as they are.

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 64

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer Default: 4
+

NEW in v0.7.6: mask only. Trailing characters left as they are, so +46701234567 lands as ********4567. A value no longer than keepFirst + keepLast is masked entirely.

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 64

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Privacy masking rule tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Privacy masking rule labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Provider-specific predicate

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Row-level policy tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Row-level policy labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Privacy policy tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Privacy policy labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "Public"
  • "Internal"
  • "Confidential"
  • "Restricted"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.1: AI/LLM usage governance - controls which AI models can consume this data and for what purposes.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Whitelist of AI models permitted to consume this data. Examples: ['gpt-4', 'claude-3-opus', 'gemini-pro']. Empty array = no AI access.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+Must match regular expression: ^[a-z0-9][a-z0-9-_.]*[a-z0-9]$|^[a-z0-9]$ + + + + + +
+
Examples:
+
"gpt-4"
+
+
"claude-3-opus"
+
+
"claude-3-sonnet"
+
+
"gemini-pro"
+
+
"gemini-ultra"
+
+
"llama-3"
+
+
"mixtral-8x7b"
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Blacklist of AI models explicitly prohibited from consuming this data. Takes precedence over allowedModels.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Maximum tokens per AI request. Prevents excessive data exposure in single queries.

+
+ + + +

Value must be greater or equal to 1

+ +
+
Examples:
+
4096
+
+
8192
+
+
16384
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Daily token limit for AI consumption. Enforces usage quotas.

+
+ + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of enum (of string)
+

Permitted AI use cases. Controls whether data can be used for training, inference, reasoning, etc.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "inference"
  • "reasoning"
  • "analysis"
  • "summarization"
  • "classification"
  • "embedding"
  • "search"
  • "qa"
  • "code_generation"
  • "fine_tuning"
  • "training"
  • "rag"
+
+ + + + + + +
+

+
Examples:
+
[
+    "inference",
+    "reasoning",
+    "summarization"
+]
+
+
[
+    "search",
+    "qa",
+    "rag"
+]
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of enum (of string)
+

Prohibited AI use cases. Example: Allow reporting but deny model training.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "inference"
  • "reasoning"
  • "analysis"
  • "summarization"
  • "classification"
  • "embedding"
  • "search"
  • "qa"
  • "code_generation"
  • "fine_tuning"
  • "training"
  • "rag"
+
+ + + + + + +
+

+
Example:
+
[
+    "training",
+    "fine_tuning"
+]
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: false
+

Whether AI agents can use this data for multi-step reasoning/chain-of-thought. False = simple retrieval only.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: false
+

Whether AI systems can cache/store this data. False = ephemeral processing only.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Data retention requirements for AI systems consuming this data.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Maximum days AI system can retain this data. 0 = no retention allowed.

+
+ + + +

Value must be greater or equal to 0

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+

Whether AI system must delete data after use.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+

Whether AI consumption must be logged/audited.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Free-text description of allowed purposes. Example: 'Customer service chatbot only, not for marketing'.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Agent policy tags for categorization.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Agent policy labels for automation.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Policy-level tags for governance.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Policy-level labels for automation.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + No Additional Properties

+ +

+
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "gcp" + + + + + + +
+
+
+
+
+ + + +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ +
+
+

Must not be:

+
+
+ + + Type: object
+Must match regular expression: ^(alias/|arn:) + + + + + + +
+
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ +
+

+ +

+
+ + + Type: string
+Must match regular expression: ^(user|group|serviceAccount|domain):.+$ + + + + + + +
+ + + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+Must match regular expression: ^(user|group|serviceAccount|domain):.+$ + + + + + + +
+
+
+ + + + + + +
+
+
+
+
+
+
+
+
+ + + + + + +
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "aws" + + + + + + +
+
+
+
+
+ + + +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ +
+
+

Must not be:

+
+
+ + + Type: object
+Must match regular expression: ^projects/ + + + + + + +
+
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ +
+

+ +

+
+ + + Type: string
+Must match regular expression: ^arn:aws[a-z0-9-]*:iam:: + + + + + + +
+ + + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+Must match regular expression: ^arn:aws[a-z0-9-]*:iam:: + + + + + + +
+
+
+ + + + + + +
+
+
+
+
+
+
+
+
+ + + + + + +
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Target platform. NEW in v0.7.5: 'pgvector' for the vector/embeddings output port (pgvector-on-PostgreSQL RAG target — emits the embeddings table + ANN index DDL for the ai-embeddable columns). NEW in v0.7.5: 'confluent' for the Confluent Cloud Tableflow managed Kafka→Iceberg emitter. NEW in v0.7.4: 'postgres' for the MCP output-port PostgreSQL driver (AWS Athena uses platform 'aws').

+
+

Must be one of:

+
  • "gcp"
  • "aws"
  • "azure"
  • "snowflake"
  • "databricks"
  • "kafka"
  • "confluent"
  • "local"
  • "kubernetes"
  • "postgres"
  • "pgvector"
  • "other"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Binding format. NEW in v0.7.5: 'pgvectortable' β€” a pgvector embeddings table target for the vector output port. NEW in v0.7.4: 'postgrestable' (PostgreSQL driver), 'athenatable' / 'gluetable' (AWS Athena driver) for the MCP output port.

+
+

Must be one of:

+
  • "bigquery_table"
  • "snowflake_table"
  • "snowflake_view"
  • "gcs_file"
  • "s3_file"
  • "http_api"
  • "grpc_api"
  • "pubsub_topic"
  • "kafka_topic"
  • "delta_table"
  • "iceberg"
  • "parquet"
  • "csv"
  • "json"
  • "redshift_table"
  • "redshift_serverless"
  • "redshift_external_schema"
  • "postgres_table"
  • "athena_table"
  • "glue_table"
  • "pgvector_table"
  • "other"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Apache Iceberg table format configuration

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of integer)
+

Iceberg table format version

+
+

Must be one of:

+
  • 1
  • 2
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Underlying file format for data files

+
+

Must be one of:

+
  • "parquet"
  • "orc"
  • "avro"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Partition transformation specifications

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Partition field name

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Source column for partitioning

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Partition transform function

+
+

Must be one of:

+
  • "identity"
  • "year"
  • "month"
  • "day"
  • "hour"
  • "bucket"
  • "truncate"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Number of buckets (for bucket transform)

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Truncate width (for truncate transform)

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Sort order specifications for data files

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Column to sort by

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Sort direction

+
+

Must be one of:

+
  • "asc"
  • "desc"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

NULL value ordering

+
+

Must be one of:

+
  • "nulls-first"
  • "nulls-last"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Iceberg table properties

+
+ + + + + + +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.5: vector / embeddings output-port configuration (platform 'pgvector', format 'pgvectortable'). Consumed by fluidbuild.output_ports.vector to emit the pgvector embeddings-table + ANN-index DDL and a RAG provenance manifest for the schema's ai-embeddable columns.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Embedding vector width (e.g. 1536 for OpenAI text-embedding-3-small, 1024 for Cohere embed-english-light). Maps to the pgvector column type vector(<dimensions>).

+
+ + + +

Value must be greater or equal to 1 and lesser or equal to 16000

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Provenance: the embedding model that produces the vectors (e.g. 'text-embedding-3-small'). Recorded in the RAG manifest and the embeddings-table's embedding_model column so a retriever can verify model parity.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "vector"
+

pgvector column type: 'vector' (float32, up to 2000 dims) or 'halfvec' (float16, half the storage, up to 4000 dims).

+
+

Must be one of:

+
  • "vector"
  • "halfvec"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "hnsw"
+

ANN index built on the embedding column. 'hnsw' (higher recall / lower query latency, buildable before data lands), 'ivfflat' (lower build cost, needs data first), or 'none' (no index β€” exact search only).

+
+

Must be one of:

+
  • "hnsw"
  • "ivfflat"
  • "none"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "cosine"
+

Distance function the index optimises for → pgvector operator class: cosine→vectorcosineops, l2→vectorl2ops, innerproduct→vectoripops, l1→vectorl1_ops.

+
+

Must be one of:

+
  • "cosine"
  • "l2"
  • "inner_product"
  • "l1"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Target embeddings table name. Defaults to '<exposeId>_embeddings' when omitted.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Provenance: the source primary-key column that ties each embedding row back to its origin row (written to the embeddings-table's source_key column). Enables update/delete propagation from the source product to the vector store.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

HNSW index tuning (used when indexType is 'hnsw').

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Max bi-directional links per node (pgvector default 16).

+
+ + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Build-time dynamic candidate-list size (pgvector default 64).

+
+ + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

IVFFlat index tuning (used when indexType is 'ivfflat').

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Number of inverted lists (pgvector guidance: rows/1000 for < 1M rows).

+
+ + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Provider-native addressing with tightened structure for better interop.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Cloud account ID

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

GCP project ID

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

BigQuery dataset or database name

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Database name

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Schema name

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Table name

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Storage bucket name

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

File or object path

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

API gateway endpoint

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.6: Host or endpoint; for a Kafka binding, the bootstrap servers. The ODCS server mapper already reads host and falls back to account, so until now the only valid way to set the bootstrap servers was a field named for something else β€” and with neither set, the exported ODCS server is the placeholder ${KAFKA_BOOTSTRAP_SERVERS}. Usually set per environment in an overlay rather than in the base contract.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+

Base URL for API endpoints

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Pub/Sub or Kafka topic name

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Pub/Sub subscription name

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Cloud region

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Cloud zone

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.5: Confluent Cloud environment id (env-xxxxx) for the Tableflow emitter.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.5: Confluent Cloud Kafka cluster id (lkc-xxxxx) the Tableflow topic belongs to.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.5: AWS Kinesis Data Stream name β€” maps to awskinesisstream in the streaming IaC emitter.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.5: Amazon Redshift Serverless namespace name β€” maps to awsredshiftserverlessnamespace.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.5: Amazon Redshift Serverless workgroup name β€” maps to awsredshiftserverlessworkgroup.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.5: ARN of the IAM role attached to the Redshift Serverless namespace / used by the redshift-data CREATE EXTERNAL SCHEMA bridge.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.5: Redshift external (Spectrum) schema name created over a Glue database via the redshift-data CREATE EXTERNAL SCHEMA bridge.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.5: Glue Data Catalog database that backs the Redshift external schema.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.5: ARN of the pre-created AWS IAM role Confluent Tableflow assumes (byobaws). Its trust policy is updated post-apply with the provider integration's externalid (two-phase IAM).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+
+

NEW in v0.7.5: The Iceberg catalog that owns this table: glue, rest, lakekeeper, polaris, unity, nessie, bigquery, hive, jdbc, hadoop, dynamodb or snowflake-managed. Spellings fold case and '-'/'', so icebergrest == rest and snowflake == snowflake-managed; fluid validate refuses any other value on an Iceberg expose. When absent: glue on platform aws, snowflake-managed on snowflake, rest elsewhere. The REST catalogs (rest, lakekeeper, polaris, unity) need location.uri plus location.warehouse as the catalog's warehouse NAME, not an object-store URI; Lakekeeper serves its Iceberg REST API under <host>/catalog (uri: http://lakekeeper:8181/catalog). On platform aws, a kind other than glue gets no Glue table and no Glue grant: that catalog creates and governs the table. Every emitter (streaming sinks, dbt catalogs.yml, the IaC, policy compile) reads one table, providers/icebergcatalog.py.

+ +
+
+ +
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.5: Iceberg warehouse β€” the canonical Iceberg REST 'warehouse' property (maps to iceberg.catalog.warehouse): an object-store URI (s3://|gs://|abfss://) for object-store catalogs, or the catalog/warehouse name for a REST catalog.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.5: REST Iceberg catalog endpoint URL β€” the canonical Iceberg REST 'uri' property (maps to iceberg.catalog.uri).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+
+

NEW in v0.7.5: Iceberg partition columns β€” a FLUID abstraction over the Iceberg PARTITIONED BY / PartitionSpec (maps to iceberg.tables.default-partition-by). On a gcp BigQuery table with exposes[].lifecycle.expire: true (fluid-schema 0.7.6), one DATE, TIMESTAMP or DATETIME column to partition by (time_partitioning.field); without it the table is partitioned by ingestion time. With a column, retention counts from the date in that column, not from landing: BigQuery deletes a partition retention after its date, so a backfill of older rows is deleted at once. Leave it out for the S3 rule's semantics (age since written).

+ +
+
+ +
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Binding configuration tags for infrastructure.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Binding configuration labels for automation.

+
Same definition as labels +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Provider-specific binding properties (e.g., cluster_by, comment for Snowflake).

+
+ + + + + + +
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Per-resource governance: registerLocation, principal grants, tag associations, row/column filters. Account-wide governance (admins, tag definitions) lives at the contract top-level governance block.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Per-resource LF settings: location registration, principal grants, LF-tag associations, row/column filters. Applies only to formats backed by S3 + Glue catalog (iceberg/parquet/csv/json/avro/orc/delta).

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: false
+

When true, emit awslakeformationresource that registers the binding's S3 path with Lake Formation. Required before LF can manage access to objects under that path.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "cross-account"
+
+

NEW in v0.7.6: Which grants[] principals get a statement in the awss3bucketpolicy emitted beside the grants (s3:ListBucket and s3:GetBucketLocation on the bucket, s3:GetObject under location.path). A statement lets its principal read the objects straight from S3, which skips Lake Formation's column, row and cell filters and its revocations. 'cross-account' (the default): only principals whose ARN names an AWS account other than the one running the apply, decided at plan time against the caller identity; same-account grantees get no statement, since on a registered location Lake Formation vends them credentials. 'none': no bucket policy at all; the tightest setting when the location is registered (registerLocation) and every reader, cross-account ones included, queries through a Lake Formation integrated engine, or when the bucket policy is managed elsewhere. 'all-grantees': every grantee, same-account included, which is the output before this field existed and reopens the direct S3 read path. An emitted awss3bucketpolicy is authoritative: it replaces every other statement on the bucket. When no grantee is left none is emitted.

+ +
+
+ +
+

Must be one of:

+
  • "cross-account"
  • "none"
  • "all-grantees"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Principal-based LF grants on this exposure's database/table. Each entry maps a principal to a permission set (and optional column projection). Emitted as awslakeformationpermissions resources. Field idiom adapted from ODCS v3 roles[] (principal + access-type), bound to IAM ARNs since LF is cloud-specific.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

IAM principal ARN (role or user) receiving the grant.

+
Must match regular expression: ^arn:aws[a-z0-9-]*:iam:: + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of enum (of string)
+

LF permissions granted. Valid values per the hashicorp/aws awslakeformationpermissions resource.

+
+ + + + + +

Must contain a minimum of 1 items

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "ALL"
  • "SELECT"
  • "ALTER"
  • "DROP"
  • "DELETE"
  • "INSERT"
  • "DESCRIBE"
  • "CREATE_TABLE"
  • "CREATE_DATABASE"
  • "DATA_LOCATION_ACCESS"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Subset of permissions the principal can re-grant. Maps to awslakeformationpermissions.permissionswithgrant_option.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Optional column-level restriction. When set, the grant is emitted against awslakeformationpermissions.tablewithcolumns instead of .table, allowing the principal to read only the listed columns.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Opposite of columns: every column EXCEPT these is granted. Maps to tablewithcolumns.excludedcolumnnames. Mutually exclusive with columns.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+
+

LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as awslakeformationresourcelftags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).

+ +
+
+ +
+ + + + + + +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Row-level (and optionally column-level) filter applied to all reads of this table via LF. Emitted as awslakeformationdatacellsfilter. The filter must then be granted to principals via grants[].permissions including SELECT β€” LF enforces both layers.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Filter name (must be unique per table).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

PartiQL-flavoured row predicate. ALL lets every row through (use when only column filtering is intended).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Columns visible through the filter. Mutually exclusive with allColumns / excludedColumnNames.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Columns hidden through the filter (everything else is visible).

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: false
+

Convenience for the LF column-wildcard form (every column visible). When true, columnNames / excludedColumnNames must be omitted.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Where the protected Glue Data Catalog views of the expose's masked columns come from (NEW in v0.7.6). Lake Formation cannot mask, so a principal an access: mask column restriction names reads a protected, SECURITY DEFINER multi-dialect view, <table>_masked, that returns the columns masked by the restriction's rule and applies its row filter; its grant on the table excludes the columns. The definer is granted SELECT with grant option on the table and DESCRIBE on the database; the principal SELECT and DESCRIBE on the view.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ARN of the IAM role the views are defined as: Glue and Lake Formation must be allowed to assume it (sts:AssumeRole, sts:SetContext), and the applying principal to pass it (iam:PassRole).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

The Glue connection of type VIEWVALIDATIONATHENA (WORKGROUP_NAME) Glue validates each view's SQL through; the applying principal and the definer need glue:PassConnection on it.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.6: Per-exposure packaging override β€” key-wise precedence over the contract-wide top-level packaging block (binding.packaging > packaging > absent-legacy).

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

NEW in v0.7.6: Blanket ownership mode for every container kind. Default when the block is present: 'isolated'. Per-kind exceptions go in containers.

+
+

Must be one of:

+
  • "isolated"
  • "shared"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.6: Pool id of the platform-owned tenant pool this product writes into. REQUIRED (enforced by the resolver) whenever any container resolves 'shared' β€” a pool must be addressable. Propagated as the fluid_pool label/tag for cost attribution.

+
+ + + +

Must be at least 1 characters long

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.6: Optional path to the platform team's pool manifest file; snapshotted into the bundle at stage 1 so bundleDigest covers it (any pool-file edit invalidates downstream plans).

+
+ + + +

Must be at least 1 characters long

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.6: Per-container-kind ownership overrides β€” each key wins over mode for that kind, yielding hybrid tiers. Kind↔platform mapping: bucket β†’ awss3bucket / googlestoragebucket; database β†’ snowflakedatabase AND awsgluecatalogdatabase; dataset β†’ googlebigquerydataset; schema β†’ snowflakeschema; warehouse β†’ snowflakewarehouse; cluster β†’ Confluent environment/cluster (v1: shared only).

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

NEW in v0.7.6: Ownership of the object-store bucket (awss3bucket / googlestoragebucket).

+
+ +
+

Must be one of:

+
  • "isolated"
  • "shared"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

NEW in v0.7.6: Ownership of the database container β€” covers Snowflake database AND AWS glue_database.

+
Same definition as bucket +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

NEW in v0.7.6: Ownership of the BigQuery dataset (googlebigquerydataset).

+
Same definition as bucket +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

NEW in v0.7.6: Ownership of the Snowflake schema (snowflake_schema).

+
Same definition as bucket +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

NEW in v0.7.6: Ownership of the Snowflake warehouse (snowflake_warehouse) β€” isolated gives per-product cost attribution.

+
Same definition as bucket +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

NEW in v0.7.6: Ownership of the Confluent environment/cluster. v1 accepts 'shared' only β€” dedicated-cluster provisioning is not yet supported (the resolver rejects 'isolated').

+
Same definition as bucket +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.6: Encryption at rest for an AWS S3 binding or a GCP BigQuery binding. Absent: nothing is emitted, and the cloud's default applies: SSE-S3 for new S3 objects, Google-managed keys for BigQuery.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Default: "product"
+
+

NEW in v0.7.6: The KMS key the bucket's objects are encrypted with (SSE-KMS with an S3 Bucket Key). 'product' (the default): fluid apply creates a customer managed key for each bucket this product owns (awskmskey, rotation on, alias alias/fluid/<contract id>/<bucket>) and makes it the bucket's default encryption (awss3bucketserversideencryptionconfiguration). Its key policy lets the account's IAM policies decide who may use it (the KMS default statement for the account root) and, when the binding carries governance.lakeFormation, lets the Lake Formation service-linked role AWSServiceRoleForLakeFormationDataAccess encrypt and decrypt, matched by aws:PrincipalArn so the role need not exist yet. A principal querying through Athena with credentials Lake Formation vends then needs no KMS permission of its own; a grantee in another account that the bucket policy lets read the objects directly (governance.lakeFormation.bucketPolicy) is also allowed kms:Decrypt, through S3 only. The writer (the identity the build runs as) needs kms:GenerateDataKey and kms:Decrypt from IAM. tofu destroy deletes the alias and schedules the key for deletion after 7 days, the minimum KMS allows; kms:CancelKeyDeletion recovers it until then. Dropping the block, or naming another key, removes the product key while the bucket keeps the objects encrypted under it, which become unreadable once the deletion window ends unless they are rewritten or the deletion is cancelled; fluid apply refuses that plan without --allow-data-loss. On a shared (pool) bucket the bucket's default encryption is the pool owner's, so 'product' is refused there. 'alias/<name>' or a key or alias ARN: an existing key, looked up at plan time (data awskmskey, so the identity running fluid apply needs kms:DescribeKey on it) and made the default encryption of a bucket this product owns by its key ARN, since S3 resolves an alias in the account of whoever writes; the plan fails unless the key is Enabled and a symmetric encryption key (SYMMETRICDEFAULT). On a pool bucket it is only checked. Its key policy is its owner's, and must let the Lake Formation role use it when the location is registered: an AWS managed key cannot be used with that role, so alias/aws/s3 is refused with registerLocation, and the plan fails for any key that is not customer managed. 'none': nothing is emitted. fluid verify checks that the key is Enabled and that the objects under the binding's prefix are SSE-KMS with it. On a gcp binding (fluid-schema 0.7.6, BigQuery tables): 'product' creates a Cloud KMS key ring and crypto key for each dataset this product writes (googlekmskeyring fluid-<contract id>-<dataset> and googlekmscryptokey 'bigquery', in the dataset's location, EU and US mapping to the europe and us key locations, rotated every 90 days), grants the project's BigQuery service agent roles/cloudkms.cryptoKeyEncrypterDecrypter on it, and makes it the dataset's defaultencryptionconfiguration and the table's encryptionconfiguration. The project must have the Cloud KMS API (cloudkms.googleapis.com) enabled. A key ring and key cannot be deleted on GCP: tofu destroy removes them from state and schedules the key's versions for destruction (30 days by default), and a later apply adopts the same names. 'projects/<p>/locations/<l>/keyRings/<r>/cryptoKeys/<k>': an existing key in the dataset's location, used as given; its owner grants the BigQuery service agent. 'none': Google-managed keys. An alias/... or arn:... value is refused on a gcp binding, and a Cloud KMS name on an aws binding. Adding a key to a table that exists replaces the table (BigQuery cannot re-key it in place), so fluid apply refuses that plan without --allow-data-loss; the next build lands the data again. fluid verify checks kmsKeyName on the dataset and the table.

+ +
+
+ +

+ +

+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "product"
  • "none"
+
+ + + + + + +
+ + + Type: object
+Must match regular expression: ^alias/[A-Za-z0-9/_-]{1,250}$ + + + + + + +
+ + + Type: object
+Must match regular expression: ^arn:aws[a-z-]*:kms:[a-z0-9-]+:[0-9]{12}:(key|alias)/[A-Za-z0-9/_-]{1,250}$ + + + + + + +
+ + + Type: object
+Must match regular expression: ^projects/[a-z0-9.:-]{1,100}/locations/[a-z0-9-]{1,63}/keyRings/[A-Za-z0-9_-]{1,63}/cryptoKeys/[A-Za-z0-9_-]{1,63}$ + + + + + + +
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+
+

NEW in v0.7.6: Maps the contract's LOGICAL principals (accessPolicy.grants[].principal and exposes[].policy.authz.columnRestrictions[].principal, keyed exactly as the contract writes them) to the identities they stand for on this binding's platform, so the base contract stays cloud-neutral and each environment's overlay binds it. On gcp an identity is an IAM member (user:, group:, serviceAccount: or domain:); on aws an IAM principal ARN. A value is one identity or a list; [] says the principal has no identity on this cloud, and nothing is granted to it there. When the block is present, every principal the contract names for this expose must be a key: an unmapped one is refused, never emitted as written. Without the block the principals are used as written, except that a GCP principal in a reserved top-level domain (.example, .test, .invalid, .localhost) is refused as a placeholder. Modelled on ODCS v3's roles[] bound per server (servers[].roles) and on dbt grants resolved per target.

+ +
+
+ +
+ + + + + + +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ +
+

+ +

+
+ + + Type: string
+ + + + +

Must be at least 1 characters long

+ + +
+ + + Type: array of string
+ + + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + +

Must be at least 1 characters long

+ + +
+
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.6: How message VALUES (and keys) are serialized on the wire. A Kafka binding today declares only format: kafka_topic, which tells a consumer nothing about whether the values are Avro or JSON β€” the one thing it has to know to read the topic at all. The first four values are the message formats ODCS v3.1.0 documents for its own Kafka server format field; the two -confluent values name those same wire formats carried through a Confluent schema registry, spelled as Apache Flink spells them.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

How message values are serialized.

+
+

Must be one of:

+
  • "json"
  • "avro"
  • "protobuf"
  • "xml"
  • "avro-confluent"
  • "json-confluent"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

How message keys are serialized, for a topic whose keys are not opaque bytes.

+
+

Must be one of:

+
  • "json"
  • "avro"
  • "protobuf"
  • "xml"
  • "avro-confluent"
  • "json-confluent"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.6: An expose's lifecycle. The same fields as the contract-root lifecycle, plus expire, which is only meaningful where there is storage to expire.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Unified lifecycle state vocabulary.

+
+ +
+

Must be one of:

+
  • "preview"
  • "active"
  • "deprecated"
  • "retired"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

How long this expose's data is kept (ISO-8601 duration, e.g. P30D, P7Y). A declaration unless expire is true. With expire: true on an AWS S3 binding, fluid apply expires the objects under the binding's prefix this long after they are written; on a GCP BigQuery table it deletes each daily partition this long after its day ends. Years and months are counted as 365 and 30 days, and a part of a day rounds up to a whole one.

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: false
+
+

NEW in v0.7.6: Delete stored data once it is older than retention. Default false, so a contract that declares a retention period does not start deleting data when forge-cli is upgraded. Honoured by the AWS emitter on an S3 binding and by the GCP emitter on a BigQuery table; on a gcp binding that is not a BigQuery table (GCS, Pub/Sub, Iceberg storage) it is refused rather than dropped. AWS: fluid apply writes one rule per expose into the bucket's awss3bucketlifecycleconfiguration, filtered to the binding's prefix (location.path, else {database}/{table}/; a binding with neither is refused, so no rule ever expires a whole bucket). The rule expires the current version retention after it is written, removes a noncurrent version (on a versioning-enabled bucket) one day after it stops being current, and aborts an incomplete multipart upload after 7 days or the retention period if shorter. The lifecycle configuration is AUTHORITATIVE for the whole bucket: S3 keeps one per bucket, so rules that fluid apply did not write are replaced. It is therefore written only for a bucket this product owns; on a shared (pool) bucket (packaging) nothing is written, the pool's owner must hold the rule, and fluid verify checks that an enabled rule covering the prefix expires objects after exactly this period. Objects already older than the period when the rule is first applied are expired on S3's next lifecycle run, typically within a day. Removing expire later deletes the lifecycle configuration, which fluid apply refuses without --allow-data-loss (it refuses every plan that removes a resource holding data or policy; removing an access grant or a policy tag is a revocation and is not gated); tofu destroy deletes it. GCP (fluid-schema 0.7.6): the BigQuery table is partitioned by day (timepartitioning type DAY, on binding.location.partitionBy when it names one date or timestamp column, else by ingestion time) with expirationms = retention in days, so BigQuery deletes each partition that long after its day ends. BigQuery counts from the partition's date, not from when its rows were written: by ingestion time (no partitionBy, the default and the S3 rule's semantics) no row is deleted sooner than retention after it landed; with partitionBy, retention is the age of the date in that column, so a backfill of rows whose date is already older than retention lands in expired partitions and is deleted at once. It is never a whole-table expiration, which would delete the product. BigQuery cannot partition a table that exists, so adding expire to a live table replaces it: fluid apply refuses that plan without --allow-data-loss, and the next build lands the data again. A later change of retention is an in-place update. Removing expire from a live table removes the partitioning's replacement trigger, which fluid apply also refuses without --allow-data-loss; BigQuery cannot un-partition a table, so to keep data longer set a longer retention instead. fluid verify checks the live table's partition expiration.

+ +
+
+ +
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Reference to replacement data product or expose.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Deprecation policy tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Deprecation policy labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Lifecycle tags for automation.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Lifecycle labels for governance.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Human/machine-readable SLI check description

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "slack"
  • "email"
  • "webhook"
  • "pagerduty"
  • "custom"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "info"
  • "warning"
  • "critical"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Alert configuration tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Alert configuration labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Default SLI trio (freshness, completeness, latency) by kind.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+

Enable auto-generation of default SLIs.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Default freshness SLI configuration.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Freshness threshold (auto-set by kind: table=PT6H, stream=PT1M, api=PT1S)

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "warning"
+
+

Must be one of:

+
  • "info"
  • "warning"
  • "critical"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Freshness SLI tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Freshness SLI labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Default completeness SLI configuration.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number Default: 0.95
+

Completeness threshold (auto-set by kind: table=0.95, stream=0.99)

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "warning"
+
+

Must be one of:

+
  • "info"
  • "warning"
  • "critical"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Completeness SLI tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Completeness SLI labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Default latency SLI configuration.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

P95 latency threshold (auto-set by kind: api=PT200MS, table=PT5S, stream=PT100MS)

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number Default: 95
+

Latency percentile to measure.

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 100

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "warning"
+
+

Must be one of:

+
  • "info"
  • "warning"
  • "critical"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Latency SLI tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Latency SLI labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Default SLI configuration tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Default SLI configuration labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Alert dispatcher config consumed by the acquisition Alerter (DLQ overflow, schema-fingerprint changes, anomaly signals).

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "log"
  • "file"
  • "webhook"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

For kind=file: NDJSON sink path (env-vars expanded).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+

For kind=webhook: Slack-compatible incoming webhook URL. Validated against the SSRF guard at construction time.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Observability configuration tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Observability configuration labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Documentation tags for organization.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Documentation labels for automation.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.2: Semantic model definition mapping physical columns to business concepts. Defines entities, measures, dimensions, and metrics for agent-consumable business logic. Eliminates semantic hallucination by providing machine-readable KPI definitions. Compatible with dbt MetricFlow, Snowflake Semantic Views, and OSI-aligned formats.

+
+ + No Additional Properties + + + + + +
+
Example:
+
{
+    "name": "orders_revenue_model",
+    "description": "Canonical semantic model for revenue analytics",
+    "defaultAggTimeDimension": "order_date",
+    "entities": [
+        {
+            "name": "order",
+            "type": "primary",
+            "expr": "order_id"
+        },
+        {
+            "name": "customer",
+            "type": "foreign",
+            "expr": "customer_id"
+        }
+    ],
+    "measures": [
+        {
+            "name": "order_amount",
+            "agg": "sum",
+            "expr": "amount",
+            "description": "Sum of all order amounts"
+        },
+        {
+            "name": "order_count",
+            "agg": "count_distinct",
+            "expr": "order_id",
+            "createMetric": true
+        }
+    ],
+    "dimensions": [
+        {
+            "name": "order_date",
+            "type": "time",
+            "expr": "completed_at",
+            "typeParams": {
+                "timeGranularity": "day"
+            }
+        },
+        {
+            "name": "region",
+            "type": "categorical"
+        }
+    ],
+    "metrics": [
+        {
+            "name": "net_revenue",
+            "description": "GAAP-compliant net revenue: completed orders minus discounts, excluding refunds",
+            "type": "derived",
+            "inputMetrics": [
+                "gross_revenue",
+                "total_discounts"
+            ],
+            "expr": "gross_revenue - total_discounts",
+            "owner": "finance-data@company.com"
+        },
+        {
+            "name": "gross_revenue",
+            "description": "Total value of completed orders",
+            "type": "simple",
+            "measure": "order_amount",
+            "filter": "status = 'completed'"
+        }
+    ]
+}
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Human-readable name for this semantic model.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business context explaining what this semantic model represents.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Default time dimension used for measure aggregation. Can be overridden per-measure.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Join keys with type annotations. Entities connect this table to others in the semantic graph.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Entity name (e.g., 'order', 'customer').

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Entity key type: primary (unique, complete), foreign (references another table), unique (unique, possibly partial), natural (real-world identifier).

+
+

Must be one of:

+
  • "primary"
  • "foreign"
  • "unique"
  • "natural"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Column expression. Defaults to entity name if omitted.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Aggregatable column expressions. Measures are the atomic building blocks from which metrics are composed.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Measure name (e.g., 'totalamount', 'ordercount').

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Aggregation function applied to the expression.

+
+

Must be one of:

+
  • "sum"
  • "avg"
  • "count"
  • "count_distinct"
  • "min"
  • "max"
  • "median"
  • "percentile"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

SQL expression to aggregate. Defaults to measure name if omitted.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Aggregation parameters (mirrors dbt-semantic-interfaces agg_params). Currently used by agg=percentile.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: number
+

Percentile in [0, 1] for agg=percentile (0.5 = median). Consumers default to 0.5 when omitted.

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+

Use the discrete percentile (PERCENTILEDISC / usediscrete_percentile) instead of continuous interpolation.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Override the model-level default time dimension for this measure.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Dimension over which this measure cannot be naively aggregated (e.g., account balances).

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Which value to pick when the non-additive dimension is present.

+
+

Must be one of:

+
  • "min"
  • "max"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: false
+

If true, automatically create a simple metric from this measure.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Categorical and time-based grouping axes. Dimensions define how data can be sliced in queries.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Dimension name (e.g., 'orderdate', 'region', 'productcategory').

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Dimension type: categorical (text/enum grouping) or time (temporal axis).

+
+

Must be one of:

+
  • "categorical"
  • "time"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

SQL expression. Defaults to dimension name if omitted.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Type-specific parameters.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Default time granularity for this dimension.

+
+

Must be one of:

+
  • "day"
  • "week"
  • "month"
  • "quarter"
  • "year"
  • "hour"
  • "minute"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Composable KPI definitions built from measures. Metrics are the named business calculations that agents and tools resolve.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Metric name (e.g., 'netrevenue', 'customerchurn_rate').

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business definition of this metric.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Metric type: simple (single measure + filter), derived (expression over other metrics), ratio (numerator/denominator).

+
+

Must be one of:

+
  • "simple"
  • "derived"
  • "ratio"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Reference to a measure name (for simple metrics).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

SQL WHERE clause filter (e.g., "status = 'completed' AND is_refunded = FALSE").

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

References to other metric names (for derived/ratio metrics).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Mathematical expression combining input metrics (for derived metrics).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Numerator metric name (for ratio metrics).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Denominator metric name (for ratio metrics).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business owner of this metric definition.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Semantic model tags for discovery and categorization.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Semantic model labels for automation and metadata.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

AWS Glue crawler configuration for auto-discovery of schema from S3 paths.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Glue crawler name.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

IAM role ARN for the crawler.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Cron schedule expression (e.g., 'cron(0 6 * * ? *)').

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Custom classifier names to apply.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Policy for schema changes detected by the crawler.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "UPDATE_IN_DATABASE"
  • "LOG"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "LOG"
  • "DELETE_FROM_DATABASE"
  • "DEPRECATE_IN_DATABASE"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

AWS Glue Iceberg table management configuration (snapshots, compaction).

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Underlying file format for Iceberg data files.

+
+

Must be one of:

+
  • "parquet"
  • "orc"
  • "avro"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Snapshot retention policy.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Maximum age of snapshots in milliseconds.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Minimum number of snapshots to retain.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Iceberg compaction settings.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Target file size in MB after compaction.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.4: Declares this expose as agent-consumable over MCP, with per-expose overrides for the consumer-side output-port server (fluid mcp output-port serve). Its presence opts the expose into the Fluid MCP gateway, where policy.agentPolicy.allowedModels / allowedUseCases are enforced at every read.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Hard cap for the sample MCP tool. Defends against an over-curious agent asking for petabyte-scale row counts. Default: 100.

+
+ + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Surfaced on the describe tool so consumer agents can declare downstream handling. Advisory; the real gate is policy.agentPolicy.

+
+

Must be one of:

+
  • "public"
  • "internal"
  • "confidential"
  • "restricted"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Upstream data product identifier (e.g., 'silver.hr.people_v2'). Renamed from 'provider' for clarity.

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

The upstream exposeId being consumed.

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.6: The upstream contract's OWN id, for an upstream that has one of its own β€” a product not built with FLUID, registered in a catalog under an id FLUID did not derive. The canonical-port builder already prefers an explicit contractId over the derived <productId>.<exposeId>; until now consumes[] was closed, so there was no way to supply it and the derived id was exported as if the upstream answered to it.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Semver range (e.g., ^1.2, ~1.4, >=1.0.0). Not strictly validated by regex; tooling enforces.

+
+ + + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+
+

NEW in v0.7.6: The exact upstream contract version this product was built and tested against, within versionConstraint. Exported as the ODPS inputPorts[].version, and as the version of the matching outputPorts[].inputContracts[] entry. Without it the export derives the range's lower bound; for a range it cannot read it falls back to 1.0.0, because ODPS v1.0.0 requires a version on every input port and the field cannot simply be omitted. versionConstraint stays the range this product accepts; this is the single version its exported lineage claims. fluid validate does not check that it falls inside versionConstraint.

+ +
+
+ +
+ + Must match regular expression: ^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$ + + + + + +
+
Example:
+
"2.1.0"
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Maximum acceptable staleness for consumed data.

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
+

Minimum completeness expectation.

+
+ + + +

Value must be greater or equal to 0 and lesser or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

QoS expectation tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

QoS expectation labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Business purpose for consuming this data.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Consumption relationship tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Consumption relationship labels.

+
Same definition as labels +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Cross-mesh only. Id of the federated workspace that owns this upstream, as declared in federation/upstreams.yaml. Set it when the upstream lives in another mesh rather than this workspace; omit it for a local upstream, which is resolved by walking the workspace.

+
+ + + +

Must be at least 1 characters long

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Cross-mesh only. The contract digest this product was composed against, as sha256:<64 hex>. fluid apply re-fetches the upstream's live digest and compares: a mismatch means the upstream changed since composition. Required whenever upstreamWorkspace is set.

+
Must match regular expression: ^sha256:[0-9a-f]{64}$ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

NEW in v0.5.5: Multiple build configurations for multi-modal data products (e.g., batch + streaming, SQL + ML).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+

Enhanced build configuration supporting v0.4.0 patterns with v0.5.5 improvements.

+
+ + No Additional Properties

+ +

+
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "hybrid-reference" + + + + + + +
+
+
+
+
+ + + +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Hybrid-reference pattern from v0.4.0 (dbt-style).

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

dbt profile target to run against. Forwarded to dbt as --target <name> by the build runner (allowlisted against the profile's real targets; falls back to the profile default when the requested target is unavailable).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+

dbt node selector(s) forwarded to dbt --select. Accepts a single selector string or an array of selector strings; the build runner drops flag-like values (those starting with '-').

+

+ +

+
+ + + Type: string
+ + + + + + + +
+ + + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+

Legacy alias for select (dbt's older --models flag). Same string-or-array shape; select takes precedence when both are set.

+

+ +

+
+ + + Type: string
+ + + + + + + +
+ + + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ + Type: enum (of string)
+
+

Must be one of:

+
  • "table"
  • "view"
  • "incremental"
  • "ephemeral"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+ + + + + + +
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "embedded-logic" + + + + + + +
+
+
+
+
+ + + +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Embedded logic pattern from v0.4.0.

+
+ + No Additional Properties

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + +
+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • stream
+
+ + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + + +
+
+
+
+
+
+
+ + + + + + +
+ + + +
+ + + Type: object
+ +
+

The following properties are required:

+
  • sql
+
+ + + + + +
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "sql"
  • "flink_sql"
  • "pyspark"
  • "scala"
  • "python"
  • "r"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.7 (preview): The stream the build reads. On a managed Flink build (parameters.flink.management) sql is optional: the lane writes the query from the mapping instead. Elsewhere the stream describes what the build's own sql reads.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+

The streaming system. Only kafka in this preview.

+
Specific value: "kafka" + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

One Kafka topic, by name: Kafka's legal characters (not . or ..), no pattern and no list.

+
Must match regular expression: ^(?!\.\.?(?![\s\S]))[A-Za-z0-9._-]{1,249}(?![\s\S]) + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

How record values are encoded. json: one JSON object per record; a field the schema does not declare is ignored, and a declared field that is missing reads as null. avro-confluent: Avro in the Confluent Schema Registry wire format. The declared schema is the reader schema, so the producers' subject must stay readable by it: FORWARD or FULL compatibility, transitive.

+
+

Must be one of:

+
  • "json"
  • "avro-confluent"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

The record value's fields, as columns. Nested data uses fields (struct), element (array) and key/value (map).

+
+ + + + + +

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+Same definition as exposes_items_contract_schema_items +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.7 (preview): The record key, read as columns next to the value's. A record without a key reads its key columns as null.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

raw: the key's bytes as one column, of type string (UTF-8) or binary. json and avro-confluent: a key record whose fields are the schema's columns.

+
+

Must be one of:

+
  • "raw"
  • "json"
  • "avro-confluent"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

The key's fields, as columns: exactly one for raw.

+
+ + + + + +

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+Same definition as exposes_items_contract_schema_items +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Put in front of every key column's name, so a key field can share its name with a value field (a Debezium key repeats the value's id). A mapping names the prefixed column.

+
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?![\s\S]) + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

Kafka record metadata, each read as a column.

+
+ + + + + +

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+

NEW in v0.7.7 (preview): One piece of Kafka record metadata, read as a column. Its type is fixed by what it reads: topic string, partition int, offset long, timestamp timestamp_ltz(3), timestamp-type string, headers map<string, binary> and leader-epoch int. leader-epoch is null when the broker does not report it; the others are never null.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

The column's name, unique among the stream's value, key and metadata columns.

+
+ + + +

Must be at least 1 characters long

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

The metadata the column reads.

+
+

Must be one of:

+
  • "topic"
  • "partition"
  • "offset"
  • "timestamp"
  • "timestamp-type"
  • "headers"
  • "leader-epoch"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.7 (preview): How the stream's records become the output's rows. Requires stream, and replaces sql.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.7 (preview): One output row per element of an array. A record whose array is null or empty produces no row.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

The array: a stream column or a nested field, such as items or order.lines.

+
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*|\['[^'\\\u0000-\u001f\u007f]{1,256}'\])*(?![\s\S]) + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

The element's name in mapping paths: item.sku for a field of a struct element, item for a scalar element.

+
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?![\s\S]) + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

Output columns, or nested fields of them, that are read from another path, renamed or converted.

+
+ + + + + +

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+

NEW in v0.7.7 (preview): Where one output column, or one nested field of one, is read from.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

The output column, or a field of a struct output column as a dotted path (customer.id). Mapping a nested field builds that struct from the paths given, so a nested output can be written from fields that are flat or elsewhere in the record.

+
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*(?![\s\S]) + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

The source: a stream column (value, key or metadata), a nested field (customer.address.country), a map entry (attributes['channel'], null when the key is absent) or the exploded element (item.sku).

+
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*|\['[^'\\\u0000-\u001f\u007f]{1,256}'\])*(?![\s\S]) + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Convert the source to this type: binary to string (UTF-8), or a widening (int to long, float to double, a decimal to a wider precision at the same scale, a timestamp to a finer precision).

+
Same definition as type +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+ + + + + + +
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "multi-stage" + + + + + + +
+
+
+
+
+ + + +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.5.5: Multi-stage orchestration pattern.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "hybrid-reference"
+
+

Must be one of:

+
  • "hybrid-reference"
  • "embedded-logic"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Base properties for all build patterns.

+
Same definition as properties +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+

Unified identifier pattern: starts/ends with alphanumeric or underscore, allows dots and hyphens in middle. Both casing conventions accepted (snake_case / dotted lowercase / PascalCase / UPPER) so existing catalog ids round-trip unchanged.

+
Same definition as id +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+ + + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+Same definition as retries +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+ + + + + + +
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "acquisition" + + + + + + +
+
+
+
+
+ + + +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.3: Source-aligned acquisition pattern. The build ingests data from an external system (no transformation). Engine ∈ {duckdb, airbyte, meltano, dlt, kafka-connect, debezium}.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

External source system specification.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Source system kind: filesystem, postgres, mysql, sqlite, http, salesforce, stripe, github, kafka, etc. Engine-specific catalog of supported kinds.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Connection details. Use ${VAR} placeholders for env values; use secretRef for credentials. Inline secrets are forbidden by validator.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+

+ +

+
+ + + Type: integer
+ + + + + + + +
+ + + Type: string
+

Allows ${ENV_VAR} placeholders that resolve to an integer at runtime.

+
+ + + + + + +
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Reference to a secret in the configured backend (vault://, aws://, gcp://, azure://, env://).

+
Must match regular expression: ^[a-z][a-z0-9_+.-]*://.+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.3: Username/principal for SQL- or REST-style sources. Required by most database drivers (Postgres/MySQL/Oracle/etc.) β€” without it the underlying client falls back to the OS-user identity and typically fails authentication. Goes alongside secretRef (or an inline password) for the credential pair.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.3: Default schema/namespace within the source database to read from. Maps to engine-specific config: dlt sqldatabase schema=, Meltano tap-postgres filterschemas=[], Airbyte source-postgres schemas=[], Debezium schema.include.list, Kafka Connect JDBC schema.pattern. Use schemas (array) to scope to multiple schemas at once.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

NEW in v0.7.3: Multi-schema variant of schema. When both are set, schemas wins and schema is treated as a deprecated alias. Engines that don't natively support multi-schema reads will iterate the list.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "full_refresh"
  • "incremental_append"
  • "incremental_dedup"
  • "incremental_merge"
  • "cdc"
  • "streaming"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Column used for incremental cursor (e.g., updated_at, SystemModstamp).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "high_water_mark"
  • "log_position"
  • "lsn"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

List of streams/tables/objects to ingest.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "csv"
  • "parquet"
  • "json"
  • "ndjson"
  • "avro"
  • "orc"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Destination format for ingested data. binding.platform on the expose remains the source of truth for where; sink.format describes how.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "iceberg"
  • "delta"
  • "parquet"
  • "csv"
  • "json"
  • "snowflake_table"
  • "bigquery_table"
  • "redshift_table"
  • "duckdb_table"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "rest"
  • "glue"
  • "nessie"
  • "unity"
  • "snowflake-managed"
  • "hive"
  • "lakekeeper"
  • "polaris"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+
+

NEW in v0.7.6: How the Iceberg sink config forge-cli DERIVES (kafka-connect with sink.format iceberg, embedded debezium) authenticates to an Iceberg REST catalog (rest, lakekeeper, polaris, unity, snowflake-managed). Names an environment variable, never a secret: forge-cli emits the secret as the catalog property value env:<NAME>, which Apache Iceberg's REST client resolves from the environment of the process that runs the sink (the Kafka Connect worker, or the Debezium Server forge-cli launches). Iceberg drops a property whose variable is unset, so the embedded Debezium run refuses to start without it; forge-cli cannot see a Connect worker's environment. type oauth2 (client credentials): credentialEnv names a variable holding clientid:clientsecret, oauth2ServerUri is the identity provider's token endpoint, scope is required (lakekeeper on a Keycloak-fronted Lakekeeper; PRINCIPAL_ROLE:ALL on Polaris with internal auth, whose oauth2ServerUri is <polaris>/api/catalog/v1/oauth/tokens). type bearer: tokenEnv names a variable holding a bearer token (Unity Catalog OSS). oauth2ServerUri, scope, audience and resource are written as they are and may not hold a ${...} expression or start with env: (Debezium Server would expand it, Kafka Connect would not). fluid validate refuses it on a hand-written sink config, on a catalog not reached over REST, and next to an override that sets rest.auth.type, credential, token, oauth2-server-uri, scope, audience, resource, rest.sigv4-enabled or a header.Authorization.

+ +
+
+ +
No Additional Properties

+ +

+
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "oauth2" + + + + + + +
+
+
+
+
+ + + +
+ + +
+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • tokenEnv
+
+ + + + + +
+
+
+
+

The following properties are required:

+
  • credentialEnv
  • oauth2ServerUri
  • scope
+
+ + + + + +
+ + + + + + +
+ + + Type: object
+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "bearer" + + + + + + +
+
+
+
+
+ + + +
+ + +
+

+ +

+
+ + +
+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • credentialEnv
+
+ + + + + +
+
+
+ + + + + + +
+ + +
+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • oauth2ServerUri
+
+ + + + + +
+
+
+ + + + + + +
+ + +
+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • scope
+
+ + + + + +
+
+
+ + + + + + +
+ + +
+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • audience
+
+ + + + + +
+
+
+ + + + + + +
+ + +
+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • resource
+
+ + + + + +
+
+
+ + + + + + +
+
+

The following properties are required:

+
  • tokenEnv
+
+ + + + + +
+ + + + + + +
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

oauth2: OAuth2 client credentials against oauth2ServerUri. bearer: a static bearer token.

+
+

Must be one of:

+
  • "oauth2"
  • "bearer"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+

type oauth2: the NAME of the environment variable holding clientid:clientsecret (Iceberg splits it on the first ':'). Emitted as credential=env:<NAME>. A value shaped like a token (a run of 20 or more letters and digits, with a digit, between underscores) is refused.

+
+

Must not be:

+
+
+ + + Type: object
+Must match regular expression: (?:^|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_|$) + + + + + + +
+
+
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?![\s\S]) + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+

type oauth2: absolute URL starting with a lowercase http:// or https:// of the identity provider's token endpoint, with no user:password@ part (the client id and secret go in credentialEnv's variable), e.g. a Keycloak realm's <keycloak>/realms/<realm>/protocol/openid-connect/token. Emitted as oauth2-server-uri; without it Iceberg would send the credential to the catalog's own v1/oauth/tokens.

+
+

Must not be:

+
+
+ + + Type: object
+Must match regular expression: \$\{|[\x00-\x1f\x7f]|^\s*env: + + + + + + +
+
+
Must match regular expression: ^https?://[^\s/?#@]+(?:[/?#]\S*)?$ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+

type oauth2: the OAuth2 scope to request (Iceberg's default, catalog, is not one most identity providers know). Emitted as scope.

+
+

Must not be:

+
+
+ + + Type: object
+Must match regular expression: \$\{|[\x00-\x1f\x7f]|^\s*env: + + + + + + +
+
+
+ + + +

Must be at least 1 characters long

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+

type oauth2, optional: sent as the audience parameter on the token request.

+
+

Must not be:

+
+
+ + + Type: object
+Must match regular expression: \$\{|[\x00-\x1f\x7f]|^\s*env: + + + + + + +
+
+
+ + + +

Must be at least 1 characters long

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+

type oauth2, optional: sent as the resource parameter on the token request.

+
+

Must not be:

+
+
+ + + Type: object
+Must match regular expression: \$\{|[\x00-\x1f\x7f]|^\s*env: + + + + + + +
+
+
+ + + +

Must be at least 1 characters long

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
+

type bearer: the NAME of the environment variable holding the bearer token. Emitted as token=env:<NAME>. A value shaped like a token (a run of 20 or more letters and digits, with a digit, between underscores, such as a dapi... personal access token) is refused.

+
+

Must not be:

+
+
+ + + Type: object
+Must match regular expression: (?:^|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_|$) + + + + + + +
+
+
Must match regular expression: ^[A-Za-z_][A-Za-z0-9_]*(?![\s\S]) + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Delivery semantics + idempotency + DLQ configuration.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "at_least_once"
+
+

Must be one of:

+
  • "at_most_once"
  • "at_least_once"
  • "exactly_once"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string Default: "{run_id}:{stream}:{record_pk}"
+

Template for the idempotency key. Default: {runid}:{stream}:{recordpk}

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "parquet"
  • "json"
  • "ndjson"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer Default: 10000
+ + + + +

Value must be greater or equal to 0

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of enum (of string)
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "pii_classification_failed"
  • "schema_violation"
  • "destination_write_failed"
  • "quality_gate_failed"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Schema evolution behavior. Stricter-wins between policy and per-change overrides.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "strict"
+
+

Must be one of:

+
  • "strict"
  • "discover_and_freeze"
  • "evolve_safe"
  • "evolve_all"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "include"
  • "warn"
  • "fail"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "drop"
  • "warn"
  • "fail"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "cast"
  • "warn"
  • "fail"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "required"
+
+

Must be one of:

+
  • "required"
  • "optional"
  • "disabled"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of enum (of string)
+

Pre-land hook chain - runs on each batch before destination write.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "dlp_scan"
  • "tokenize_pii"
  • "quality_gate"
  • "emit_lineage_input"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Pre-land quality gates + anomaly signals.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "not_null"
  • "unique"
  • "regex"
  • "range"
  • "row_count_anomaly"
  • "freshness"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "ewma"
  • "iqr"
  • "exact"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "info"
  • "warn"
  • "error"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "route_to_dlq"
+
+

Must be one of:

+
  • "route_to_dlq"
  • "abort_run"
  • "best_effort"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "record_count_drop"
  • "record_count_surge"
  • "bytes_per_record_outlier"
  • "latency_outlier"
  • "dlq_rate_spike"
  • "cursor_stalled"
  • "schema_fingerprint_changed"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "ewma"
  • "iqr"
  • "exact"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "info"
  • "warn"
  • "error"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Cost tracking + budget enforcement.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+ + + + +

Value must be greater or equal to 0

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Human-readable size with unit suffix (e.g., '50GB', '500MB').

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer
+ + + + +

Value must be greater or equal to 0

+ + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "warn"
+
+

Must be one of:

+
  • "warn"
  • "abort"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Catalog auto-registration on first apply.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of enum (of string)
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "datahub"
  • "openmetadata"
  • "datamesh_manager"
  • "unity"
  • "glue"
  • "snowflake_horizon"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "auto"
+
+

Must be one of:

+
  • "auto"
  • "manual"
  • "none"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Single-flight concurrency control.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "product"
+
+

Must be one of:

+
  • "product"
  • "build"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "abort"
+
+

Must be one of:

+
  • "abort"
  • "queue"
  • "replace"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Connector image supply-chain verification.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "cosign"
+
+

Must be one of:

+
  • "cosign"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "optional"
+
+

Must be one of:

+
  • "required"
  • "optional"
  • "disabled"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Engine deployment mode and target.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "embedded"
+
+

Must be one of:

+
  • "embedded"
  • "bring-your-own"
  • "managed"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

URL for bring-your-own mode.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "docker"
  • "kubernetes"
  • "terraform"
  • "opentofu"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "small"
+
+

Must be one of:

+
  • "small"
  • "medium"
  • "large"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Engine deployment mode and target.

+
Same definition as deployment +
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer Default: 1
+ + + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Confluent-compatible Schema Registry config. When url is set, the runner injects key.converter/value.converter for Avro into the emitted connector config.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: uri
+

Schema Registry HTTP endpoint, e.g. http://schema-registry:8081

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Connector image supply-chain verification.

+
Same definition as image_signature +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Engine deployment mode and target.

+
Same definition as deployment +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+

Opt-in: derive the Iceberg sink connector config. Defaults OFF when a hand-written sinkconnectorconfig is present (RFC Β§6.2).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Explicit topics the derived Iceberg sink consumes (else derived from source streams).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Operator escape hatch: connector config keys merged LAST over the derived Iceberg sink config.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Optional Iceberg streaming-sink tuning (RFC Β§6.2). All keys optional.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+ + + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "initial"
  • "schema_only"
  • "never"
  • "when_needed"
  • "always"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Engine deployment mode and target.

+
Same definition as deployment +
+
+
+
+
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+ + + + + + +
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Build identifier for multi-build scenarios (e.g., 'batch-processing', 'ml-training').

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Human-readable description of this build configuration.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "hybrid-reference"
+

Build pattern: hybrid-reference (dbt-style), embedded-logic (raw SQL/code), multi-stage (complex pipelines), or acquisition (source-aligned ingestion from external systems).

+
+

Must be one of:

+
  • "hybrid-reference"
  • "embedded-logic"
  • "multi-stage"
  • "acquisition"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Default: "dbt"
+
+

Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-qualified dbt engines (dbt-<adapter>, e.g. dbt-glue, dbt-snowflake, dbt-bigquery) are accepted too and route through the dbt build runner, which derives the adapter from the suffix. Ingestion engines (acquisition pattern only): duckdb (zero-infra files/JDBC), airbyte (350+ SaaS), meltano (Singer 600+ taps), dlt (Python-native), kafka-connect (streaming), debezium (CDC). AWS Glue: glue (Spark ETL jobs managed by Glue). NEW in v0.7.6: flink (Apache Flink jobs, usually long-running streaming SQL with properties.language: flink_sql). forge-cli validates and plans a flink build but ships no runner for it: the platform that runs the job, such as the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, executes it. Put deployment settings under properties.parameters, for example parallelism and upgradeMode, as the Flink Kubernetes Operator's job spec names them, and flinkConfiguration.

+ +
+
+ +

+ +

+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "dbt"
  • "sql"
  • "python"
  • "spark"
  • "glue"
  • "custom"
  • "duckdb"
  • "airbyte"
  • "meltano"
  • "dlt"
  • "kafka-connect"
  • "debezium"
  • "flink"
+
+ + + + + + +
+ + + Type: object
+Must match regular expression: ^dbt-[a-z0-9]+([_-][a-z0-9]+)*$ + + + + + + +
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of enum (of string)
+

Capabilities the build asks for (acquisition pattern). Validation enforces capabilities βŠ† runner.declared_capabilities.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "full_refresh"
  • "incremental_append"
  • "incremental_dedup"
  • "incremental_merge"
  • "cdc"
  • "streaming"
  • "schema_discovery"
  • "schema_evolution"
  • "dlp_scan"
  • "at_most_once"
  • "at_least_once"
  • "exactly_once"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Pattern-specific build properties with conditional validation.

+
+ + + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Build execution configuration (triggers, runtime, retries).

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + +

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "continuous" + + + + + + +
+
+
+
+
+ + + +
+ + +
+

+ +

+
+ + +
+

cron is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove cron, or set type: schedule.

+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • cron
+
+ + + + + +
+
+
+ + + + + + +
+ + +
+

schedule is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove schedule, or set type: schedule.

+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • schedule
+
+ + + + + +
+
+
+ + + + + + +
+ + +
+

datasets is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove datasets, or set type: dataset.

+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • datasets
+
+ + + + + +
+
+
+ + + + + + +
+ + +
+

event is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove event, or set type: event.

+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • event
+
+ + + + + +
+
+
+ + + + + + +
+ + +
+

timetable is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove timetable, or set type: timetable.

+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • timetable
+
+ + + + + +
+
+
+ + + + + + +
+ + + + + + +
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

NEW in v0.7.6: continuous, for a build that runs as a long-lived job on its own platform, such as an engine: flink job on the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, or a kafka-connect or debezium connector. The platform keeps the job running, so no scheduler starts runs: fluid generate schedule, fluid schedule-sync and the fluid apply DAGs leave the build out. A continuous trigger takes no other trigger, so cron, schedule, datasets, event and timetable are refused next to it, as Databricks Jobs refuses a continuous trigger next to any other.

+ +
+
+ +
+

Must be one of:

+
  • "schedule"
  • "event"
  • "manual"
  • "dependency"
  • "dataset"
  • "schedule_and_dataset"
  • "timetable"
  • "continuous"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

NEW in v0.7.0: Can reference data products directly via exposeRef for strong typing and auto-URI generation. Backward compatible with raw URIs.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + +
+

+ +

+
+ + + Type: string
+

Raw dataset URI (legacy, e.g., 'dataset://gcp/project/dataset/table')

+
+ + + + + + +
+ + + Type: object
+

NEW in v0.7.0: Data product reference with strong typing

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Data product ID to depend on

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Expose ID within the data product

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Optional version constraint (e.g., '^2.0.0')

+
Same definition as versionConstraint +
+
+
+
+
+ + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "all"
+
+

Must be one of:

+
  • "any"
  • "all"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Cron expression for schedule trigger

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string Default: "UTC"
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Custom timetable configuration

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "gcp"
  • "aws"
  • "azure"
  • "local"
  • "kubernetes"
  • "composer"
  • "mwaa"
  • "databricks"
  • "snowflake"
  • "athena"
  • "glue"
  • "redshift"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "LocalExecutor"
  • "CeleryExecutor"
  • "KubernetesExecutor"
  • "SequentialExecutor"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Container image for execution

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Service account for cloud execution

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+ + + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "fixed"
  • "exponential"
  • "linear"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "email"
  • "slack"
  • "webhook"
  • "pagerduty"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "success"
  • "failure"
  • "always"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "airflow"
  • "dagster"
  • "prefect"
  • "kubeflow"
  • "custom"
  • "none"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "generated"
+
+

Must be one of:

+
  • "generated"
  • "manual"
  • "hybrid"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+Must match regular expression: ^[a-z0-9_][a-z0-9_.-]*[a-z0-9_]$|^[a-z0-9_]$ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: false
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer
+ + + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Airflow DAG tags (different from FLUID tags)

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Default configuration for all tasks (can be overridden per task)

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+ + + + +

Value must be greater or equal to 0

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "all_success"
  • "all_failed"
  • "all_done"
  • "one_success"
  • "one_failed"
  • "none_failed"
  • "none_skipped"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.0: Enhanced error categorization and handling

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "exponential_backoff"
+
+

Must be one of:

+
  • "fixed"
  • "exponential_backoff"
  • "linear_backoff"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Error types that should trigger retries

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+

+
Example:
+
[
+    "TemporaryFailure",
+    "RateLimitExceeded",
+    "ConnectionTimeout",
+    "DeadlineExceeded"
+]
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Error types that should fail immediately

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+

+
Example:
+
[
+    "ValidationError",
+    "SchemaViolation",
+    "PermissionDenied",
+    "InvalidCredentials"
+]
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "notify"
  • "create_incident"
  • "log"
  • "custom"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "low"
  • "medium"
  • "high"
  • "critical"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "always"
  • "maxAttemptsExceeded"
  • "nonRetryableError"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+ + + + + + +

Must contain a minimum of 1 items

No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + +

+ +

+
+ + + Type: object
+

Backward compatibility: If 'operator' is specified but not 'type', map operator to type

+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + +
+
+

Must not be:

+
+
+ + + Type: object
+ +
+

The following properties are required:

+
  • type
+
+ + + + + +
+
+
+
+

The following properties are required:

+
  • operator
+
+ + + + + +
+ + + +
+ + + Type: object
+

Operator field is deprecated but still works. Maps: FluidExecuteOperator→fluid_execute, BashOperator→bash, etc.

+
+ + + + + + +
+ + + + + + +
+ + + Type: object
+

NEW in v0.7.0: Validate provider_action tasks have required fields

+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "provider_action" + + + + + + +
+
+
+
+
+ + + +
+ + + Type: object
+ +
+

The following properties are required:

+
  • provider
  • action
  • params
+
+ + + + + +
+ + + + + + +
+ + + Type: object
+

Validate fluid_execute tasks

+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + +
+

+ +

+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "fluid_execute" + + + + + + +
+
+
+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+Must match regular expression: FluidExecute + + + + + + +
+
+
+
+
+ + + + + + +
+ + + +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+ + + + + + +
+ + + Type: object
+

Validate bash tasks

+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + +
+

+ +

+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "bash" + + + + + + +
+
+
+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+Must match regular expression: Bash + + + + + + +
+
+
+
+
+ + + + + + +
+ + + +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ +
+

+ +

+
+ + + Type: object
+ +
+

The following properties are required:

+
  • bash_command
+
+ + + + + +
+ + + Type: object
+ +
+

The following properties are required:

+
  • command
+
+ + + + + +
+ + + + + + +
+
+
+
+
+ + + + + + +
+ + + Type: object
+

Validate python tasks

+

+ +

+

If the conditions in the "If" tab are respected, then the conditions in the "Then" tab should be respected. + Otherwise, the conditions in the "Else" tab should be respected.

+ + +
+ +
+ + +
+

+ +

+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: const
+Specific value: "python" + + + + + + +
+
+
+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+Must match regular expression: Python + + + + + + +
+
+
+
+
+ + + + + + +
+ + + +
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ +
+

+ +

+
+ + + Type: object
+ +
+

The following properties are required:

+
  • python_callable
+
+ + + + + +
+ + + Type: object
+ +
+

The following properties are required:

+
  • callable
+
+ + + + + +
+ + + + + + +
+
+
+
+
+ + + + + + +
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+Must match regular expression: ^[a-z0-9_][a-z0-9_.-]*[a-z0-9_]$|^[a-z0-9_]$ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

NEW in v0.7.0: Simplified task type taxonomy. Replaces 'operator' field (backward compatible). Types: provideraction (NEW!), fluidvalidate, fluidplan, fluidapply, fluidexecute, fluidverify, fluiddqcheck, bash, python, sensor, or provider-specific operators.

+
+

Must be one of:

+
  • "provider_action"
  • "fluid_validate"
  • "fluid_plan"
  • "fluid_apply"
  • "fluid_execute"
  • "fluid_verify"
  • "fluid_dq_check"
  • "bash"
  • "python"
  • "branch_python"
  • "sensor"
  • "email"
  • "http"
  • "snowflake_query"
  • "bigquery_query"
  • "bigquery_job"
  • "glue_job"
  • "databricks_job"
  • "custom"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

DEPRECATED in v0.7.0: Use 'type' instead. Maintained for backward compatibility with v0.6.1 contracts. Maps to 'type' automatically (FluidExecuteOperator β†’ fluid_execute, BashOperator β†’ bash).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

NEW in v0.7.0: Provider name when type=provider_action. Examples: aws, gcp, azure, snowflake, databricks.

+
+

Must be one of:

+
  • "aws"
  • "gcp"
  • "azure"
  • "snowflake"
  • "databricks"
  • "kafka"
  • "kubernetes"
  • "local"
  • "custom"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.0: Provider action when type=provideraction. Format: service.operation (e.g., s3.ensurebucket, table.ensure, kinesis.create_stream).

+
Must match regular expression: ^[a-z0-9_]+\.[a-z0-9_]+$ + + + + + +
+
Examples:
+
"s3.ensure_bucket"
+
+
"table.ensure"
+
+
"database.ensure"
+
+
"kinesis.create_stream"
+
+
"glue.create_job"
+
+
"bigquery.create_table"
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

NEW in v0.7.0: Reference to transformation step in builds[].transformations[]. Links orchestration tasks to build steps for clearer mapping.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

NEW in v0.7.0: Declares what exposes[] this task produces. Enables automatic dataset registration and lineage tracking.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

The exposeId from exposes[] that this task produces

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Optional: Airflow dataset URI. Auto-generated from binding.location if not specified.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "replace"
+

How this task updates the data

+
+

Must be one of:

+
  • "replace"
  • "append"
  • "merge"
  • "upsert"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

NEW in v0.7.0: Tasks can declare dependencies on other data products with version constraints and freshness requirements.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Data product ID to depend on

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Specific expose within the product

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Required version (e.g., '^2.0.0', '>=1.5.0')

+
Same definition as versionConstraint +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Maximum age of data to wait for (e.g., PT6H = 6 hours)

+
Same definition as window +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Task-specific overrides for taskDefaults

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

XCom configuration for inter-task communication (from v0.6.1)

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "json"
  • "pickle"
  • "cloudpickle"
  • "custom"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+ + + + + + + No Additional Items +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Task-level access control (from v0.6.1)

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Cost tracking (enhanced in v0.7.0)

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: number
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string Default: "USD"
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: number
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: false
+

NEW in v0.7.0: Track actual costs vs estimate

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: number
+

NEW in v0.7.0: Alert if actual > threshold * estimate (e.g., 1.5 = 150%)

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.0: Budget enforcement

+
+ + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: number
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: false
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "FileSensor"
  • "ExternalTaskSensor"
  • "TimeSensor"
  • "HttpSensor"
  • "S3KeySensor"
  • "GCSObjectSensor"
  • "custom"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer Default: 60
+ + + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: integer Default: 3600
+ + + + +

Value must be greater or equal to 1

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "poke"
+
+

Must be one of:

+
  • "poke"
  • "reschedule"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+ + + + + + + No Additional Items

Each item of this array must be:

+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

List of exposeIds produced by this build.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+

Unified identifier pattern: starts/ends with alphanumeric or underscore, allows dots and hyphens in middle. Both casing conventions accepted (snake_case / dotted lowercase / PascalCase / UPPER) so existing catalog ids round-trip unchanged.

+
Same definition as id +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Dependencies on other builds (for multi-build orchestration).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Unified identifier pattern: starts/ends with alphanumeric or underscore, allows dots and hyphens in middle. Both casing conventions accepted (snake_case / dotted lowercase / PascalCase / UPPER) so existing catalog ids round-trip unchanged.

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "success"
  • "completion"
  • "always"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

Legacy transformations array for backward compatibility.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Path/script identifier (dbt model, sql file, etc.)

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

List of exposeIds materialized by this step. VALIDATION: Tools must validate that every output exists in exposes[].exposeId.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+

Unified identifier pattern: starts/ends with alphanumeric or underscore, allows dots and hyphens in middle. Both casing conventions accepted (snake_case / dotted lowercase / PascalCase / UPPER) so existing catalog ids round-trip unchanged.

+
Same definition as id +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Transformation step tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Transformation step labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Single build configuration (legacy). Use 'builds' array for multi-modal.

+
Same definition as builds_items +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Optional lineage tracking for better data governance.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "table_level"
+

Level of lineage tracking granularity.

+
+

Must be one of:

+
  • "table_level"
  • "field_level"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Upstream dependencies and their field mappings.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Upstream data product ID.

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Upstream expose ID being consumed.

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Field-level lineage mappings (optional).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "direct_mapping"
  • "aggregation"
  • "calculation"
  • "lookup"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Transformation logic or SQL expression.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Transformation tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Transformation labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Field mapping tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Field mapping labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Type of data relationship.

+
+

Must be one of:

+
  • "direct_consumption"
  • "lookup_enrichment"
  • "aggregation"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Upstream relationship tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Upstream relationship labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Known downstream consumers (optional).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Downstream data product ID.

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Impact level if this data product changes.

+
+

Must be one of:

+
  • "critical"
  • "high"
  • "medium"
  • "low"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Downstream relationship tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Downstream relationship labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Lineage configuration tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Lineage configuration labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Optional schema evolution strategy for managing changes.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Versioning strategy for schema changes.

+
+

Must be one of:

+
  • "semantic_versioning"
  • "date_based"
  • "sequential"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Compatibility approach for schema evolution.

+
+

Must be one of:

+
  • "backward_compatible"
  • "forward_compatible"
  • "full_compatible"
  • "breaking"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: integer
+

Advance notice period for breaking changes.

+
+ + + +

Value must be greater or equal to 0

+ + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean
+

Whether schema changes require approval.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

List of approvers for schema changes.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Change policy tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Change policy labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Schema evolution tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Schema evolution labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Optional ML model specifications for MLPipeline kind.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: false
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "mlflow"
  • "kubeflow"
  • "sagemaker"
  • "vertex_ai"
  • "custom"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+ + + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Semantic version (semver.org).

+
Same definition as version +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "classification"
  • "regression"
  • "clustering"
  • "recommendation"
  • "other"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

List of feature names used by this model.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Target variable for supervised learning.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Validation metrics (e.g., accuracy, auc_roc).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Minimum acceptable metric values.

+
+ + + + + + +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ + Type: number
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Model validation tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Model validation labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

ML model tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

ML model labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

ML configuration tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

ML configuration labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Optional environment-specific overrides (dev, staging, prod).

+
+ + + + + + +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Environment-specific metadata overrides.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Environment-specific metadata tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Environment-specific metadata labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

Environment-specific expose overrides.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Unified identifier pattern: starts/ends with alphanumeric or underscore, allows dots and hyphens in middle. Both casing conventions accepted (snake_case / dotted lowercase / PascalCase / UPPER) so existing catalog ids round-trip unchanged.

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+Same definition as binding +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+Same definition as qos +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Environment-specific expose tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Environment-specific expose labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Environment configuration tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Environment configuration labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Unified lifecycle state vocabulary.

+
Same definition as state +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Reference to replacement data product or expose.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Deprecation policy tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Deprecation policy labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Lifecycle tags for automation.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Lifecycle labels for governance.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+Same definition as docs +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.1: Jurisdiction and data residency requirements for compliance enforcement.

+
+ + No Additional Properties + + + + + +
+
Examples:
+
{
+    "jurisdiction": "EU",
+    "allowedRegions": [
+        "eu-west-1",
+        "eu-central-1"
+    ],
+    "dataResidency": true,
+    "crossBorderTransfer": false,
+    "regulatoryFramework": [
+        "GDPR"
+    ],
+    "enforcementMode": "strict"
+}
+
+
{
+    "jurisdiction": "US",
+    "allowedRegions": [
+        "us-east-1",
+        "us-west-2"
+    ],
+    "dataResidency": true,
+    "regulatoryFramework": [
+        "CCPA",
+        "HIPAA"
+    ],
+    "enforcementMode": "strict"
+}
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

Required legal jurisdiction for data storage and processing. Used to validate binding.location matches sovereignty intent.

+
+

Must be one of:

+
  • "EU"
  • "US"
  • "UK"
  • "CA"
  • "AU"
  • "JP"
  • "CN"
  • "IN"
  • "BR"
  • "Global"
  • "Multi-Region"
+
+ + + + + +
+
Examples:
+
"EU"
+
+
"US"
+
+
"Global"
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Explicit list of allowed cloud regions. Example: ['eu-west-1', 'eu-central-1'] for EU-only.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+

+
Examples:
+
[
+    "eu-west-1",
+    "eu-central-1"
+]
+
+
[
+    "us-east-1",
+    "us-west-2"
+]
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Explicit list of prohibited cloud regions. Takes precedence over allowedRegions.

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+

Whether data must remain within jurisdiction boundaries at rest and in transit. True = strict residency.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: false
+

Whether cross-border data transfer is permitted. False = data never leaves jurisdiction.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of enum (of string)
+

Allowed legal mechanisms for cross-border transfer (if permitted). Example: ['SCCs', 'BCRs', 'Adequacy'].

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "SCCs"
  • "BCRs"
  • "Adequacy"
  • "DPF"
  • "Consent"
  • "Derogation"
+
+ + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of enum (of string)
+

Applicable regulatory frameworks. Example: ['GDPR', 'CCPA', 'HIPAA'].

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "GDPR"
  • "CCPA"
  • "CPRA"
  • "HIPAA"
  • "PIPEDA"
  • "LGPD"
  • "PDPA"
  • "POPIA"
  • "DPA"
  • "APPI"
+
+ + + + + + +
+

+
Examples:
+
[
+    "GDPR"
+]
+
+
[
+    "GDPR",
+    "CCPA"
+]
+
+
[
+    "HIPAA"
+]
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "strict"
+

How sovereignty violations are handled. strict = block deployment, advisory = warn, audit = log only.

+
+

Must be one of:

+
  • "strict"
  • "advisory"
  • "audit"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+

Whether binding.location must be validated against sovereignty rules before deployment.

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Sovereignty policy tags.

+
Same definition as tags +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Sovereignty policy labels.

+
Same definition as labels +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Root-level access policy for automated IAM binding generation. Defines principals, permissions, and resources for data product access control.

+
+ + No Additional Properties + + + + + +
+
Example:
+
{
+    "grants": [
+        {
+            "principal": "group:data-analytics@company.com",
+            "permissions": [
+                "read",
+                "select",
+                "query"
+            ]
+        },
+        {
+            "principal": "serviceAccount:pipeline@project.iam.gserviceaccount.com",
+            "permissions": [
+                "write",
+                "insert",
+                "update",
+                "delete"
+            ]
+        }
+    ]
+}
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of object
+

List of access grants to principals (users, groups, service accounts).

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Identity receiving access. Format: 'user:email@domain.com', 'group:name@domain.com', 'serviceAccount:name@project.iam.gserviceaccount.com'

+
+ + + + + +
+
Examples:
+
"group:data-analytics@company.com"
+
+
"serviceAccount:pipeline@project.iam.gserviceaccount.com"
+
+
"user:analyst@company.com"
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of enum (of string)
+

Permissions to grant. Will be mapped to provider-specific roles (e.g., BigQuery roles).

+
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: enum (of string)
+
+

Must be one of:

+
  • "read"
  • "select"
  • "query"
  • "write"
  • "insert"
  • "update"
  • "delete"
  • "create"
  • "admin"
  • "manage"
+
+ + + + + + +
+

+
Examples:
+
[
+    "read",
+    "select"
+]
+
+
[
+    "write",
+    "insert",
+    "update"
+]
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+

Resources this grant applies to. JSONPath expressions referencing exposes[] items. If omitted, applies to all exposed resources.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+

+
Examples:
+
"$.exposes[?(@.exposeId=='customer_table')]"
+
+
"$.exposes[?(@.kind=='table')]"
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Optional conditions for conditional access (e.g., IP restrictions, time windows).

+
+ + + + + + +
+
+
+

+ +

+
+ +
+

Additional Properties of any type are allowed.

+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Top-level orchestration configuration for scheduling and workflow engines.

+
+ + + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+
+

Must be one of:

+
  • "airflow"
  • "dagster"
  • "prefect"
  • "kubeflow"
  • "custom"
  • "none"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string) Default: "generated"
+
+

Must be one of:

+
  • "generated"
  • "manual"
  • "hybrid"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: boolean Default: true
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+Same definition as airflow +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.3: Retention durations for run state, run logs, lineage events, and DLQ records. Honored by the retention sweeper invoked from policy-apply.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

ISO-8601 duration (e.g., P1D, PT15M, P2Y6M).

+
Same definition as window +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+
+

NEW in v0.7.6: Declared downstream consumers of this data product β€” the business artifacts built on it (dashboards, notebooks, analyses, ML apps). Shape follows dbt exposures so existing dbt exposure definitions map field-for-field. Purely declarative and additive: consumers never affect plan/apply. RESERVED, not yet consumed β€” no code reads this key today; it is carried verbatim into plan.json and the contract COMMENT so it survives round-trip, and is intended to feed lineage terminal nodes and impact statements ('Affects: Revenue dashboard') in a later release.

+ +
+
+ +
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Stable identifier for this consumer (unique within the contract).

+
Same definition as id +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Human-facing display name, e.g. "Weekly Revenue Dashboard".

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

What kind of artifact consumes this product.

+
+

Must be one of:

+
  • "dashboard"
  • "notebook"
  • "analysis"
  • "ml"
  • "application"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Owning team for this consumer (routable identity, as in metadata.owner).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: stringFormat: email
+

Contact address for the consumer's owner.

+
+ + + + + + +
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+

Link to the live artifact (dashboard URL, notebook, app).

+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: enum (of string)
+

How production-hardened the consumer is (dbt exposures semantics).

+
+

Must be one of:

+
  • "high"
  • "medium"
  • "low"
+
+ + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: array
+

Which output ports this consumer reads. Absent = all exposes.

+
+ + + + + + No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+

Unified identifier pattern: starts/ends with alphanumeric or underscore, allows dots and hyphens in middle. Both casing conventions accepted (snake_case / dotted lowercase / PascalCase / UPPER) so existing catalog ids round-trip unchanged.

+
Same definition as id +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

Contract-level governance: account/project-wide settings (e.g. AWS Lake Formation admins and LF-tag definitions). Per-resource grants and tag associations live under binding.governance.

+
+ + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: object
+

AWS Lake Formation account-wide settings: data-lake admins and LF-tag (TBAC) definitions. Applies to the AWS account the contract's IaC apply targets.

+
No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ + Type: array of string
+
+

IAM principal ARNs (roles or users) granted Lake Formation admin privileges. Emitted as awslakeformationdatalakesettings.admins. AUTHORITATIVE, not additive: Lake Formation's PutDataLakeSettings replaces the account's admin list with the one sent, so an admin NOT listed here is REMOVED on apply. List every principal that must stay a data lake admin, including the role or user that runs the apply: left out, it loses Lake Formation admin on that apply, and the Lake Formation grants and registrations that need an admin can then fail. The terraform-provider-aws resource also clears the data lake settings its configuration omits, and this block sets only the admins, so the account's create-database and create-table default permissions (the IAMAllowedPrincipals default for new databases and tables), trusted resource owners and parameters are cleared too. Destroying the resource empties the admin list and resets CROSSACCOUNTVERSION to 1. Re-applying the same list is a no-op.

+ +
+
+ +
+ + + + + +

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+Must match regular expression: ^arn:aws[a-z0-9-]*:iam:: + + + + + + +
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+
+

LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one awslakeformationlftag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and ab. Field idiom borrowed from OpenMetadata's classification-label model β€” see ODCS roles + OpenMetadata data contracts spec.

+ +
+
+ +
+ + + + + + +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ + Type: array of string
+ + + + + + +

Must contain a minimum of 1 items

All items must be unique

No Additional Items

Each item of this array must be:

+
+
+ + + Type: string
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ + Type: object
+

NEW in v0.7.6: Contract-wide packaging default β€” declarative container ownership (isolated = this product creates and owns its infrastructure containers; shared = containers are pre-existing, platform-owned pools the product writes into but does not own). Overridable per exposure via binding.packaging. An ABSENT block means legacy behavior: the IaC emit is byte-identical to pre-packaging releases.

+
Same definition as packaging +
+
+
+
+ + + \ No newline at end of file diff --git a/specs/0.7.7/schema_doc.css b/specs/0.7.7/schema_doc.css new file mode 100644 index 0000000..e1f3a51 --- /dev/null +++ b/specs/0.7.7/schema_doc.css @@ -0,0 +1,181 @@ +body { + font: 16px/1.5em "Overpass", "Open Sans", Helvetica, sans-serif; + color: #333; + font-weight: 300; + padding: 40px; +} + +.btn.btn-link { + font-size: 18px; + user-select: text; +} + +.jsfh-animated-property { + animation: eclair; + animation-iteration-count: 1; + animation-fill-mode: forwards; + animation-duration: .75s; + +} + +@keyframes eclair { + 0%,100% { + transform: scale(1); + } + 50% { + transform: scale(1.03); + } +} + +.btn.btn-primary { + margin: 10px; +} + +.btn.example-show.collapsed:before { + content: "show" +} + +.btn.example-show:before { + content: "hide" +} + +.description.collapse:not(.show) { + max-height: 100px !important; + overflow: hidden; + + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; +} + +.description.collapsing { + min-height: 100px !important; +} + +.collapse-description-link.collapsed:after { + content: '+ Read More'; +} + +.collapse-description-link:not(.collapsed):after { + content: '- Read Less'; +} + +.badge { + font-size: 100%; + margin-bottom: 0.5rem; + margin-top: 0.5rem; +} + +.badge.value-type { + font-size: 120%; + margin-right: 5px; + margin-bottom: 10px; +} + + +.badge.default-value { + font-size: 120%; + margin-left: 5px; + margin-bottom: 10px; +} + +.badge.restriction { + display: inline-block; +} + +.badge.required-property,.badge.deprecated-property,.badge.pattern-property,.badge.no-additional { + font-size: 100%; + margin-left: 10px; +} + +.accordion div.card:only-child { + border-bottom: 1px solid rgba(0, 0, 0, 0.125); +} + +.examples { + padding: 1rem !important; +} + +.examples pre { + margin-bottom: 0; +} + +.highlight.jumbotron { + padding: 1rem !important; +} + +.generated-by-footer { + margin-top: 1em; + text-align: right; +} + +/* From https://github.com/richleland/pygments-css/blob/master/friendly.css, see https://github.com/trentm/python-markdown2/wiki/fenced-code-blocks */ +.highlight { background: #e9ecef; } /* Changed from #f0f0f0 in the original style to be the same as bootstrap's jumbotron */ +.highlight .hll { background-color: #ffffcc } +.highlight .c { color: #60a0b0; font-style: italic } /* Comment */ +.highlight .err { border: 1px solid #FF0000 } /* Error */ +.highlight .k { color: #007020; font-weight: bold } /* Keyword */ +.highlight .o { color: #666666 } /* Operator */ +.highlight .ch { color: #60a0b0; font-style: italic } /* Comment.Hashbang */ +.highlight .cm { color: #60a0b0; font-style: italic } /* Comment.Multiline */ +.highlight .cp { color: #007020 } /* Comment.Preproc */ +.highlight .cpf { color: #60a0b0; font-style: italic } /* Comment.PreprocFile */ +.highlight .c1 { color: #60a0b0; font-style: italic } /* Comment.Single */ +.highlight .cs { color: #60a0b0; background-color: #fff0f0 } /* Comment.Special */ +.highlight .gd { color: #A00000 } /* Generic.Deleted */ +.highlight .ge { font-style: italic } /* Generic.Emph */ +.highlight .gr { color: #FF0000 } /* Generic.Error */ +.highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ +.highlight .gi { color: #00A000 } /* Generic.Inserted */ +.highlight .go { color: #888888 } /* Generic.Output */ +.highlight .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ +.highlight .gs { font-weight: bold } /* Generic.Strong */ +.highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ +.highlight .gt { color: #0044DD } /* Generic.Traceback */ +.highlight .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ +.highlight .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ +.highlight .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ +.highlight .kp { color: #007020 } /* Keyword.Pseudo */ +.highlight .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ +.highlight .kt { color: #902000 } /* Keyword.Type */ +.highlight .m { color: #40a070 } /* Literal.Number */ +.highlight .s { color: #4070a0 } /* Literal.String */ +.highlight .na { color: #4070a0 } /* Name.Attribute */ +.highlight .nb { color: #007020 } /* Name.Builtin */ +.highlight .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ +.highlight .no { color: #60add5 } /* Name.Constant */ +.highlight .nd { color: #555555; font-weight: bold } /* Name.Decorator */ +.highlight .ni { color: #d55537; font-weight: bold } /* Name.Entity */ +.highlight .ne { color: #007020 } /* Name.Exception */ +.highlight .nf { color: #06287e } /* Name.Function */ +.highlight .nl { color: #002070; font-weight: bold } /* Name.Label */ +.highlight .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ +.highlight .nt { color: #062873; font-weight: bold } /* Name.Tag */ +.highlight .nv { color: #bb60d5 } /* Name.Variable */ +.highlight .ow { color: #007020; font-weight: bold } /* Operator.Word */ +.highlight .w { color: #bbbbbb } /* Text.Whitespace */ +.highlight .mb { color: #40a070 } /* Literal.Number.Bin */ +.highlight .mf { color: #40a070 } /* Literal.Number.Float */ +.highlight .mh { color: #40a070 } /* Literal.Number.Hex */ +.highlight .mi { color: #40a070 } /* Literal.Number.Integer */ +.highlight .mo { color: #40a070 } /* Literal.Number.Oct */ +.highlight .sa { color: #4070a0 } /* Literal.String.Affix */ +.highlight .sb { color: #4070a0 } /* Literal.String.Backtick */ +.highlight .sc { color: #4070a0 } /* Literal.String.Char */ +.highlight .dl { color: #4070a0 } /* Literal.String.Delimiter */ +.highlight .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ +.highlight .s2 { color: #4070a0 } /* Literal.String.Double */ +.highlight .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ +.highlight .sh { color: #4070a0 } /* Literal.String.Heredoc */ +.highlight .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ +.highlight .sx { color: #c65d09 } /* Literal.String.Other */ +.highlight .sr { color: #235388 } /* Literal.String.Regex */ +.highlight .s1 { color: #4070a0 } /* Literal.String.Single */ +.highlight .ss { color: #517918 } /* Literal.String.Symbol */ +.highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */ +.highlight .fm { color: #06287e } /* Name.Function.Magic */ +.highlight .vc { color: #bb60d5 } /* Name.Variable.Class */ +.highlight .vg { color: #bb60d5 } /* Name.Variable.Global */ +.highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */ +.highlight .vm { color: #bb60d5 } /* Name.Variable.Magic */ +.highlight .il { color: #40a070 } /* Literal.Number.Integer.Long */ diff --git a/specs/0.7.7/schema_doc.min.js b/specs/0.7.7/schema_doc.min.js new file mode 100644 index 0000000..17eceaf --- /dev/null +++ b/specs/0.7.7/schema_doc.min.js @@ -0,0 +1 @@ +$(document).on("click",'a[href^="#"]',function(event){event.preventDefault();history.pushState({},"",this.href)});function flashElement(elementId){myElement=document.getElementById(elementId);myElement.classList.add("jsfh-animated-property");setTimeout(function(){myElement.classList.remove("jsfh-animated-property")},1e3)}function setAnchor(anchorLinkDestination){history.pushState({},"",anchorLinkDestination)}function anchorOnLoad(){let linkTarget=decodeURIComponent(window.location.hash.split("?")[0].split("&")[0]);if(linkTarget[0]==="#"){linkTarget=linkTarget.substr(1)}if(linkTarget.length>0){anchorLink(linkTarget)}}function anchorLink(linkTarget){const target=$("#"+linkTarget);target.parents().addBack().filter(".collapse:not(.show), .tab-pane, [role='tab']").each(function(index){if($(this).hasClass("collapse")){$(this).collapse("show")}else if($(this).hasClass("tab-pane")){const tabToShow=$("a[href='#"+$(this).attr("id")+"']");if(tabToShow){tabToShow.tab("show")}}else if($(this).attr("role")==="tab"){$(this).tab("show")}});setTimeout(function(){let targetElement=document.getElementById(linkTarget);if(targetElement){targetElement.scrollIntoView({block:"center",behavior:"smooth"});setTimeout(function(){flashElement(linkTarget)},500)}},1e3)} \ No newline at end of file diff --git a/tests/0.7.6/binding-security.json b/tests/0.7.6/binding-security.json index 5290b24..5ccd79d 100644 --- a/tests/0.7.6/binding-security.json +++ b/tests/0.7.6/binding-security.json @@ -2,7 +2,7 @@ "corpusVersion": "0.7.6", "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json", "group": "binding-security", - "description": "Binding encryption and principal mapping (0.7.6 preview): encryption's closed property set; kms as 'product', 'none', an AWS alias or key/alias ARN, or a Cloud KMS crypto-key name; the platform-conditional refusals (a gcp binding refuses AWS key forms, an aws binding refuses Cloud KMS names); principals' non-empty keys, one-identity-or-unique-list values, and the platform-conditional identity forms (IAM members on gcp, IAM ARNs on aws). Also documents that platforms other than gcp and aws carry no principal-form constraint.", + "description": "Binding encryption and principal mapping (0.7.6): encryption's closed property set; kms as 'product', 'none', an AWS alias or key/alias ARN, or a Cloud KMS crypto-key name; the platform-conditional refusals (a gcp binding refuses AWS key forms, an aws binding refuses Cloud KMS names); principals' non-empty keys, one-identity-or-unique-list values, and the platform-conditional identity forms (IAM members on gcp, IAM ARNs on aws). Also documents that platforms other than gcp and aws carry no principal-form constraint.", "cases": [ { "description": "a gcp binding uses a Cloud KMS key and maps logical principals to IAM members", diff --git a/tests/0.7.6/binding.json b/tests/0.7.6/binding.json index 6180ae8..e3ce91e 100644 --- a/tests/0.7.6/binding.json +++ b/tests/0.7.6/binding.json @@ -500,7 +500,7 @@ "format": "kafka_topic", "location": { "topic": "customer-events", - "host": "kafka.internal:9092" + "bootstrapServers": "kafka.internal:9092" } }, "contract": { @@ -519,7 +519,7 @@ { "pointer": "/exposes/0/binding/location", "keyword": "additionalProperties", - "detail": "host" + "detail": "bootstrapServers" } ] }, diff --git a/tests/0.7.6/bitol-fields.json b/tests/0.7.6/bitol-fields.json new file mode 100644 index 0000000..111f2f6 --- /dev/null +++ b/tests/0.7.6/bitol-fields.json @@ -0,0 +1,2416 @@ +{ + "corpusVersion": "0.7.6", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json", + "group": "bitol-fields", + "description": "Optional Bitol-shaped fields added in 0.7.6: metadata.team (a roster, each member needing only a username), the root description as a string or as {purpose, usage, limitations}, consumes[].contractId and contractVersion, binding.location.host and binding.serialization for Kafka, and the ODCS-shaped quality rules on a column.", + "cases": [ + { + "description": "metadata.team names a team and a roster whose member carries every field", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "name": "Sales Analytics", + "members": [ + { + "username": "mjones", + "name": "Mia Jones", + "role": "data owner", + "email": "mia.jones@example.com" + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "metadata.team with a member that carries only the required username", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + { + "username": "mjones" + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "metadata.team with a name and no members", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "name": "Sales Analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "metadata.team carries a member outside its closed set", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "name": "Sales Analytics", + "lead": "mjones" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team", + "keyword": "additionalProperties", + "detail": "lead" + } + ] + }, + { + "description": "metadata.team is a string, not an object", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": "Sales Analytics" + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team", + "keyword": "type" + } + ] + }, + { + "description": "metadata.team.name is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "name": 7 + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/name", + "keyword": "type" + } + ] + }, + { + "description": "metadata.team.members is an object, not an array", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": { + "username": "mjones" + } + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members", + "keyword": "type" + } + ] + }, + { + "description": "a team member omits the required username", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + { + "name": "Mia Jones" + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members/0", + "keyword": "required", + "detail": "username" + } + ] + }, + { + "description": "a team member carries a property outside its closed set", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + { + "username": "mjones", + "phone": "555" + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members/0", + "keyword": "additionalProperties", + "detail": "phone" + } + ] + }, + { + "description": "a team member is a string, not an object", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + "mjones" + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members/0", + "keyword": "type" + } + ] + }, + { + "description": "a team member's username is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + { + "username": 7 + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members/0/username", + "keyword": "type" + } + ] + }, + { + "description": "a team member's name is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + { + "username": "mjones", + "name": 7 + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members/0/name", + "keyword": "type" + } + ] + }, + { + "description": "a team member's role is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + { + "username": "mjones", + "role": 7 + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members/0/role", + "keyword": "type" + } + ] + }, + { + "description": "a team member's email is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + { + "username": "mjones", + "email": 7 + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members/0/email", + "keyword": "type" + } + ] + }, + { + "description": "root description as a plain string", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "description": "Customer profiles for analytics." + }, + "valid": true + }, + { + "description": "root description as an object with purpose, usage and limitations", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "description": { + "purpose": "Customer profiles.", + "usage": "Join on customer_id.", + "limitations": "Not for marketing." + } + }, + "valid": true + }, + { + "description": "root description is a number: neither a string nor the object", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "description": 7 + }, + "valid": false, + "errors": [ + { + "pointer": "/description", + "keyword": "oneOf" + } + ] + }, + { + "description": "root description object carries a member outside its closed set", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "description": { + "purpose": "Customer profiles.", + "audience": "analysts" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/description", + "keyword": "oneOf" + } + ] + }, + { + "description": "root description object: purpose is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "description": { + "purpose": 7 + } + }, + "valid": false, + "errors": [ + { + "pointer": "/description", + "keyword": "oneOf" + } + ] + }, + { + "description": "root description object: usage is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "description": { + "usage": 7 + } + }, + "valid": false, + "errors": [ + { + "pointer": "/description", + "keyword": "oneOf" + } + ] + }, + { + "description": "root description object: limitations is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "description": { + "limitations": 7 + } + }, + "valid": false, + "errors": [ + { + "pointer": "/description", + "keyword": "oneOf" + } + ] + }, + { + "description": "a consumes entry names the upstream's own contract id and the exact version built against", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumes": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "contractId": "urn:acme:contracts:raw-customers", + "versionConstraint": "^2.0.0", + "contractVersion": "2.1.0" + } + ] + }, + "valid": true + }, + { + "description": "consumes[].contractId is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumes": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "contractId": 7 + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/consumes/0/contractId", + "keyword": "type" + } + ] + }, + { + "description": "consumes[].contractVersion is not a semantic version", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumes": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "contractVersion": "2.1" + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/consumes/0/contractVersion", + "keyword": "pattern" + } + ] + }, + { + "description": "a Kafka binding location names the bootstrap servers as host", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events", + "host": "kafka.internal:9092" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "binding.location.host is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events", + "host": 9092 + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/location/host", + "keyword": "type" + } + ] + }, + { + "description": "binding.serialization declares json values with json keys", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "json", + "key": "json" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "binding.serialization declares avro values with avro keys", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "avro", + "key": "avro" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "binding.serialization declares protobuf values with protobuf keys", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "protobuf", + "key": "protobuf" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "binding.serialization declares xml values with xml keys", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "xml", + "key": "xml" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "binding.serialization declares avro-confluent values with avro-confluent keys", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "avro-confluent", + "key": "avro-confluent" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "binding.serialization declares json-confluent values with json-confluent keys", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "json-confluent", + "key": "json-confluent" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "binding.serialization.value is outside its enum", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "csv" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/serialization/value", + "keyword": "enum" + } + ] + }, + { + "description": "binding.serialization.key is outside its enum", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "key": "csv" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/serialization/key", + "keyword": "enum" + } + ] + }, + { + "description": "binding.serialization carries a member outside its closed set", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "json", + "headers": "json" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/serialization", + "keyword": "additionalProperties", + "detail": "headers" + } + ] + }, + { + "description": "binding.serialization is a string, not an object", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": "json" + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/serialization", + "keyword": "type" + } + ] + }, + { + "description": "column quality rules in ODCS shape: library, sql, text and custom", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "id": "customer_id-nullValues", + "type": "library", + "metric": "nullValues", + "dimension": "completeness", + "severity": "error", + "unit": "rows", + "description": "No nulls." + }, + { + "id": "customer_id-invalid", + "type": "library", + "metric": "invalidValues", + "arguments": { + "pattern": "^C[0-9]+$" + } + }, + { + "id": "customer_id-sql", + "type": "sql", + "query": "SELECT COUNT(*) FROM profiles WHERE customer_id IS NULL" + }, + { + "id": "customer_id-text", + "type": "text", + "description": "Assigned by the CRM." + }, + { + "id": "customer_id-soda", + "type": "custom", + "engine": "soda", + "implementation": "checks for profiles: ..." + } + ] + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "column quality is an object, not an array", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": { + "id": "a" + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule is an empty object", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + {} + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0", + "keyword": "minProperties" + } + ] + }, + { + "description": "a quality rule is a string, not an object", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + "nullValues" + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule id with a character outside the id pattern", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "id": "no nulls!" + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/id", + "keyword": "pattern" + } + ] + }, + { + "description": "a quality rule id is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "id": 7 + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/id", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule type is outside its enum", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "type": "regex" + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/type", + "keyword": "enum" + } + ] + }, + { + "description": "a quality rule metric is outside the five ODCS library metrics", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "type": "library", + "metric": "outliers" + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/metric", + "keyword": "enum" + } + ] + }, + { + "description": "a quality rule dimension is outside its enum", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "dimension": "freshness" + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/dimension", + "keyword": "enum" + } + ] + }, + { + "description": "quality rule arguments is an array, not an object", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "type": "library", + "arguments": [ + "a" + ] + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/arguments", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule query is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "query": 7 + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/query", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule engine is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "engine": 7 + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/engine", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule severity is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "severity": 7 + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/severity", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule unit is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "unit": 7 + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/unit", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule description is a number", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "description": 7 + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/description", + "keyword": "type" + } + ] + } + ] +} diff --git a/tests/0.7.6/consumers.json b/tests/0.7.6/consumers.json index 97b0f74..3bb275f 100644 --- a/tests/0.7.6/consumers.json +++ b/tests/0.7.6/consumers.json @@ -2,7 +2,7 @@ "corpusVersion": "0.7.6", "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json", "group": "consumers", - "description": "Declared downstream consumers (0.7.6 preview): the consumers array, each consumerRef's required name and type, the type and maturity enums, the closed owner object, the identifier rules on name and exposeIds, and consumerRef's closed property set.", + "description": "Declared downstream consumers (0.7.6): the consumers array, each consumerRef's required name and type, the type and maturity enums, the closed owner object, the identifier rules on name and exposeIds, and consumerRef's closed property set.", "cases": [ { "description": "a consumer carries every member", diff --git a/tests/0.7.6/consumes-pinning.json b/tests/0.7.6/consumes-pinning.json index ad975b8..a17c7ad 100644 --- a/tests/0.7.6/consumes-pinning.json +++ b/tests/0.7.6/consumes-pinning.json @@ -2,7 +2,7 @@ "corpusVersion": "0.7.6", "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json", "group": "consumes-pinning", - "description": "Cross-mesh upstream pinning on consumes[] (0.7.6 preview): upstreamWorkspace requires upstreamDigest (dependentRequired, the first JSON Schema 2020-12-only assertion keyword a FLUID schema uses -- a Draft 7 validator ignores it and accepts the invalid case below), the sha256:<64 lowercase hex> digest pattern, and the non-empty workspace id.", + "description": "Cross-mesh upstream pinning on consumes[] (0.7.6): upstreamWorkspace requires upstreamDigest (dependentRequired, the first JSON Schema 2020-12-only assertion keyword a FLUID schema uses -- a Draft 7 validator ignores it and accepts the invalid case below), the sha256:<64 lowercase hex> digest pattern, and the non-empty workspace id.", "cases": [ { "description": "a cross-mesh upstream names its workspace and the digest it was composed against", diff --git a/tests/0.7.6/lifecycle-expire-bucketpolicy.json b/tests/0.7.6/lifecycle-expire-bucketpolicy.json index 2cc43d8..284ff0d 100644 --- a/tests/0.7.6/lifecycle-expire-bucketpolicy.json +++ b/tests/0.7.6/lifecycle-expire-bucketpolicy.json @@ -2,7 +2,7 @@ "corpusVersion": "0.7.6", "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json", "group": "lifecycle-expire-bucketpolicy", - "description": "Expose-level additions in 0.7.6 preview: lifecycle.expire (a boolean on the expose lifecycle, which keeps the closed property set of 0.7.5's lifecycle plus that one member), and lakeFormation.bucketPolicy's enum.", + "description": "Expose-level additions in 0.7.6: lifecycle.expire (a boolean on the expose lifecycle, which keeps the closed property set of 0.7.5's lifecycle plus that one member), and lakeFormation.bucketPolicy's enum.", "cases": [ { "description": "an expose lifecycle declares a retention period and opts in to expiry", diff --git a/tests/0.7.6/masking-aggparams.json b/tests/0.7.6/masking-aggparams.json index afd9b26..2c3f7d5 100644 --- a/tests/0.7.6/masking-aggparams.json +++ b/tests/0.7.6/masking-aggparams.json @@ -2,7 +2,7 @@ "corpusVersion": "0.7.6", "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json", "group": "masking-aggparams", - "description": "Typed parameters added in 0.7.6 preview: masking params' saltEnv/keyEnv environment-variable-name pattern and keepFirst/keepLast integer bounds (0..64), which leave params open to other members as the schema states; and a semantic measure's closed aggParams object with its 0..1 percentile bounds.", + "description": "Typed parameters added in 0.7.6: masking params' saltEnv/keyEnv environment-variable-name pattern and keepFirst/keepLast integer bounds (0..64), which leave params open to other members as the schema states; and a semantic measure's closed aggParams object with its 0..1 percentile bounds.", "cases": [ { "description": "masking params name the salt and key environment variables and keep a prefix and suffix", diff --git a/tests/0.7.6/packaging.json b/tests/0.7.6/packaging.json index 58b3042..4096b0b 100644 --- a/tests/0.7.6/packaging.json +++ b/tests/0.7.6/packaging.json @@ -2,7 +2,7 @@ "corpusVersion": "0.7.6", "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json", "group": "packaging", - "description": "Declarative packaging modes (0.7.6 preview): the top-level packaging block and its per-binding override, the isolated/shared mode enum, the non-empty pool and poolManifest strings, the closed containers map and its per-kind mode enum, and packaging's own closed property set.", + "description": "Declarative packaging modes (0.7.6): the top-level packaging block and its per-binding override, the isolated/shared mode enum, the non-empty pool and poolManifest strings, the closed containers map and its per-kind mode enum, and packaging's own closed property set.", "cases": [ { "description": "top-level packaging declares a shared pool with a per-kind containers override", diff --git a/tests/0.7.6/trigger-continuous.json b/tests/0.7.6/trigger-continuous.json new file mode 100644 index 0000000..479f5ec --- /dev/null +++ b/tests/0.7.6/trigger-continuous.json @@ -0,0 +1,1175 @@ +{ + "corpusVersion": "0.7.6", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.6.json", + "group": "trigger-continuous", + "description": "build.execution.trigger.type continuous (0.7.6): a long-lived job whose platform keeps it running, so its trigger takes no other trigger. cron, schedule, datasets, event and timetable are each refused next to it; every other trigger type keeps accepting them; a trigger that is not an object gets exactly one error; and each trigger member keeps its own type.", + "cases": [ + { + "description": "a continuous trigger names the type and nothing else", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuous" + } + } + } + }, + "valid": true + }, + { + "description": "a continuous trigger may carry the members that are not triggers (condition, datasetsOperator, timezone)", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuous", + "condition": "always", + "datasetsOperator": "all", + "timezone": "UTC" + } + } + } + }, + "valid": true + }, + { + "description": "a continuous trigger also carries cron", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuous", + "cron": "0 * * * *" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger", + "keyword": "not" + } + ] + }, + { + "description": "a continuous trigger also carries schedule", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuous", + "schedule": "@hourly" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger", + "keyword": "not" + } + ] + }, + { + "description": "a continuous trigger also carries datasets", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuous", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers" + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger", + "keyword": "not" + } + ] + }, + { + "description": "a continuous trigger also carries event", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuous", + "event": "payments.landed" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger", + "keyword": "not" + } + ] + }, + { + "description": "a continuous trigger also carries timetable", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuous", + "timetable": { + "class": "acme.Timetable" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger", + "keyword": "not" + } + ] + }, + { + "description": "a schedule trigger keeps carrying cron and schedule", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "cron": "0 * * * *", + "schedule": "@hourly", + "timezone": "UTC" + } + } + } + }, + "valid": true + }, + { + "description": "a dataset trigger keeps carrying datasets, and an event trigger keeps carrying event", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers" + ], + "event": "payments.landed" + } + } + } + }, + "valid": true + }, + { + "description": "a timetable trigger keeps carrying timetable", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "timetable", + "timetable": { + "class": "acme.Timetable" + } + } + } + } + }, + "valid": true + }, + { + "description": "a trigger with no type is not a continuous trigger, so it may carry cron", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "cron": "0 * * * *" + } + } + } + }, + "valid": true + }, + { + "description": "a trigger that is a string, not an object", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": "continuous" + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger", + "keyword": "type" + } + ] + }, + { + "description": "a trigger that is an array, not an object", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": [ + "continuous" + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger", + "keyword": "type" + } + ] + }, + { + "description": "a trigger type outside the enum, near continuous", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuously" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/type", + "keyword": "enum" + } + ] + }, + { + "description": "trigger.schedule has the wrong type", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "schedule": 5 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/schedule", + "keyword": "type" + } + ] + }, + { + "description": "trigger.event has the wrong type", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "event": 5 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/event", + "keyword": "type" + } + ] + }, + { + "description": "trigger.condition has the wrong type", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "condition": 5 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/condition", + "keyword": "type" + } + ] + }, + { + "description": "trigger.datasets has the wrong type", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "datasets": "dataset://gcp/acme-prod/customers/raw_customers" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/datasets", + "keyword": "type" + } + ] + }, + { + "description": "trigger.cron has the wrong type", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "cron": 5 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/cron", + "keyword": "type" + } + ] + }, + { + "description": "trigger.timezone has the wrong type", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "timezone": 5 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/timezone", + "keyword": "type" + } + ] + }, + { + "description": "trigger.timetable has the wrong type", + "contract": { + "fluidVersion": "0.7.6", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "timetable": "acme.Timetable" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/timetable", + "keyword": "type" + } + ] + } + ] +} diff --git a/tests/0.7.7/acquisition.json b/tests/0.7.7/acquisition.json new file mode 100644 index 0000000..6ffef97 --- /dev/null +++ b/tests/0.7.7/acquisition.json @@ -0,0 +1,5664 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "acquisition", + "description": "The acquisition build pattern: source-aligned ingestion from an external system (source, sink (its catalog, and from 0.7.6 the catalogAuth that names a REST catalog's credential by environment variable), delivery/DLQ semantics, schema evolution policy, pre-land quality gates and anomaly signals, cost budgets, catalog auto-registration, single-flight concurrency, lineage emission, the kafka-connect and debezium engine blocks, and the shared deployment block for engine-managed provisioning).", + "cases": [ + { + "description": "streaming CDC acquisition reaching every sub-block: source, sink, delivery, schema evolution, quality gates and anomalies, cost budget, catalog, concurrency lock, lineage, kafka-connect (schema registry + streaming sink + managed deployment), and debezium", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "cdc", + "connection": { + "host": "db.internal", + "port": 5432, + "database": "crm", + "secretRef": "vault://secret/data/crm-db", + "schema": "public", + "schemas": [ + "public", + "audit" + ] + }, + "cursor_field": "updated_at", + "watermark": { + "strategy": "lsn", + "allowedLateness": "PT5M" + }, + "streams": [ + "customers" + ], + "reader": { + "format": "avro" + } + }, + "sink": { + "format": "iceberg", + "catalog": "rest", + "partitionBy": [ + "ingestion_date" + ] + }, + "delivery": { + "guarantee": "exactly_once", + "dlq": { + "enabled": true, + "sink": { + "format": "parquet", + "location": "s3://acme-dlq/customers" + }, + "maxRecordsBeforeAbort": 5000, + "alertOn": [ + "schema_violation", + "destination_write_failed" + ] + } + }, + "schemaEvolution": { + "policy": "evolve_safe", + "onAddedColumn": "include", + "onRemovedColumn": "warn", + "onTypeChange": "cast", + "sourceFingerprint": "required" + }, + "preLand": [ + "dlp_scan", + "quality_gate" + ], + "quality": { + "gates": [ + { + "rule": "not_null", + "columns": [ + "customer_id" + ], + "severity": "error" + }, + { + "rule": "regex", + "column": "email", + "pattern": "^.+@.+$", + "severity": "warn" + } + ], + "onError": "route_to_dlq", + "anomalies": [ + { + "signal": "record_count_drop", + "algorithm": "ewma", + "sensitivity": 0.5, + "severity": "warn" + } + ] + }, + "cost": { + "budget": { + "monthly": { + "rows": 1000000, + "bytes": "50GB", + "computeMinutes": 600 + }, + "onExceed": "warn" + }, + "chargeback": { + "team": "customer-analytics" + } + }, + "catalog": { + "register": [ + "datahub", + "glue" + ], + "documentation": "auto" + }, + "concurrency": { + "lock": { + "scope": "product", + "timeout": "PT30M", + "onContended": "queue" + } + }, + "lineage": { + "emit": true + }, + "kafka-connect": { + "connector_class": "io.debezium.connector.postgresql.PostgresConnector", + "connector_name": "customers-source", + "tasks_max": 2, + "schema_registry": { + "url": "http://schema-registry:8081", + "auth": { + "secretRef": "vault://secret/data/sr-auth" + } + }, + "streamingSink": { + "commitIntervalMs": 60000, + "routeField": "table", + "dynamicEnabled": true + }, + "deployment": { + "mode": "managed", + "managed": { + "target": "kubernetes", + "profile": "medium", + "secrets": [ + { + "name": "sr-auth", + "ref": "vault://secret/data/sr-auth" + } + ], + "network": { + "egressAllowList": [ + "schema-registry.internal" + ] + } + } + } + }, + "debezium": { + "connector_class": "io.debezium.connector.postgresql.PostgresConnector", + "snapshot_mode": "initial" + } + } + } + }, + "valid": true + }, + { + "description": "acquisition build carrying only the required source kind and mode", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + } + } + } + }, + "valid": true + }, + { + "description": "preLand hook not in the recognized chain", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "preLand": [ + "dlp_scan", + "hand_wave_at_it" + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/preLand/1", + "keyword": "enum" + } + ] + }, + { + "description": "kafka-connect tasks_max at zero", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "kafka-connect": { + "tasks_max": 0 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/kafka-connect/tasks_max", + "keyword": "minimum" + } + ] + }, + { + "description": "debezium snapshot_mode outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "debezium": { + "snapshot_mode": "full_dump" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/debezium/snapshot_mode", + "keyword": "enum" + } + ] + }, + { + "description": "acquisitionPattern.lineage carrying an unrecognized member alongside emit", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "lineage": { + "emit": true, + "backend": "openlineage" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/lineage", + "keyword": "additionalProperties", + "detail": "backend" + } + ] + }, + { + "description": "kafka-connect schema_registry carrying an unrecognized member", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "kafka-connect": { + "schema_registry": { + "url": "http://sr:8081", + "compatibility": "BACKWARD" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/kafka-connect/schema_registry", + "keyword": "additionalProperties", + "detail": "compatibility" + } + ] + }, + { + "description": "kafka-connect schema_registry.auth carrying an unrecognized member", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "kafka-connect": { + "schema_registry": { + "auth": { + "secretRef": "vault://secret/data/sr", + "mechanism": "SASL" + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/kafka-connect/schema_registry/auth", + "keyword": "additionalProperties", + "detail": "mechanism" + } + ] + }, + { + "description": "kafka-connect streamingSink carrying an unrecognized member", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "kafka-connect": { + "streamingSink": { + "commitIntervalMs": 30000, + "flushOnBatch": true + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/kafka-connect/streamingSink", + "keyword": "additionalProperties", + "detail": "flushOnBatch" + } + ] + }, + { + "description": "kafka-connect streamingSink commitIntervalMs at zero", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "kafka-connect": { + "streamingSink": { + "commitIntervalMs": 0 + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/kafka-connect/streamingSink/commitIntervalMs", + "keyword": "minimum" + } + ] + }, + { + "description": "source omits the required kind", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "mode": "full_refresh" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source", + "keyword": "required", + "detail": "kind" + } + ] + }, + { + "description": "source omits the required mode", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source", + "keyword": "required", + "detail": "mode" + } + ] + }, + { + "description": "source carrying an unrecognized member alongside kind and mode", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh", + "driverVersion": "42.7.3" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source", + "keyword": "additionalProperties", + "detail": "driverVersion" + } + ] + }, + { + "description": "source connection secretRef with no scheme prefix", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh", + "connection": { + "secretRef": "crm-db-password" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source/connection/secretRef", + "keyword": "pattern" + } + ] + }, + { + "description": "source connection schemas repeating the same schema twice", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh", + "connection": { + "schemas": [ + "public", + "public" + ] + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source/connection/schemas", + "keyword": "uniqueItems" + } + ] + }, + { + "description": "source mode outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "batch_dump" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source/mode", + "keyword": "enum" + } + ] + }, + { + "description": "source watermark strategy outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh", + "watermark": { + "strategy": "sequence_number" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source/watermark/strategy", + "keyword": "enum" + } + ] + }, + { + "description": "source reader format outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh", + "reader": { + "format": "xml" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source/reader/format", + "keyword": "enum" + } + ] + }, + { + "description": "source watermark carrying an unrecognized member alongside strategy", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh", + "watermark": { + "strategy": "lsn", + "checkpointInterval": "PT1M" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source/watermark", + "keyword": "additionalProperties", + "detail": "checkpointInterval" + } + ] + }, + { + "description": "sink carrying an unrecognized member alongside format", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "format": "iceberg", + "compression": "zstd" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink", + "keyword": "additionalProperties", + "detail": "compression" + } + ] + }, + { + "description": "sink format outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "format": "avro_table" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/format", + "keyword": "enum" + } + ] + }, + { + "description": "sink catalog lakekeeper (recognized from 0.7.6)", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "lakekeeper" + } + } + } + }, + "valid": true + }, + { + "description": "sink catalog polaris (recognized from 0.7.6)", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "polaris" + } + } + } + }, + "valid": true + }, + { + "description": "sink catalog outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "hadoop" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalog", + "keyword": "enum" + } + ] + }, + { + "description": "sink catalogAuth oauth2 names the client-credentials variable, token endpoint, scope and audience (0.7.6)", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "lakekeeper", + "catalogAuth": { + "type": "oauth2", + "credentialEnv": "LAKEKEEPER_CLIENT", + "oauth2ServerUri": "https://keycloak.example.com/realms/lake/protocol/openid-connect/token", + "scope": "lakekeeper", + "audience": "lakekeeper" + } + } + } + } + }, + "valid": true + }, + { + "description": "sink catalogAuth bearer names the token variable (0.7.6)", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "unity", + "catalogAuth": { + "type": "bearer", + "tokenEnv": "UNITY_CATALOG_TOKEN" + } + } + } + } + }, + "valid": true + }, + { + "description": "sink catalogAuth omits the required type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "unity", + "catalogAuth": { + "tokenEnv": "UNITY_CATALOG_TOKEN" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "sink catalogAuth type outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "unity", + "catalogAuth": { + "type": "sigv4", + "tokenEnv": "UNITY_CATALOG_TOKEN" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth/type", + "keyword": "enum" + } + ] + }, + { + "description": "sink catalogAuth carries the token itself instead of naming its variable", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "unity", + "catalogAuth": { + "type": "bearer", + "tokenEnv": "UNITY_CATALOG_TOKEN", + "token": "redacted" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth", + "keyword": "additionalProperties", + "detail": "token" + } + ] + }, + { + "description": "sink catalogAuth oauth2 omits the required scope", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "lakekeeper", + "catalogAuth": { + "type": "oauth2", + "credentialEnv": "LAKEKEEPER_CLIENT", + "oauth2ServerUri": "https://keycloak.example.com/realms/lake/protocol/openid-connect/token" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth", + "keyword": "required", + "detail": "scope" + } + ] + }, + { + "description": "sink catalogAuth oauth2 also names a bearer tokenEnv", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "lakekeeper", + "catalogAuth": { + "type": "oauth2", + "credentialEnv": "LAKEKEEPER_CLIENT", + "oauth2ServerUri": "https://keycloak.example.com/realms/lake/protocol/openid-connect/token", + "scope": "lakekeeper", + "tokenEnv": "UNITY_CATALOG_TOKEN" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth", + "keyword": "not" + } + ] + }, + { + "description": "sink catalogAuth bearer omits the required tokenEnv", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "unity", + "catalogAuth": { + "type": "bearer" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth", + "keyword": "required", + "detail": "tokenEnv" + } + ] + }, + { + "description": "sink catalogAuth bearer also names an oauth2 credentialEnv", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "unity", + "catalogAuth": { + "type": "bearer", + "tokenEnv": "UNITY_CATALOG_TOKEN", + "credentialEnv": "LAKEKEEPER_CLIENT" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth", + "keyword": "not" + } + ] + }, + { + "description": "sink catalogAuth tokenEnv is a variable reference, not a variable name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "unity", + "catalogAuth": { + "type": "bearer", + "tokenEnv": "${UNITY_CATALOG_TOKEN}" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth/tokenEnv", + "keyword": "pattern" + } + ] + }, + { + "description": "sink catalogAuth tokenEnv is shaped like a token, not a variable name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "unity", + "catalogAuth": { + "type": "bearer", + "tokenEnv": "dapi0123456789abcdef0123456789" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth/tokenEnv", + "keyword": "not" + } + ] + }, + { + "description": "sink catalogAuth credentialEnv is shaped like a token, not a variable name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "lakekeeper", + "catalogAuth": { + "type": "oauth2", + "credentialEnv": "CLIENT_a1b2c3d4e5f6g7h8i9j0k1", + "oauth2ServerUri": "https://keycloak.example.com/realms/lake/protocol/openid-connect/token", + "scope": "lakekeeper" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth/credentialEnv", + "keyword": "not" + } + ] + }, + { + "description": "sink catalogAuth oauth2ServerUri carries a user:password@ part", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "lakekeeper", + "catalogAuth": { + "type": "oauth2", + "credentialEnv": "LAKEKEEPER_CLIENT", + "oauth2ServerUri": "https://client:secret@keycloak.example.com/realms/lake/token", + "scope": "lakekeeper" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth/oauth2ServerUri", + "keyword": "pattern" + } + ] + }, + { + "description": "sink catalogAuth scope is a variable reference", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "lakekeeper", + "catalogAuth": { + "type": "oauth2", + "credentialEnv": "LAKEKEEPER_CLIENT", + "oauth2ServerUri": "https://keycloak.example.com/realms/lake/protocol/openid-connect/token", + "scope": "${CATALOG_SCOPE}" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth/scope", + "keyword": "not" + } + ] + }, + { + "description": "sink catalogAuth oauth2 omits the required credentialEnv", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "lakekeeper", + "catalogAuth": { + "type": "oauth2", + "oauth2ServerUri": "https://keycloak.example.com/realms/lake/protocol/openid-connect/token", + "scope": "lakekeeper" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth", + "keyword": "required", + "detail": "credentialEnv" + } + ] + }, + { + "description": "sink catalogAuth oauth2 omits the required oauth2ServerUri", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "lakekeeper", + "catalogAuth": { + "type": "oauth2", + "credentialEnv": "LAKEKEEPER_CLIENT", + "scope": "lakekeeper" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth", + "keyword": "required", + "detail": "oauth2ServerUri" + } + ] + }, + { + "description": "sink catalogAuth credentialEnv is a variable reference, not a variable name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "lakekeeper", + "catalogAuth": { + "type": "oauth2", + "credentialEnv": "${LAKEKEEPER_CLIENT}", + "oauth2ServerUri": "https://keycloak.example.com/realms/lake/protocol/openid-connect/token", + "scope": "lakekeeper" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth/credentialEnv", + "keyword": "pattern" + } + ] + }, + { + "description": "sink catalogAuth scope is empty", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "lakekeeper", + "catalogAuth": { + "type": "oauth2", + "credentialEnv": "LAKEKEEPER_CLIENT", + "oauth2ServerUri": "https://keycloak.example.com/realms/lake/protocol/openid-connect/token", + "scope": "" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth/scope", + "keyword": "minLength" + } + ] + }, + { + "description": "sink catalogAuth resource starts with env:", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "lakekeeper", + "catalogAuth": { + "type": "oauth2", + "credentialEnv": "LAKEKEEPER_CLIENT", + "oauth2ServerUri": "https://keycloak.example.com/realms/lake/protocol/openid-connect/token", + "scope": "lakekeeper", + "resource": "env:CATALOG_RESOURCE" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalogAuth/resource", + "keyword": "not" + } + ] + }, + { + "description": "delivery guarantee outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "delivery": { + "guarantee": "best_effort" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/delivery/guarantee", + "keyword": "enum" + } + ] + }, + { + "description": "delivery dlq sink format outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "delivery": { + "dlq": { + "sink": { + "format": "avro" + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/delivery/dlq/sink/format", + "keyword": "enum" + } + ] + }, + { + "description": "delivery dlq maxRecordsBeforeAbort below zero", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "delivery": { + "dlq": { + "maxRecordsBeforeAbort": -1 + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/delivery/dlq/maxRecordsBeforeAbort", + "keyword": "minimum" + } + ] + }, + { + "description": "delivery dlq alertOn signal outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "delivery": { + "dlq": { + "alertOn": [ + "disk_full" + ] + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/delivery/dlq/alertOn/0", + "keyword": "enum" + } + ] + }, + { + "description": "delivery carrying an unrecognized member alongside guarantee", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "delivery": { + "guarantee": "at_least_once", + "retryPolicy": "exponential" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/delivery", + "keyword": "additionalProperties", + "detail": "retryPolicy" + } + ] + }, + { + "description": "delivery dlq carrying an unrecognized member alongside enabled", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "delivery": { + "dlq": { + "enabled": true, + "retentionDays": 30 + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/delivery/dlq", + "keyword": "additionalProperties", + "detail": "retentionDays" + } + ] + }, + { + "description": "delivery dlq sink carrying an unrecognized member alongside format", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "delivery": { + "dlq": { + "sink": { + "format": "json", + "compression": "gzip" + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/delivery/dlq/sink", + "keyword": "additionalProperties", + "detail": "compression" + } + ] + }, + { + "description": "schemaEvolution policy outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "schemaEvolution": { + "policy": "auto" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/schemaEvolution/policy", + "keyword": "enum" + } + ] + }, + { + "description": "schemaEvolution onAddedColumn outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "schemaEvolution": { + "onAddedColumn": "ignore" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/schemaEvolution/onAddedColumn", + "keyword": "enum" + } + ] + }, + { + "description": "schemaEvolution onRemovedColumn outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "schemaEvolution": { + "onRemovedColumn": "ignore" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/schemaEvolution/onRemovedColumn", + "keyword": "enum" + } + ] + }, + { + "description": "schemaEvolution onTypeChange outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "schemaEvolution": { + "onTypeChange": "ignore" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/schemaEvolution/onTypeChange", + "keyword": "enum" + } + ] + }, + { + "description": "schemaEvolution sourceFingerprint outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "schemaEvolution": { + "sourceFingerprint": "best_effort" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/schemaEvolution/sourceFingerprint", + "keyword": "enum" + } + ] + }, + { + "description": "schemaEvolution carrying an unrecognized member alongside policy", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "schemaEvolution": { + "policy": "strict", + "onSchemaMismatch": "fail" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/schemaEvolution", + "keyword": "additionalProperties", + "detail": "onSchemaMismatch" + } + ] + }, + { + "description": "quality gate omits the required rule", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "gates": [ + { + "severity": "error" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/gates/0", + "keyword": "required", + "detail": "rule" + } + ] + }, + { + "description": "quality gate omits the required severity", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "gates": [ + { + "rule": "not_null" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/gates/0", + "keyword": "required", + "detail": "severity" + } + ] + }, + { + "description": "quality gate rule outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "gates": [ + { + "rule": "checksum", + "severity": "error" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/gates/0/rule", + "keyword": "enum" + } + ] + }, + { + "description": "quality gate algorithm outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "gates": [ + { + "rule": "row_count_anomaly", + "severity": "warn", + "algorithm": "zscore" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/gates/0/algorithm", + "keyword": "enum" + } + ] + }, + { + "description": "quality gate severity outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "gates": [ + { + "rule": "not_null", + "severity": "critical" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/gates/0/severity", + "keyword": "enum" + } + ] + }, + { + "description": "quality onError outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "onError": "ignore" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/onError", + "keyword": "enum" + } + ] + }, + { + "description": "quality anomaly omits the required signal", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "anomalies": [ + { + "severity": "warn" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/anomalies/0", + "keyword": "required", + "detail": "signal" + } + ] + }, + { + "description": "quality anomaly omits the required severity", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "anomalies": [ + { + "signal": "cursor_stalled" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/anomalies/0", + "keyword": "required", + "detail": "severity" + } + ] + }, + { + "description": "quality anomaly signal outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "anomalies": [ + { + "signal": "disk_pressure", + "severity": "warn" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/anomalies/0/signal", + "keyword": "enum" + } + ] + }, + { + "description": "quality anomaly algorithm outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "anomalies": [ + { + "signal": "record_count_drop", + "severity": "warn", + "algorithm": "zscore" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/anomalies/0/algorithm", + "keyword": "enum" + } + ] + }, + { + "description": "quality anomaly severity outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "anomalies": [ + { + "signal": "record_count_drop", + "severity": "critical" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/anomalies/0/severity", + "keyword": "enum" + } + ] + }, + { + "description": "quality carrying an unrecognized member alongside onError", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "onError": "route_to_dlq", + "notifyChannel": "#data-alerts" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality", + "keyword": "additionalProperties", + "detail": "notifyChannel" + } + ] + }, + { + "description": "quality gate pattern given as a number instead of a regex string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "gates": [ + { + "rule": "regex", + "severity": "warn", + "pattern": 42 + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/gates/0/pattern", + "keyword": "type" + } + ] + }, + { + "description": "quality anomaly carrying an unrecognized member alongside signal and severity", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "anomalies": [ + { + "signal": "record_count_drop", + "severity": "warn", + "runbookUrl": "https://x" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/anomalies/0", + "keyword": "additionalProperties", + "detail": "runbookUrl" + } + ] + }, + { + "description": "cost budget monthly rows below zero", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "cost": { + "budget": { + "monthly": { + "rows": -1 + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/cost/budget/monthly/rows", + "keyword": "minimum" + } + ] + }, + { + "description": "cost budget monthly computeMinutes below zero", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "cost": { + "budget": { + "monthly": { + "computeMinutes": -1 + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/cost/budget/monthly/computeMinutes", + "keyword": "minimum" + } + ] + }, + { + "description": "cost budget onExceed outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "cost": { + "budget": { + "onExceed": "pause" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/cost/budget/onExceed", + "keyword": "enum" + } + ] + }, + { + "description": "cost carrying an unrecognized member alongside budget", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "cost": { + "budget": { + "onExceed": "warn" + }, + "currency": "USD" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/cost", + "keyword": "additionalProperties", + "detail": "currency" + } + ] + }, + { + "description": "cost budget carrying an unrecognized member alongside monthly", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "cost": { + "budget": { + "onExceed": "warn", + "annualCap": "600GB" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/cost/budget", + "keyword": "additionalProperties", + "detail": "annualCap" + } + ] + }, + { + "description": "cost budget monthly carrying an unrecognized member alongside rows", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "cost": { + "budget": { + "monthly": { + "rows": 1000, + "gbScanned": 5 + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/cost/budget/monthly", + "keyword": "additionalProperties", + "detail": "gbScanned" + } + ] + }, + { + "description": "catalog register target outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "catalog": { + "register": [ + "alation" + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/catalog/register/0", + "keyword": "enum" + } + ] + }, + { + "description": "catalog documentation outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "catalog": { + "documentation": "generated" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/catalog/documentation", + "keyword": "enum" + } + ] + }, + { + "description": "catalog carrying an unrecognized member alongside documentation", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "catalog": { + "documentation": "manual", + "owner": "data-platform" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/catalog", + "keyword": "additionalProperties", + "detail": "owner" + } + ] + }, + { + "description": "concurrency lock scope outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "concurrency": { + "lock": { + "scope": "organization" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/concurrency/lock/scope", + "keyword": "enum" + } + ] + }, + { + "description": "concurrency lock onContended outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "concurrency": { + "lock": { + "onContended": "retry_forever" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/concurrency/lock/onContended", + "keyword": "enum" + } + ] + }, + { + "description": "concurrency carrying an unrecognized member alongside lock", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "concurrency": { + "lock": { + "scope": "build" + }, + "maxParallelRuns": 1 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/concurrency", + "keyword": "additionalProperties", + "detail": "maxParallelRuns" + } + ] + }, + { + "description": "concurrency lock carrying an unrecognized member alongside scope", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "concurrency": { + "lock": { + "scope": "product", + "leaseId": "abc123" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/concurrency/lock", + "keyword": "additionalProperties", + "detail": "leaseId" + } + ] + }, + { + "description": "airbyte deployment omits the required mode", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": {} + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment", + "keyword": "required", + "detail": "mode" + } + ] + }, + { + "description": "airbyte deployment mode outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "self-hosted" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment/mode", + "keyword": "enum" + } + ] + }, + { + "description": "airbyte deployment managed block omits the required target", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "managed", + "managed": {} + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment/managed", + "keyword": "required", + "detail": "target" + } + ] + }, + { + "description": "airbyte deployment managed target outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "managed", + "managed": { + "target": "nomad" + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment/managed/target", + "keyword": "enum" + } + ] + }, + { + "description": "airbyte deployment managed profile outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "managed", + "managed": { + "target": "docker", + "profile": "xlarge" + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment/managed/profile", + "keyword": "enum" + } + ] + }, + { + "description": "airbyte deployment carrying an unrecognized member alongside mode", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "embedded", + "region": "us-east-1" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment", + "keyword": "additionalProperties", + "detail": "region" + } + ] + }, + { + "description": "airbyte deployment managed carrying an unrecognized member alongside target", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "managed", + "managed": { + "target": "docker", + "namespace": "data-platform" + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment/managed", + "keyword": "additionalProperties", + "detail": "namespace" + } + ] + }, + { + "description": "airbyte deployment managed secret entry carrying an unrecognized member", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "managed", + "managed": { + "target": "docker", + "secrets": [ + { + "name": "sr-auth", + "ref": "vault://secret/data/sr", + "rotate": true + } + ] + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment/managed/secrets/0", + "keyword": "additionalProperties", + "detail": "rotate" + } + ] + }, + { + "description": "airbyte deployment managed network carrying an unrecognized member", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "managed", + "managed": { + "target": "docker", + "network": { + "egressAllowList": [ + "api.stripe.com" + ], + "vpcId": "vpc-1" + } + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment/managed/network", + "keyword": "additionalProperties", + "detail": "vpcId" + } + ] + } + ] +} diff --git a/tests/0.7.7/agentpolicy.json b/tests/0.7.7/agentpolicy.json new file mode 100644 index 0000000..5649be7 --- /dev/null +++ b/tests/0.7.7/agentpolicy.json @@ -0,0 +1,715 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "agentpolicy", + "description": "agentPolicy, the AI/LLM usage-governance block reachable at exposes[].policy.agentPolicy: its eleven governance members, their types and bounds, and the closed property set that keeps 'reporting yes, training no' policies from silently drifting.", + "cases": [ + { + "description": "agentPolicy sets a full model, usage, and retention governance profile", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "agentPolicy": { + "allowedModels": ["gpt-4", "claude-3-opus", "claude-3-sonnet"], + "deniedModels": ["gpt-3.5-turbo-finetuned-marketing"], + "maxTokensPerRequest": 8192, + "maxTokensPerDay": 500000, + "allowedUseCases": ["inference", "reasoning", "summarization"], + "deniedUseCases": ["training", "fine_tuning"], + "canReason": true, + "canStore": false, + "retentionPolicy": { "maxRetentionDays": 30, "requireDeletion": true }, + "auditRequired": true, + "purposeLimitation": "Customer service chatbot only, not for marketing or profiling." + } + } + } + ] + }, + "valid": true + }, + { + "description": "agentPolicy present as an empty object with every member left to its default", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": {} } + } + ] + }, + "valid": true + }, + { + "description": "allowedModels is an empty array, denying all AI access", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "allowedModels": [] } } + } + ] + }, + "valid": true + }, + { + "description": "maxTokensPerRequest at its minimum boundary of one", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "maxTokensPerRequest": 1 } } + } + ] + }, + "valid": true + }, + { + "description": "retentionPolicy.maxRetentionDays set to zero, permitting no retention", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "retentionPolicy": { "maxRetentionDays": 0 } } } + } + ] + }, + "valid": true + }, + { + "description": "deniedModels carries an identifier with uppercase letters and spaces", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "deniedModels": ["GPT-4 Legacy (deprecated)"] } } + } + ] + }, + "valid": true + }, + { + "description": "allowedModels carries a single-character model identifier", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "allowedModels": ["x"] } } + } + ] + }, + "valid": true + }, + { + "description": "agentPolicy carries the shared tags and labels members alongside its governance fields", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "agentPolicy": { + "tags": ["pii", "real-time"], + "labels": { "team": "customer-analytics" } + } + } + } + ] + }, + "valid": true + }, + { + "description": "maxTokensPerRequest of zero", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "maxTokensPerRequest": 0 } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/maxTokensPerRequest", "keyword": "minimum" } + ] + }, + { + "description": "maxTokensPerDay is a negative number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "maxTokensPerDay": -1000 } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/maxTokensPerDay", "keyword": "minimum" } + ] + }, + { + "description": "maxTokensPerRequest given as a boolean", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "maxTokensPerRequest": true } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/maxTokensPerRequest", "keyword": "type" } + ] + }, + { + "description": "allowedModels given as a bare string instead of an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "allowedModels": "gpt-4" } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/allowedModels", "keyword": "type" } + ] + }, + { + "description": "allowedUseCases contains a use case outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "allowedUseCases": ["marketing"] } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/allowedUseCases/0", "keyword": "enum" } + ] + }, + { + "description": "deniedUseCases contains a use case outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "deniedUseCases": ["deletion"] } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/deniedUseCases/0", "keyword": "enum" } + ] + }, + { + "description": "allowedModels contains an uppercase model identifier", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "allowedModels": ["GPT-4"] } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/allowedModels/0", "keyword": "pattern" } + ] + }, + { + "description": "allowedModels contains a duplicate model identifier", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "allowedModels": ["gpt-4", "gpt-4"] } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/allowedModels", "keyword": "uniqueItems" } + ] + }, + { + "description": "retentionPolicy.maxRetentionDays is negative", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "retentionPolicy": { "maxRetentionDays": -1 } } } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/policy/agentPolicy/retentionPolicy/maxRetentionDays", + "keyword": "minimum" + } + ] + }, + { + "description": "agentPolicy carries a property outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "requiresHumanReview": true } } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/policy/agentPolicy", + "keyword": "additionalProperties", + "detail": "requiresHumanReview" + } + ] + }, + { + "description": "auditRequired given as an integer instead of a boolean", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "auditRequired": 1 } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/auditRequired", "keyword": "type" } + ] + }, + { + "description": "deniedModels given as a bare string instead of an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "deniedModels": "gpt-3.5-turbo" } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/deniedModels", "keyword": "type" } + ] + }, + { + "description": "allowedModels holds a non-string element", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "allowedModels": [42] } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/allowedModels/0", "keyword": "type" } + ] + }, + { + "description": "deniedModels holds a non-string element", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "deniedModels": [42] } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/deniedModels/0", "keyword": "type" } + ] + }, + { + "description": "maxTokensPerDay given as a boolean", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "maxTokensPerDay": true } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/maxTokensPerDay", "keyword": "type" } + ] + }, + { + "description": "canReason given as a string instead of a boolean", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "canReason": "yes" } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/canReason", "keyword": "type" } + ] + }, + { + "description": "canStore given as a string instead of a boolean", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "canStore": "no" } } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/agentPolicy/canStore", "keyword": "type" } + ] + }, + { + "description": "retentionPolicy.requireDeletion given as a string instead of a boolean", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { "agentPolicy": { "retentionPolicy": { "requireDeletion": "yes" } } } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/policy/agentPolicy/retentionPolicy/requireDeletion", + "keyword": "type" + } + ] + } + ] +} diff --git a/tests/0.7.7/binding-governance.json b/tests/0.7.7/binding-governance.json new file mode 100644 index 0000000..c5fc73b --- /dev/null +++ b/tests/0.7.7/binding-governance.json @@ -0,0 +1,1168 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "binding-governance", + "description": "Per-resource bindingGovernance mounted at exposes[].binding.governance: the closed governance and lakeFormation property sets, the grants[] items object (required principal/permissions, the ARN pattern on principal, the LF permission enum, minItems/uniqueItems on permissions, uniqueItems on permissionsWithGrantOption/columns/excludedColumns, and the closed per-grant property set), the rowFilter object (required name/rowExpression, uniqueItems on columnNames/excludedColumnNames, and its closed property set), and from 0.7.6 the maskedViews object (required definer/validationConnection and its closed property set).", + "cases": [ + { + "description": "bindingGovernance.lakeFormation with a column allow-list grant, a column deny-list grant, LF tags, and a row- and column-filtered rowFilter", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "registerLocation": true, + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + }, + { + "principal": "arn:aws:iam::123456789012:role/compliance-auditor", + "permissions": ["SELECT"], + "excludedColumns": ["ssn", "date_of_birth"] + } + ], + "tags": { + "pii-classification": "restricted" + }, + "rowFilter": { + "name": "region_filter", + "rowExpression": "region = 'us-east-1'", + "columnNames": ["customer_id", "region"] + } + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "governance carries a future cloud-specific block not yet defined in the schema", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "registerLocation": true, + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + }, + { + "principal": "arn:aws:iam::123456789012:role/compliance-auditor", + "permissions": ["SELECT"], + "excludedColumns": ["ssn", "date_of_birth"] + } + ], + "tags": { + "pii-classification": "restricted" + }, + "rowFilter": { + "name": "region_filter", + "rowExpression": "region = 'us-east-1'", + "columnNames": ["customer_id", "region"] + } + }, + "snowflakeMaskingPolicy": { + "enabled": true + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance", + "keyword": "additionalProperties", + "detail": "snowflakeMaskingPolicy" + } + ] + }, + { + "description": "lakeFormation carries a retention setting outside its documented property set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "registerLocation": true, + "retentionPolicy": "90d", + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + }, + { + "principal": "arn:aws:iam::123456789012:role/compliance-auditor", + "permissions": ["SELECT"], + "excludedColumns": ["ssn", "date_of_birth"] + } + ], + "tags": { + "pii-classification": "restricted" + }, + "rowFilter": { + "name": "region_filter", + "rowExpression": "region = 'us-east-1'", + "columnNames": ["customer_id", "region"] + } + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation", + "keyword": "additionalProperties", + "detail": "retentionPolicy" + } + ] + }, + { + "description": "lakeFormation maskedViews names the view definer role and the validation connection (0.7.6)", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { "bucket": "acme-lake", "path": "warehouse/customers/profiles" }, + "governance": { + "lakeFormation": { + "maskedViews": { + "definer": "arn:aws:iam::123456789012:role/fluid-view-definer", + "validationConnection": "athena-view-validation" + } + } + } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "lakeFormation maskedViews omits the required definer", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { "bucket": "acme-lake", "path": "warehouse/customers/profiles" }, + "governance": { + "lakeFormation": { + "maskedViews": { "validationConnection": "athena-view-validation" } + } + } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/maskedViews", + "keyword": "required", + "detail": "definer" + } + ] + }, + { + "description": "lakeFormation maskedViews omits the required validationConnection", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { "bucket": "acme-lake", "path": "warehouse/customers/profiles" }, + "governance": { + "lakeFormation": { + "maskedViews": { "definer": "arn:aws:iam::123456789012:role/fluid-view-definer" } + } + } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/maskedViews", + "keyword": "required", + "detail": "validationConnection" + } + ] + }, + { + "description": "lakeFormation maskedViews carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { "bucket": "acme-lake", "path": "warehouse/customers/profiles" }, + "governance": { + "lakeFormation": { + "maskedViews": { + "definer": "arn:aws:iam::123456789012:role/fluid-view-definer", + "validationConnection": "athena-view-validation", + "dialects": ["athena"] + } + } + } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/maskedViews", + "keyword": "additionalProperties", + "detail": "dialects" + } + ] + }, + { + "description": "a grant omits the required principal", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0", + "keyword": "required", + "detail": "principal" + } + ] + }, + { + "description": "a grant omits the required permissions", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0", + "keyword": "required", + "detail": "permissions" + } + ] + }, + { + "description": "a grant carries a grant-option-expiry property outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"], + "grantOptionExpiry": "2027-01-01" + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0", + "keyword": "additionalProperties", + "detail": "grantOptionExpiry" + } + ] + }, + { + "description": "a grant's principal is a bare role name instead of an IAM ARN", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "analytics-reader-role", + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0/principal", + "keyword": "pattern" + } + ] + }, + { + "description": "a grant's permissions array is empty", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": [], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0/permissions", + "keyword": "minItems" + } + ] + }, + { + "description": "a grant repeats the same permission twice", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "SELECT"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0/permissions", + "keyword": "uniqueItems" + } + ] + }, + { + "description": "a grant's permissions list includes a value outside the LF permission enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "MODIFY"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0/permissions/1", + "keyword": "enum" + } + ] + }, + { + "description": "a grant's permissionsWithGrantOption repeats the same permission twice", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT", "SELECT"], + "columns": ["customer_id", "email", "region"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0/permissionsWithGrantOption", + "keyword": "uniqueItems" + } + ] + }, + { + "description": "a grant's columns list repeats the same column twice", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "customer_id"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0/columns", + "keyword": "uniqueItems" + } + ] + }, + { + "description": "a grant's excludedColumns list repeats the same column twice", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/compliance-auditor", + "permissions": ["SELECT"], + "excludedColumns": ["ssn", "ssn"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0/excludedColumns", + "keyword": "uniqueItems" + } + ] + }, + { + "description": "the rowFilter omits the required name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "rowFilter": { + "rowExpression": "region = 'us-east-1'", + "columnNames": ["customer_id", "region"] + } + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/rowFilter", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "the rowFilter omits the required rowExpression", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "rowFilter": { + "name": "region_filter", + "columnNames": ["customer_id", "region"] + } + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/rowFilter", + "keyword": "required", + "detail": "rowExpression" + } + ] + }, + { + "description": "the rowFilter carries a priority property outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "rowFilter": { + "name": "region_filter", + "rowExpression": "region = 'us-east-1'", + "columnNames": ["customer_id", "region"], + "priority": 1 + } + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/rowFilter", + "keyword": "additionalProperties", + "detail": "priority" + } + ] + }, + { + "description": "the rowFilter's columnNames repeats the same column twice", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "rowFilter": { + "name": "region_filter", + "rowExpression": "region = 'us-east-1'", + "columnNames": ["customer_id", "customer_id"] + } + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/rowFilter/columnNames", + "keyword": "uniqueItems" + } + ] + }, + { + "description": "the rowFilter's excludedColumnNames repeats the same column twice", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "rowFilter": { + "name": "region_filter", + "rowExpression": "region = 'us-east-1'", + "excludedColumnNames": ["ssn", "ssn"] + } + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/rowFilter/excludedColumnNames", + "keyword": "uniqueItems" + } + ] + } + ] +} diff --git a/tests/0.7.7/binding-security.json b/tests/0.7.7/binding-security.json new file mode 100644 index 0000000..449a62d --- /dev/null +++ b/tests/0.7.7/binding-security.json @@ -0,0 +1,648 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "binding-security", + "description": "Binding encryption and principal mapping (0.7.6): encryption's closed property set; kms as 'product', 'none', an AWS alias or key/alias ARN, or a Cloud KMS crypto-key name; the platform-conditional refusals (a gcp binding refuses AWS key forms, an aws binding refuses Cloud KMS names); principals' non-empty keys, one-identity-or-unique-list values, and the platform-conditional identity forms (IAM members on gcp, IAM ARNs on aws). Also documents that platforms other than gcp and aws carry no principal-form constraint.", + "cases": [ + { + "description": "a gcp binding uses a Cloud KMS key and maps logical principals to IAM members", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + }, + "encryption": { + "kms": "projects/acme-prod/locations/europe-west1/keyRings/fluid/cryptoKeys/customers" + }, + "principals": { + "analysts": "group:analysts@acme.example", + "stewards": [ + "user:steward@acme.example", + "serviceAccount:etl@acme-prod.iam.gserviceaccount.com" + ], + "outsiders": [] + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "an aws binding uses a KMS key ARN and maps logical principals to IAM ARNs", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "encryption": { + "kms": "arn:aws:kms:eu-west-1:123456789012:key/1234abcd-12ab-34cd-56ef-1234567890ab" + }, + "principals": { + "analysts": "arn:aws:iam::123456789012:role/analytics-reader" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "an aws binding names its key by alias", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "encryption": { + "kms": "alias/fluid/customers" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "kms 'product' and 'none' are accepted on any platform", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + }, + "encryption": { + "kms": "none" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "a platform other than gcp or aws maps principals to any non-empty identity", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "snowflake", + "format": "snowflake_table", + "location": { + "database": "ANALYTICS", + "schema": "PUBLIC", + "table": "PROFILES" + }, + "principals": { + "analysts": "ANALYST_ROLE" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "encryption carries a property outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + }, + "encryption": { + "kms": "product", + "algorithm": "AES256" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/encryption", + "keyword": "additionalProperties", + "detail": "algorithm" + } + ] + }, + { + "description": "kms is neither a keyword nor a recognised key name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + }, + "encryption": { + "kms": "customer-managed" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/encryption/kms", + "keyword": "anyOf" + } + ] + }, + { + "description": "a gcp binding names an AWS KMS alias", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + }, + "encryption": { + "kms": "alias/fluid/customers" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/encryption/kms", + "keyword": "not" + } + ] + }, + { + "description": "an aws binding names a Cloud KMS crypto key", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "encryption": { + "kms": "projects/acme-prod/locations/europe-west1/keyRings/fluid/cryptoKeys/customers" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/encryption/kms", + "keyword": "not" + } + ] + }, + { + "description": "a gcp principal maps to an identity that is not an IAM member", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + }, + "principals": { + "analysts": "analysts@acme.example" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/principals/analysts", + "keyword": "anyOf" + } + ] + }, + { + "description": "an aws principal maps to an identity that is not an IAM ARN", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "principals": { + "analysts": "group:analysts@acme.example" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/principals/analysts", + "keyword": "anyOf" + } + ] + }, + { + "description": "a principal maps to a list that repeats an identity", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + }, + "principals": { + "analysts": [ + "group:analysts@acme.example", + "group:analysts@acme.example" + ] + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/principals/analysts", + "keyword": "anyOf" + } + ] + }, + { + "description": "a principal maps to an empty identity string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "snowflake", + "format": "snowflake_table", + "location": { + "database": "ANALYTICS", + "schema": "PUBLIC", + "table": "PROFILES" + }, + "principals": { + "analysts": "" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/principals/analysts", + "keyword": "anyOf" + } + ] + }, + { + "description": "principals is a list instead of a map", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + }, + "principals": [ + "group:analysts@acme.example" + ] + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/principals", + "keyword": "type" + } + ] + } + ] +} diff --git a/tests/0.7.7/binding.json b/tests/0.7.7/binding.json new file mode 100644 index 0000000..0c5df8a --- /dev/null +++ b/tests/0.7.7/binding.json @@ -0,0 +1,2099 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "binding", + "description": "The binding object embedded in every expose: its required platform/format/location members, the platform and format enums, the closed bindingLocation property set (including the v0.7.5 Confluent Tableflow, Iceberg catalog-profile, Redshift Serverless and Kinesis fields), icebergConfig's own structure -- writeVersion/fileFormat enums, its closed property set, and the partitionSpec/sortOrder arrays' per-entry required fields and enums -- and the v0.7.5 vector output port: platform pgvector, format pgvector_table, and vectorConfig's closed property set, required dimensions and its inclusive 1..16000 bounds, its vectorType/indexType/distanceMetric enums, and the closed hnsw/ivfflat parameter objects with their minimums. Also documents that the schema does not cross-validate platform against format or location fields against platform.", + "cases": [ + { + "description": "binding carrying only the three required members", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "bindingLocation combines the general-purpose addressing properties in one object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "other", + "location": { + "account": "123456789012", + "project": "acme-prod", + "dataset": "customers", + "database": "analytics", + "schema": "public", + "table": "profiles", + "bucket": "acme-data-lake", + "path": "raw/customers/profiles/", + "gateway": "internal-gw.acme.com", + "baseUrl": "https://api.acme.com/v1/customers", + "topic": "customer-profile-updates", + "subscription": "profile-updates-sub", + "region": "us-central1", + "zone": "us-central1-a" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "Confluent Tableflow location fields accompany a confluent platform binding", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "confluent", + "format": "iceberg", + "location": { + "topic": "orders", + "environment_id": "env-abc123", + "kafka_cluster_id": "lkc-xyz789", + "confluent_role_arn": "arn:aws:iam::123456789012:role/confluent-tableflow-role" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "platform and format name binding to unrelated cloud ecosystems", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "azure", + "format": "bigquery_table", + "location": { + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "icebergConfig specifies partitioning, sort order, and table properties in full", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "icebergConfig": { + "writeVersion": 2, + "fileFormat": "parquet", + "partitionSpec": [ + { + "name": "created_year", + "sourceColumn": "created_at", + "transform": "year" + }, + { + "name": "customer_bucket", + "sourceColumn": "customer_id", + "transform": "bucket", + "numBuckets": 16 + } + ], + "sortOrder": [ + { + "column": "created_at", + "direction": "desc", + "nullOrder": "nulls-last" + } + ], + "properties": { + "write.format.default": "parquet", + "history.expire.max-snapshot-age-ms": "604800000" + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "icebergConfig at writeVersion 1 carries only a file format", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "databricks", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/legacy/" + }, + "icebergConfig": { + "writeVersion": 1, + "fileFormat": "avro" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "platform is absent from the binding", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "format": "kafka_topic", + "location": { + "topic": "customer-events" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding", + "keyword": "required", + "detail": "platform" + } + ] + }, + { + "description": "format is absent from the binding", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "location": { + "topic": "customer-events" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding", + "keyword": "required", + "detail": "format" + } + ] + }, + { + "description": "location is absent from the binding", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic" + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding", + "keyword": "required", + "detail": "location" + } + ] + }, + { + "description": "platform names a cloud provider outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "digitalocean", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/platform", + "keyword": "enum" + } + ] + }, + { + "description": "format names a file type outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "excel_sheet", + "location": { + "topic": "customer-events" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/format", + "keyword": "enum" + } + ] + }, + { + "description": "bindingLocation carries an undocumented addressing property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events", + "bootstrapServers": "kafka.internal:9092" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/location", + "keyword": "additionalProperties", + "detail": "bootstrapServers" + } + ] + }, + { + "description": "binding carries a property outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "connectionString": "kafka://broker1:9092,broker2:9092" + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding", + "keyword": "additionalProperties", + "detail": "connectionString" + } + ] + }, + { + "description": "icebergConfig.writeVersion is outside the {1,2} enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "databricks", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/legacy/" + }, + "icebergConfig": { + "writeVersion": 3, + "fileFormat": "avro" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/icebergConfig/writeVersion", + "keyword": "enum" + } + ] + }, + { + "description": "icebergConfig.fileFormat names a format outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "databricks", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/legacy/" + }, + "icebergConfig": { + "writeVersion": 1, + "fileFormat": "csv" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/icebergConfig/fileFormat", + "keyword": "enum" + } + ] + }, + { + "description": "icebergConfig carries a property outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "databricks", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/legacy/" + }, + "icebergConfig": { + "writeVersion": 1, + "fileFormat": "parquet", + "compressionCodec": "zstd" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/icebergConfig", + "keyword": "additionalProperties", + "detail": "compressionCodec" + } + ] + }, + { + "description": "a partitionSpec entry omits the required transform", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "icebergConfig": { + "writeVersion": 2, + "fileFormat": "parquet", + "partitionSpec": [ + { + "sourceColumn": "created_at" + } + ] + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/icebergConfig/partitionSpec/0", + "keyword": "required", + "detail": "transform" + } + ] + }, + { + "description": "bindingLocation.baseUrl is a number instead of a string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "http_api", + "location": { + "baseUrl": 42 + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/location/baseUrl", + "keyword": "type" + } + ] + }, + { + "description": "a partitionSpec entry omits the required sourceColumn", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "icebergConfig": { + "writeVersion": 2, + "fileFormat": "parquet", + "partitionSpec": [ + { + "transform": "identity" + } + ] + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/icebergConfig/partitionSpec/0", + "keyword": "required", + "detail": "sourceColumn" + } + ] + }, + { + "description": "a partitionSpec entry names a transform outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "icebergConfig": { + "writeVersion": 2, + "fileFormat": "parquet", + "partitionSpec": [ + { + "sourceColumn": "id", + "transform": "quarterly" + } + ] + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/icebergConfig/partitionSpec/0/transform", + "keyword": "enum" + } + ] + }, + { + "description": "a sortOrder entry omits the required column", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "icebergConfig": { + "writeVersion": 2, + "fileFormat": "parquet", + "sortOrder": [ + { + "direction": "asc" + } + ] + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/icebergConfig/sortOrder/0", + "keyword": "required", + "detail": "column" + } + ] + }, + { + "description": "pgvector platform with a pgvector_table format and only the required vectorConfig member", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 1536 + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "vectorConfig specifies every member, with an HNSW index", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 1024, + "embeddingModel": "embed-english-light", + "vectorType": "halfvec", + "indexType": "hnsw", + "distanceMetric": "inner_product", + "table": "profile_embeddings", + "sourceKeyColumn": "customer_id", + "hnsw": { + "m": 16, + "efConstruction": 64 + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "vectorConfig selects an IVFFlat index with the smallest admitted list count", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 1536, + "indexType": "ivfflat", + "distanceMetric": "l2", + "ivfflat": { + "lists": 1 + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "vectorConfig.dimensions at its inclusive lower bound of 1", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 1, + "indexType": "none", + "distanceMetric": "l1" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "vectorConfig.dimensions at its inclusive upper bound of 16000", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 16000, + "vectorType": "vector", + "distanceMetric": "cosine" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "vectorConfig omits the required dimensions", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "embeddingModel": "text-embedding-3-small" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig", + "keyword": "required", + "detail": "dimensions" + } + ] + }, + { + "description": "vectorConfig carries a property outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 1536, + "metric": "cosine" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig", + "keyword": "additionalProperties", + "detail": "metric" + } + ] + }, + { + "description": "vectorConfig.dimensions is zero, below the minimum of 1", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 0 + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig/dimensions", + "keyword": "minimum" + } + ] + }, + { + "description": "vectorConfig.dimensions is 16001, above the maximum of 16000", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 16001 + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig/dimensions", + "keyword": "maximum" + } + ] + }, + { + "description": "vectorConfig.dimensions is a string instead of an integer", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": "1536" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig/dimensions", + "keyword": "type" + } + ] + }, + { + "description": "vectorConfig is a bare dimension count instead of an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": 1536 + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig", + "keyword": "type" + } + ] + }, + { + "description": "vectorConfig.vectorType names a column type outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 1536, + "vectorType": "sparsevec" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig/vectorType", + "keyword": "enum" + } + ] + }, + { + "description": "vectorConfig.indexType names an index method outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 1536, + "indexType": "diskann" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig/indexType", + "keyword": "enum" + } + ] + }, + { + "description": "vectorConfig.distanceMetric names a metric outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 1536, + "distanceMetric": "hamming" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig/distanceMetric", + "keyword": "enum" + } + ] + }, + { + "description": "vectorConfig.hnsw carries a query-time parameter outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 1536, + "hnsw": { + "m": 16, + "efSearch": 40 + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig/hnsw", + "keyword": "additionalProperties", + "detail": "efSearch" + } + ] + }, + { + "description": "vectorConfig.hnsw.m is zero, below the minimum of 1", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 1536, + "hnsw": { + "m": 0 + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig/hnsw/m", + "keyword": "minimum" + } + ] + }, + { + "description": "vectorConfig.hnsw.efConstruction is zero, below the minimum of 1", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 1536, + "hnsw": { + "efConstruction": 0 + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig/hnsw/efConstruction", + "keyword": "minimum" + } + ] + }, + { + "description": "vectorConfig.hnsw.m is a fraction instead of an integer", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 1536, + "hnsw": { + "m": 16.5 + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig/hnsw/m", + "keyword": "type" + } + ] + }, + { + "description": "vectorConfig.ivfflat carries a query-time parameter outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 1536, + "ivfflat": { + "lists": 100, + "probes": 10 + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig/ivfflat", + "keyword": "additionalProperties", + "detail": "probes" + } + ] + }, + { + "description": "vectorConfig.ivfflat.lists is zero, below the minimum of 1", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "pgvector", + "format": "pgvector_table", + "location": { + "database": "rag", + "schema": "public", + "table": "profile_embeddings" + }, + "vectorConfig": { + "dimensions": 1536, + "ivfflat": { + "lists": 0 + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/vectorConfig/ivfflat/lists", + "keyword": "minimum" + } + ] + }, + { + "description": "Iceberg catalog-profile location fields accompany an iceberg binding", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "catalog": "rest", + "warehouse": "s3://acme-lakehouse/warehouse", + "uri": "https://catalog.acme.example/api/catalog", + "database": "sales", + "table": "orders", + "partitionBy": [ + "order_date", + "region" + ] + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "Redshift Serverless and Kinesis location fields accompany an aws binding", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "redshift_external_schema", + "location": { + "namespace": "analytics-ns", + "workgroup": "analytics-wg", + "iam_role_arn": "arn:aws:iam::123456789012:role/redshift-spectrum", + "external_schema": "lake", + "glue_database": "sales_lake", + "stream": "order-events", + "region": "eu-west-1" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "bindingLocation.partitionBy is a single string instead of an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "catalog": "glue", + "warehouse": "s3://acme-lakehouse/warehouse", + "partitionBy": "order_date" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/location/partitionBy", + "keyword": "type" + } + ] + }, + { + "description": "a bindingLocation.partitionBy entry is a number instead of a column name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "catalog": "glue", + "warehouse": "s3://acme-lakehouse/warehouse", + "partitionBy": [ + 2026 + ] + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/location/partitionBy/0", + "keyword": "type" + } + ] + } + ] +} diff --git a/tests/0.7.7/bitol-fields.json b/tests/0.7.7/bitol-fields.json new file mode 100644 index 0000000..61255e1 --- /dev/null +++ b/tests/0.7.7/bitol-fields.json @@ -0,0 +1,2416 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "bitol-fields", + "description": "Optional Bitol-shaped fields added in 0.7.6: metadata.team (a roster, each member needing only a username), the root description as a string or as {purpose, usage, limitations}, consumes[].contractId and contractVersion, binding.location.host and binding.serialization for Kafka, and the ODCS-shaped quality rules on a column.", + "cases": [ + { + "description": "metadata.team names a team and a roster whose member carries every field", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "name": "Sales Analytics", + "members": [ + { + "username": "mjones", + "name": "Mia Jones", + "role": "data owner", + "email": "mia.jones@example.com" + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "metadata.team with a member that carries only the required username", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + { + "username": "mjones" + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "metadata.team with a name and no members", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "name": "Sales Analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "metadata.team carries a member outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "name": "Sales Analytics", + "lead": "mjones" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team", + "keyword": "additionalProperties", + "detail": "lead" + } + ] + }, + { + "description": "metadata.team is a string, not an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": "Sales Analytics" + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team", + "keyword": "type" + } + ] + }, + { + "description": "metadata.team.name is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "name": 7 + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/name", + "keyword": "type" + } + ] + }, + { + "description": "metadata.team.members is an object, not an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": { + "username": "mjones" + } + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members", + "keyword": "type" + } + ] + }, + { + "description": "a team member omits the required username", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + { + "name": "Mia Jones" + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members/0", + "keyword": "required", + "detail": "username" + } + ] + }, + { + "description": "a team member carries a property outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + { + "username": "mjones", + "phone": "555" + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members/0", + "keyword": "additionalProperties", + "detail": "phone" + } + ] + }, + { + "description": "a team member is a string, not an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + "mjones" + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members/0", + "keyword": "type" + } + ] + }, + { + "description": "a team member's username is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + { + "username": 7 + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members/0/username", + "keyword": "type" + } + ] + }, + { + "description": "a team member's name is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + { + "username": "mjones", + "name": 7 + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members/0/name", + "keyword": "type" + } + ] + }, + { + "description": "a team member's role is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + { + "username": "mjones", + "role": 7 + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members/0/role", + "keyword": "type" + } + ] + }, + { + "description": "a team member's email is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + }, + "team": { + "members": [ + { + "username": "mjones", + "email": 7 + } + ] + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata/team/members/0/email", + "keyword": "type" + } + ] + }, + { + "description": "root description as a plain string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "description": "Customer profiles for analytics." + }, + "valid": true + }, + { + "description": "root description as an object with purpose, usage and limitations", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "description": { + "purpose": "Customer profiles.", + "usage": "Join on customer_id.", + "limitations": "Not for marketing." + } + }, + "valid": true + }, + { + "description": "root description is a number: neither a string nor the object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "description": 7 + }, + "valid": false, + "errors": [ + { + "pointer": "/description", + "keyword": "oneOf" + } + ] + }, + { + "description": "root description object carries a member outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "description": { + "purpose": "Customer profiles.", + "audience": "analysts" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/description", + "keyword": "oneOf" + } + ] + }, + { + "description": "root description object: purpose is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "description": { + "purpose": 7 + } + }, + "valid": false, + "errors": [ + { + "pointer": "/description", + "keyword": "oneOf" + } + ] + }, + { + "description": "root description object: usage is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "description": { + "usage": 7 + } + }, + "valid": false, + "errors": [ + { + "pointer": "/description", + "keyword": "oneOf" + } + ] + }, + { + "description": "root description object: limitations is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "description": { + "limitations": 7 + } + }, + "valid": false, + "errors": [ + { + "pointer": "/description", + "keyword": "oneOf" + } + ] + }, + { + "description": "a consumes entry names the upstream's own contract id and the exact version built against", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumes": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "contractId": "urn:acme:contracts:raw-customers", + "versionConstraint": "^2.0.0", + "contractVersion": "2.1.0" + } + ] + }, + "valid": true + }, + { + "description": "consumes[].contractId is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumes": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "contractId": 7 + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/consumes/0/contractId", + "keyword": "type" + } + ] + }, + { + "description": "consumes[].contractVersion is not a semantic version", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumes": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "contractVersion": "2.1" + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/consumes/0/contractVersion", + "keyword": "pattern" + } + ] + }, + { + "description": "a Kafka binding location names the bootstrap servers as host", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events", + "host": "kafka.internal:9092" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "binding.location.host is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events", + "host": 9092 + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/location/host", + "keyword": "type" + } + ] + }, + { + "description": "binding.serialization declares json values with json keys", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "json", + "key": "json" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "binding.serialization declares avro values with avro keys", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "avro", + "key": "avro" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "binding.serialization declares protobuf values with protobuf keys", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "protobuf", + "key": "protobuf" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "binding.serialization declares xml values with xml keys", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "xml", + "key": "xml" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "binding.serialization declares avro-confluent values with avro-confluent keys", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "avro-confluent", + "key": "avro-confluent" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "binding.serialization declares json-confluent values with json-confluent keys", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "json-confluent", + "key": "json-confluent" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "binding.serialization.value is outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "csv" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/serialization/value", + "keyword": "enum" + } + ] + }, + { + "description": "binding.serialization.key is outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "key": "csv" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/serialization/key", + "keyword": "enum" + } + ] + }, + { + "description": "binding.serialization carries a member outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": { + "value": "json", + "headers": "json" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/serialization", + "keyword": "additionalProperties", + "detail": "headers" + } + ] + }, + { + "description": "binding.serialization is a string, not an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { + "topic": "customer-events" + }, + "serialization": "json" + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/serialization", + "keyword": "type" + } + ] + }, + { + "description": "column quality rules in ODCS shape: library, sql, text and custom", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "id": "customer_id-nullValues", + "type": "library", + "metric": "nullValues", + "dimension": "completeness", + "severity": "error", + "unit": "rows", + "description": "No nulls." + }, + { + "id": "customer_id-invalid", + "type": "library", + "metric": "invalidValues", + "arguments": { + "pattern": "^C[0-9]+$" + } + }, + { + "id": "customer_id-sql", + "type": "sql", + "query": "SELECT COUNT(*) FROM profiles WHERE customer_id IS NULL" + }, + { + "id": "customer_id-text", + "type": "text", + "description": "Assigned by the CRM." + }, + { + "id": "customer_id-soda", + "type": "custom", + "engine": "soda", + "implementation": "checks for profiles: ..." + } + ] + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "column quality is an object, not an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": { + "id": "a" + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule is an empty object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + {} + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0", + "keyword": "minProperties" + } + ] + }, + { + "description": "a quality rule is a string, not an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + "nullValues" + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule id with a character outside the id pattern", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "id": "no nulls!" + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/id", + "keyword": "pattern" + } + ] + }, + { + "description": "a quality rule id is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "id": 7 + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/id", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule type is outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "type": "regex" + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/type", + "keyword": "enum" + } + ] + }, + { + "description": "a quality rule metric is outside the five ODCS library metrics", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "type": "library", + "metric": "outliers" + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/metric", + "keyword": "enum" + } + ] + }, + { + "description": "a quality rule dimension is outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "dimension": "freshness" + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/dimension", + "keyword": "enum" + } + ] + }, + { + "description": "quality rule arguments is an array, not an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "type": "library", + "arguments": [ + "a" + ] + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/arguments", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule query is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "query": 7 + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/query", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule engine is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "engine": 7 + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/engine", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule severity is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "severity": 7 + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/severity", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule unit is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "unit": 7 + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/unit", + "keyword": "type" + } + ] + }, + { + "description": "a quality rule description is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "quality": [ + { + "description": 7 + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/quality/0/description", + "keyword": "type" + } + ] + } + ] +} diff --git a/tests/0.7.7/build-consumes.json b/tests/0.7.7/build-consumes.json new file mode 100644 index 0000000..1391238 --- /dev/null +++ b/tests/0.7.7/build-consumes.json @@ -0,0 +1,911 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "build-consumes", + "description": "The build block and its four pattern variants (hybrid-reference, embedded-logic, multi-stage, acquisition) -- including each variant's own closed property set, not just its required member -- the build-level capabilities vocabulary and dependencies[].condition enum, the builds array of independent build configurations, the consumes array and consumeRef, and the inline environments overrides object. A conformant implementation reproduces every verdict below.", + "cases": [ + { + "description": "hybrid-reference build carries every optional member: outputs, dependencies, tags, labels, an adapter-qualified engine, and pattern-specific select/materializations/vars", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "id": "stg_customers_build", + "description": "Staging build for customer profile enrichment.", + "pattern": "hybrid-reference", + "engine": "dbt-snowflake", + "outputs": ["profiles_table"], + "dependencies": [{ "buildId": "raw_customers_build", "condition": "success" }], + "tags": ["batch"], + "labels": { "team": "customer-analytics" }, + "properties": { + "model": "stg_customers", + "select": ["stg_customers", "+downstream"], + "materializations": { "stg_customers": "incremental" }, + "vars": { "start_date": "2024-01-01" } + } + } + }, + "valid": true + }, + { + "description": "build.pattern is set to a value outside its four-member enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { "pattern": "streaming", "engine": "dbt", "properties": { "model": "stg_customers" } } + }, + "valid": false, + "errors": [{ "pointer": "/build/pattern", "keyword": "enum" }] + }, + { + "description": "build carries a top-level member the schema does not define", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { "model": "stg_customers" }, + "priority": 5 + } + }, + "valid": false, + "errors": [{ "pointer": "/build", "keyword": "additionalProperties", "detail": "priority" }] + }, + { + "description": "build.engine matches neither the fixed enum nor the adapter-qualified dbt- pattern", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "snowflake-native", + "properties": { "model": "stg_customers" } + } + }, + "valid": false, + "errors": [{ "pointer": "/build/engine", "keyword": "anyOf" }] + }, + { + "description": "hybrid-reference build's properties omit the required model", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { "pattern": "hybrid-reference", "engine": "dbt", "properties": {} } + }, + "valid": false, + "errors": [{ "pointer": "/build/properties", "keyword": "required", "detail": "model" }] + }, + { + "description": "hybrid-reference build's properties carry an embedded-logic field (sql) that its own closed set does not admit", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { "model": "stg_customers", "sql": "SELECT 1" } + } + }, + "valid": false, + "errors": [{ "pointer": "/build/properties", "keyword": "additionalProperties", "detail": "sql" }] + }, + { + "description": "embedded-logic build carries sql, a language choice, and free-form parameters", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "python", + "properties": { + "sql": "SELECT * FROM raw.customers", + "language": "pyspark", + "parameters": { "batch_size": 100 } + } + } + }, + "valid": true + }, + { + "description": "build.engine flink (new in 0.7.6) runs Flink SQL, with the deployment settings under parameters", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments", + "language": "flink_sql", + "parameters": { + "parallelism": 4, + "upgradeMode": "savepoint", + "flinkConfiguration": { "execution.checkpointing.interval": "60s" } + } + } + } + }, + "valid": true + }, + { + "description": "embedded-logic build's properties omit the required sql", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { "pattern": "embedded-logic", "engine": "sql", "properties": { "language": "sql" } } + }, + "valid": false, + "errors": [{ "pointer": "/build/properties", "keyword": "required", "detail": "sql" }] + }, + { + "description": "embedded-logic build's language falls outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "sql", + "properties": { "sql": "SELECT 1", "language": "javascript" } + } + }, + "valid": false, + "errors": [{ "pointer": "/build/properties/language", "keyword": "enum" }] + }, + { + "description": "embedded-logic build's properties carry a hybrid-reference field (model) that its own closed set does not admit", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "sql", + "properties": { "sql": "SELECT 1", "model": "should_not_be_here" } + } + }, + "valid": false, + "errors": [{ "pointer": "/build/properties", "keyword": "additionalProperties", "detail": "model" }] + }, + { + "description": "hybrid-reference build declares a dependency whose condition falls outside its three-member enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { "model": "stg_customers" }, + "dependencies": [{ "buildId": "raw_customers_build", "condition": "sometimes" }] + } + }, + "valid": false, + "errors": [{ "pointer": "/build/dependencies/0/condition", "keyword": "enum" }] + }, + { + "description": "multi-stage build orchestrates two dependent stages under a bounded parallelism", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "multi-stage", + "engine": "spark", + "properties": { + "stages": [ + { "name": "extract", "pattern": "embedded-logic", "outputs": ["raw_stage"] }, + { "name": "transform", "pattern": "hybrid-reference", "dependsOn": ["extract"], "outputs": ["profiles_table"] } + ], + "orchestration": { "parallelism": 2 } + } + } + }, + "valid": true + }, + { + "description": "multi-stage build's stage carries a pattern value that is valid at the build level but outside the stage's narrower two-member enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "multi-stage", + "engine": "spark", + "properties": { "stages": [{ "name": "ingest", "pattern": "acquisition" }] } + } + }, + "valid": false, + "errors": [{ "pointer": "/build/properties/stages/0/pattern", "keyword": "enum" }] + }, + { + "description": "multi-stage build's orchestration.parallelism is set below its minimum of one", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "multi-stage", + "engine": "spark", + "properties": { + "stages": [{ "name": "only_stage" }], + "orchestration": { "parallelism": 0 } + } + } + }, + "valid": false, + "errors": [{ "pointer": "/build/properties/orchestration/parallelism", "keyword": "minimum" }] + }, + { + "description": "multi-stage build's properties carry a top-level member outside multiStagePattern's closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "multi-stage", + "engine": "spark", + "properties": { "stages": [{ "name": "only_stage" }], "priority": 5 } + } + }, + "valid": false, + "errors": [{ "pointer": "/build/properties", "keyword": "additionalProperties", "detail": "priority" }] + }, + { + "description": "acquisition build ingests from a minimal external source with no transformation", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "acquisition", + "engine": "dlt", + "properties": { "source": { "kind": "postgres", "mode": "full_refresh" } } + } + }, + "valid": true + }, + { + "description": "acquisition build's properties omit the required source", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { "pattern": "acquisition", "engine": "dlt", "properties": {} } + }, + "valid": false, + "errors": [{ "pointer": "/build/properties", "keyword": "required", "detail": "source" }] + }, + { + "description": "acquisition build's properties carry a hybrid-reference field (model) that its own closed set does not admit", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "acquisition", + "engine": "dlt", + "properties": { + "source": { "kind": "postgres", "mode": "full_refresh" }, + "model": "should_not_be_here" + } + } + }, + "valid": false, + "errors": [{ "pointer": "/build/properties", "keyword": "additionalProperties", "detail": "model" }] + }, + { + "description": "acquisition build declares a capability value outside the build.capabilities vocabulary", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "build": { + "pattern": "acquisition", + "engine": "dlt", + "properties": { "source": { "kind": "postgres", "mode": "full_refresh" } }, + "capabilities": ["full_refresh", "at_random_once"] + } + }, + "valid": false, + "errors": [{ "pointer": "/build/capabilities/1", "keyword": "enum" }] + }, + { + "description": "builds array declares two independent configurations for a multi-modal product, batch and streaming", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "builds": [ + { "id": "batch-processing", "pattern": "hybrid-reference", "engine": "dbt", "properties": { "model": "stg_customers" } }, + { "id": "streaming-ingest", "pattern": "acquisition", "engine": "kafka-connect", "properties": { "source": { "kind": "kafka", "mode": "streaming" } } } + ] + }, + "valid": true + }, + { + "description": "second entry of the builds array fails its own pattern-specific required member", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "builds": [ + { "id": "batch-processing", "pattern": "hybrid-reference", "engine": "dbt", "properties": { "model": "stg_customers" } }, + { "id": "bad-build", "pattern": "embedded-logic", "engine": "sql", "properties": {} } + ] + }, + "valid": false, + "errors": [{ "pointer": "/builds/1/properties", "keyword": "required", "detail": "sql" }] + }, + { + "description": "consumeRef declares every optional member: version constraint, full qos expectations, required policies, purpose, tags, and labels", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "consumes": [ + { + "productId": "silver.hr.people_v2", + "exposeId": "people_table", + "versionConstraint": "^1.2", + "qosExpectations": { + "freshnessMax": "PT15M", + "maxStaleness": "P1D", + "minCompleteness": 0.95, + "tags": ["critical"], + "labels": { "tier": "gold" } + }, + "requiredPolicies": ["pii-masking"], + "purpose": "customer 360 enrichment", + "tags": ["upstream-hr"], + "labels": { "domain": "hr" } + } + ] + }, + "valid": true + }, + { + "description": "consumeRef omits the required productId", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "consumes": [{ "exposeId": "people_table" }] + }, + "valid": false, + "errors": [{ "pointer": "/consumes/0", "keyword": "required", "detail": "productId" }] + }, + { + "description": "consumeRef omits the required exposeId", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "consumes": [{ "productId": "silver.hr.people_v2" }] + }, + "valid": false, + "errors": [{ "pointer": "/consumes/0", "keyword": "required", "detail": "exposeId" }] + }, + { + "description": "consumeRef uses the legacy 'provider' member name the schema comment says was renamed away", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "consumes": [{ "productId": "silver.hr.people_v2", "exposeId": "people_table", "provider": "legacy-alias" }] + }, + "valid": false, + "errors": [{ "pointer": "/consumes/0", "keyword": "additionalProperties", "detail": "provider" }] + }, + { + "description": "consumeRef's qosExpectations.minCompleteness is set above its maximum of one", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "consumes": [{ "productId": "x", "exposeId": "y", "qosExpectations": { "minCompleteness": 1.5 } }] + }, + "valid": false, + "errors": [{ "pointer": "/consumes/0/qosExpectations/minCompleteness", "keyword": "maximum" }] + }, + { + "description": "consumeRef's qosExpectations carries a member outside its own closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "consumes": [{ "productId": "x", "exposeId": "y", "qosExpectations": { "staleness_minutes": 30 } }] + }, + "valid": false, + "errors": [{ "pointer": "/consumes/0/qosExpectations", "keyword": "additionalProperties", "detail": "staleness_minutes" }] + }, + { + "description": "environments overrides declare a dev environment with metadata, an expose override, tags, and labels", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "environments": { + "dev": { + "metadata": { + "owner": { "team": "customer-analytics", "email": "data@acme.com" }, + "tags": ["dev"], + "labels": { "cost-center": "eng" } + }, + "exposes": [{ "exposeId": "profiles_table", "tags": ["sandbox"] }], + "tags": ["low-cost"], + "labels": { "tier": "dev" } + } + } + }, + "valid": true + }, + { + "description": "environments.dev carries a member outside environmentConfig's closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "environments": { "dev": { "region": "us-east1" } } + }, + "valid": false, + "errors": [{ "pointer": "/environments/dev", "keyword": "additionalProperties", "detail": "region" }] + }, + { + "description": "environments.dev's expose override names an exposeId that violates the identifier grammar", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "environments": { "dev": { "exposes": [{ "exposeId": "-bad" }] } } + }, + "valid": false, + "errors": [{ "pointer": "/environments/dev/exposes/0/exposeId", "keyword": "pattern" }] + } + ] +} diff --git a/tests/0.7.7/consumers.json b/tests/0.7.7/consumers.json new file mode 100644 index 0000000..88bf048 --- /dev/null +++ b/tests/0.7.7/consumers.json @@ -0,0 +1,462 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "consumers", + "description": "Declared downstream consumers (0.7.6): the consumers array, each consumerRef's required name and type, the type and maturity enums, the closed owner object, the identifier rules on name and exposeIds, and consumerRef's closed property set.", + "cases": [ + { + "description": "a consumer carries every member", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumers": [ + { + "name": "ltv_dashboard", + "label": "Customer LTV dashboard", + "type": "dashboard", + "owner": { + "team": "bi", + "email": "bi@acme.example" + }, + "url": "https://bi.acme.example/d/ltv", + "maturity": "high", + "exposeIds": [ + "profiles_table" + ] + } + ] + }, + "valid": true + }, + { + "description": "a consumer carrying only its required name and type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumers": [ + { + "name": "churn_model", + "type": "ml" + } + ] + }, + "valid": true + }, + { + "description": "a consumer omits its required type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumers": [ + { + "name": "churn_model" + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/consumers/0", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "a consumer omits its required name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumers": [ + { + "type": "ml" + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/consumers/0", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "consumer.type names a kind outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumers": [ + { + "name": "ltv_report", + "type": "report" + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/consumers/0/type", + "keyword": "enum" + } + ] + }, + { + "description": "consumer.maturity names a level outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumers": [ + { + "name": "churn_model", + "type": "ml", + "maturity": "experimental" + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/consumers/0/maturity", + "keyword": "enum" + } + ] + }, + { + "description": "a consumer carries a property outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumers": [ + { + "name": "churn_model", + "type": "ml", + "sla": "P1D" + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/consumers/0", + "keyword": "additionalProperties", + "detail": "sla" + } + ] + }, + { + "description": "consumer.owner carries a property outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumers": [ + { + "name": "churn_model", + "type": "ml", + "owner": { + "team": "ds", + "slack": "#ds" + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/consumers/0/owner", + "keyword": "additionalProperties", + "detail": "slack" + } + ] + }, + { + "description": "consumers is a single object instead of an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumers": { + "name": "churn_model", + "type": "ml" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/consumers", + "keyword": "type" + } + ] + } + ] +} diff --git a/tests/0.7.7/consumes-pinning.json b/tests/0.7.7/consumes-pinning.json new file mode 100644 index 0000000..2302d88 --- /dev/null +++ b/tests/0.7.7/consumes-pinning.json @@ -0,0 +1,306 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "consumes-pinning", + "description": "Cross-mesh upstream pinning on consumes[] (0.7.6): upstreamWorkspace requires upstreamDigest (dependentRequired, the first JSON Schema 2020-12-only assertion keyword a FLUID schema uses -- a Draft 7 validator ignores it and accepts the invalid case below), the sha256:<64 lowercase hex> digest pattern, and the non-empty workspace id.", + "cases": [ + { + "description": "a cross-mesh upstream names its workspace and the digest it was composed against", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumes": [ + { + "productId": "silver.hr.people_v2", + "exposeId": "people_table", + "upstreamWorkspace": "finance-mesh", + "upstreamDigest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + ] + }, + "valid": true + }, + { + "description": "a digest without a workspace is admitted: the dependency runs one way only", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumes": [ + { + "productId": "silver.hr.people_v2", + "exposeId": "people_table", + "upstreamDigest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + ] + }, + "valid": true + }, + { + "description": "upstreamWorkspace is set without the upstreamDigest it requires", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumes": [ + { + "productId": "silver.hr.people_v2", + "exposeId": "people_table", + "upstreamWorkspace": "finance-mesh" + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/consumes/0", + "keyword": "dependentRequired" + } + ] + }, + { + "description": "upstreamDigest uses uppercase hex", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumes": [ + { + "productId": "silver.hr.people_v2", + "exposeId": "people_table", + "upstreamWorkspace": "finance-mesh", + "upstreamDigest": "sha256:0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF" + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/consumes/0/upstreamDigest", + "keyword": "pattern" + } + ] + }, + { + "description": "upstreamDigest names an algorithm other than sha256", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumes": [ + { + "productId": "silver.hr.people_v2", + "exposeId": "people_table", + "upstreamWorkspace": "finance-mesh", + "upstreamDigest": "sha512:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/consumes/0/upstreamDigest", + "keyword": "pattern" + } + ] + }, + { + "description": "upstreamWorkspace is an empty string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "consumes": [ + { + "productId": "silver.hr.people_v2", + "exposeId": "people_table", + "upstreamWorkspace": "", + "upstreamDigest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/consumes/0/upstreamWorkspace", + "keyword": "minLength" + } + ] + } + ] +} diff --git a/tests/0.7.7/contract-schema.json b/tests/0.7.7/contract-schema.json new file mode 100644 index 0000000..cf5a322 --- /dev/null +++ b/tests/0.7.7/contract-schema.json @@ -0,0 +1,1111 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "contract-schema", + "description": "The tabular contract surface of an expose: schemaSignature and schemaPolicy on exposeContract, the schema/openapiRef anyOf, the guarantees and simplified quality sub-objects, the column required-field, minLength and closed property-set rules, the deliberately generous column.type anyOf (canonical enum plus a parameterised cross-dialect pattern), sensitivity classification against sensitivityLevel, and the dqSpec/dqRule required members, enums and monitoring block.", + "cases": [ + { + "description": "signed schema, evolve_safe policy, parameterized decimal column, dq rule with optional operator/threshold/window", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "lifetime_value", + "type": "decimal(18,4)" + } + ], + "schemaSignature": "sha256:a1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4", + "schemaPolicy": "evolve_safe", + "dq": { + "rules": [ + { + "id": "freshness_customer_id", + "type": "freshness", + "severity": "warn", + "operator": ">=", + "threshold": 0.95, + "window": "P1D" + } + ], + "monitoring": { + "enabled": true, + "window": "PT15M", + "owner": "data-eng" + } + } + } + } + ] + }, + "valid": true + }, + { + "description": "expose contract satisfied via openapiRef instead of a tabular schema", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "api", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "openapiRef": "https://api.example.com/openapi.json" + } + } + ] + }, + "valid": true + }, + { + "description": "column classified with a pii sensitivity level", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "ssn", + "type": "string", + "sensitivity": "pii" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "expose contract declares neither schema nor openapiRef", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": {} + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract", + "keyword": "anyOf" + } + ] + }, + { + "description": "schemaSignature missing the sha256 hex digest", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ], + "schemaSignature": "sha256:not-a-real-hash" + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schemaSignature", + "keyword": "pattern" + } + ] + }, + { + "description": "schemaPolicy set to an unsupported evolution value", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ], + "schemaPolicy": "evolve_fast" + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schemaPolicy", + "keyword": "enum" + } + ] + }, + { + "description": "expose contract carries an undeclared columnDefs property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ], + "columnDefs": [] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract", + "keyword": "additionalProperties", + "detail": "columnDefs" + } + ] + }, + { + "description": "column type is not a recognized SQL or FLUID type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "MYSTERY_TYPE" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/type", + "keyword": "anyOf" + } + ] + }, + { + "description": "column missing the required name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "column missing the required type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "column carries an undeclared primaryKey property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string", + "primaryKey": true + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0", + "keyword": "additionalProperties", + "detail": "primaryKey" + } + ] + }, + { + "description": "column sensitivity outside the accepted classification enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "ssn", + "type": "string", + "sensitivity": "topsecret" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/sensitivity", + "keyword": "enum" + } + ] + }, + { + "description": "data quality rule missing the required id", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ], + "dq": { + "rules": [ + { + "type": "completeness", + "severity": "error" + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/dq/rules/0", + "keyword": "required", + "detail": "id" + } + ] + }, + { + "description": "data quality rule missing the required type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ], + "dq": { + "rules": [ + { + "id": "completeness_customer_id", + "severity": "error" + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/dq/rules/0", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "data quality rule missing the required severity", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ], + "dq": { + "rules": [ + { + "id": "completeness_customer_id", + "type": "completeness" + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/dq/rules/0", + "keyword": "required", + "detail": "severity" + } + ] + }, + { + "description": "data quality rule type outside the accepted enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ], + "dq": { + "rules": [ + { + "id": "completeness_customer_id", + "type": "outlier_detection", + "severity": "error" + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/dq/rules/0/type", + "keyword": "enum" + } + ] + }, + { + "description": "data quality rule severity outside the accepted enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ], + "dq": { + "rules": [ + { + "id": "completeness_customer_id", + "type": "completeness", + "severity": "fatal" + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/dq/rules/0/severity", + "keyword": "enum" + } + ] + }, + { + "description": "data quality specification carries an undeclared thresholds property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ], + "dq": { + "rules": [ + { + "id": "completeness_customer_id", + "type": "completeness", + "severity": "error" + } + ], + "thresholds": {} + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/dq", + "keyword": "additionalProperties", + "detail": "thresholds" + } + ] + }, + { + "description": "data quality rule operator outside the accepted comparison enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ], + "dq": { + "rules": [ + { + "id": "completeness_customer_id", + "type": "completeness", + "severity": "error", + "operator": "eq" + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/dq/rules/0/operator", + "keyword": "enum" + } + ] + }, + { + "description": "column name is an empty string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/0/name", + "keyword": "minLength" + } + ] + }, + { + "description": "dq monitoring block carries an undeclared cronSchedule property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ], + "dq": { + "rules": [ + { + "id": "completeness_customer_id", + "type": "completeness", + "severity": "error" + } + ], + "monitoring": { + "enabled": true, + "cronSchedule": "0 * * * *" + } + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/dq/monitoring", + "keyword": "additionalProperties", + "detail": "cronSchedule" + } + ] + }, + { + "description": "expose contract's simplified quality rule missing the required severity", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ], + "quality": [ + { + "rule": "non_negative_ltv", + "expression": "lifetime_value >= 0" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/quality/0", + "keyword": "required", + "detail": "severity" + } + ] + }, + { + "description": "expose contract guarantees compatibility value outside the accepted enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ], + "guarantees": { + "compatibility": "loose" + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/guarantees/compatibility", + "keyword": "enum" + } + ] + } + ] +} diff --git a/tests/0.7.7/envelope.json b/tests/0.7.7/envelope.json new file mode 100644 index 0000000..83e6644 --- /dev/null +++ b/tests/0.7.7/envelope.json @@ -0,0 +1,638 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "envelope", + "description": "The shared document envelope: the six required top-level members, the closed top-level property set, and the identifier and version grammars every FLUID document is bound by. A conformant implementation reproduces every verdict below.", + "cases": [ + { + "description": "minimal document carrying exactly the required members", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "fluidVersion absent", + "contract": { + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "", + "keyword": "required", + "detail": "fluidVersion" + } + ] + }, + { + "description": "metadata absent", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "", + "keyword": "required", + "detail": "metadata" + } + ] + }, + { + "description": "metadata present but owner absent", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "layer": "gold" + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/metadata", + "keyword": "required", + "detail": "owner" + } + ] + }, + { + "description": "exposes is an empty array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes", + "keyword": "minItems" + } + ] + }, + { + "description": "unknown top-level member is refused because the envelope is closed", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "owner": "customer-analytics", + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "", + "keyword": "additionalProperties", + "detail": "owner" + } + ] + }, + { + "description": "kind outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataMesh", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/kind", + "keyword": "enum" + } + ] + }, + { + "description": "MLPipeline is an accepted kind", + "contract": { + "fluidVersion": "0.7.7", + "kind": "MLPipeline", + "id": "churn_model", + "name": "Churn Model", + "metadata": { + "owner": { + "team": "ml-platform" + } + }, + "exposes": [ + { + "exposeId": "churn_scores", + "kind": "model", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "dataset": "ml", + "table": "churn_scores" + } + }, + "contract": { + "schema": [ + { + "name": "score", + "type": "double" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "id starting with a hyphen violates the identifier grammar", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "-customer-profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/id", + "keyword": "pattern" + } + ] + }, + { + "description": "single-character id is legal under the identifier grammar's second branch", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "x", + "name": "Single character id", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "a fluidVersion the 0.7.7 schema does not admit", + "contract": { + "fluidVersion": "0.7.2", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/fluidVersion", + "keyword": "enum" + } + ] + }, + { + "description": "kind absent", + "contract": { + "fluidVersion": "0.7.7", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "", + "keyword": "required", + "detail": "kind" + } + ] + }, + { + "description": "id absent", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "", + "keyword": "required", + "detail": "id" + } + ] + }, + { + "description": "name absent", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "exposes absent", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "", + "keyword": "required", + "detail": "exposes" + } + ] + }, + { + "description": "name is the empty string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/name", + "keyword": "minLength" + } + ] + } + ] +} diff --git a/tests/0.7.7/expose-mcp.json b/tests/0.7.7/expose-mcp.json new file mode 100644 index 0000000..9911cac --- /dev/null +++ b/tests/0.7.7/expose-mcp.json @@ -0,0 +1,339 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "expose-mcp", + "description": "The exposeMcp gateway block (exposes[].mcp: sampling.maxRows, classification.dataClass) and the completeness/latency members of defaultSLIs (exposes[].observability.defaultSLIs): each object's closed property set and the dataClass/severity enums.", + "cases": [ + { + "description": "expose combines an MCP gateway block with completeness and latency SLI severities", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "mcp": { + "sampling": { "maxRows": 500 }, + "classification": { "dataClass": "confidential" } + }, + "observability": { + "defaultSLIs": { + "enabled": true, + "completeness": { "severity": "info" }, + "latency": { "severity": "critical" } + } + } + } + ] + }, + "valid": true + }, + { + "description": "MCP classification carries the public dataClass value", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "mcp": { + "classification": { "dataClass": "public" } + } + } + ] + }, + "valid": true + }, + { + "description": "defaultSLIs completeness carries an unrecognized property alongside severity", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { + "completeness": { "severity": "info", "sampleRate": 0.5 } + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/defaultSLIs/completeness", "keyword": "additionalProperties", "detail": "sampleRate" } + ] + }, + { + "description": "defaultSLIs completeness severity outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { + "completeness": { "severity": "catastrophic" } + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/defaultSLIs/completeness/severity", "keyword": "enum" } + ] + }, + { + "description": "defaultSLIs latency carries an unrecognized property alongside severity", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { + "latency": { "severity": "critical", "sampleRate": 0.5 } + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/defaultSLIs/latency", "keyword": "additionalProperties", "detail": "sampleRate" } + ] + }, + { + "description": "defaultSLIs latency severity outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { + "latency": { "severity": "catastrophic" } + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/defaultSLIs/latency/severity", "keyword": "enum" } + ] + }, + { + "description": "mcp block carries an unrecognized property alongside sampling and classification", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "mcp": { + "sampling": { "maxRows": 500 }, + "classification": { "dataClass": "confidential" }, + "auditLog": true + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/mcp", "keyword": "additionalProperties", "detail": "auditLog" } + ] + }, + { + "description": "mcp sampling carries an unrecognized property alongside maxRows", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "mcp": { + "sampling": { "maxRows": 500, "maxRowsPerSecond": 50 } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/mcp/sampling", "keyword": "additionalProperties", "detail": "maxRowsPerSecond" } + ] + }, + { + "description": "mcp sampling maxRows at zero", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "mcp": { + "sampling": { "maxRows": 0 } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/mcp/sampling/maxRows", "keyword": "minimum" } + ] + }, + { + "description": "mcp classification carries an unrecognized property alongside dataClass", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "mcp": { + "classification": { "dataClass": "confidential", "reviewedBy": "security-team" } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/mcp/classification", "keyword": "additionalProperties", "detail": "reviewedBy" } + ] + }, + { + "description": "mcp classification dataClass outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "mcp": { + "classification": { "dataClass": "top-secret" } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/mcp/classification/dataClass", "keyword": "enum" } + ] + } + ] +} diff --git a/tests/0.7.7/expose-policy.json b/tests/0.7.7/expose-policy.json new file mode 100644 index 0000000..5d53612 --- /dev/null +++ b/tests/0.7.7/expose-policy.json @@ -0,0 +1,860 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "expose-policy", + "description": "The exposePolicy block reached via exposes[].policy: its own closed property set, the authn enum, the authz sub-object (closed property set, columnRestrictions items' closed property set and access enum, and from 0.7.6 access: mask with its mask rule and rowFilters items' required principal/where, name pattern and closed property set), the privacy sub-object (closed property set, masking items' required column/strategy, closed property set and strategy enum, rowLevelPolicy's required expression and closed property set), and the classification enum.", + "cases": [ + { + "description": "policy block populated with authn, authz, privacy, and classification", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authn": "oidc", + "authz": { + "readers": ["group:analytics"], + "writers": ["group:platform"], + "columnRestrictions": [ + { + "principal": "group:analytics", + "columns": ["ssn"], + "access": "deny", + "tags": ["pii-guard"], + "labels": { "reviewed-by": "security-team" } + } + ], + "tags": ["policy-authz"], + "labels": { "owner-team": "security" } + }, + "privacy": { + "masking": [ + { + "column": "ssn", + "strategy": "tokenize", + "params": { "format": "fpe" }, + "tags": ["masking-rule"], + "labels": { "applies-to": "ssn" } + } + ], + "rowLevelPolicy": { + "expression": "region = current_region()", + "tags": ["rls-rule"], + "labels": { "scope": "region" } + }, + "tags": ["privacy-policy"], + "labels": { "data-class": "sensitive" } + }, + "classification": "Confidential", + "tags": ["policy-top"], + "labels": { "policy-owner": "governance-team" } + } + } + ] + }, + "valid": true + }, + { + "description": "policy carries an unrecognized top-level property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authn": "oidc", + "classification": "Confidential", + "encryption": { "atRest": "kms" } + } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/policy", "keyword": "additionalProperties", "detail": "encryption" }] + }, + { + "description": "authn value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authn": "saml" + } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/policy/authn", "keyword": "enum" }] + }, + { + "description": "authz carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { + "readers": ["group:analytics"], + "writers": ["group:platform"], + "approvers": ["group:governance"] + } + } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/policy/authz", "keyword": "additionalProperties", "detail": "approvers" }] + }, + { + "description": "columnRestrictions item carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { + "columnRestrictions": [ + { + "principal": "group:analytics", + "columns": ["ssn"], + "access": "deny", + "reason": "PII exposure" + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/authz/columnRestrictions/0", "keyword": "additionalProperties", "detail": "reason" } + ] + }, + { + "description": "columnRestrictions access value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { + "columnRestrictions": [ + { + "principal": "group:analytics", + "columns": ["ssn"], + "access": "redact" + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/policy/authz/columnRestrictions/0/access", "keyword": "enum" }] + }, + { + "description": "authz masks a column for one principal and filters rows for another (0.7.6)", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { + "readers": ["group:support", "group:analysts-eu"], + "columnRestrictions": [ + { + "principal": "group:support", + "columns": ["phone"], + "access": "mask", + "mask": "last_four" + } + ], + "rowFilters": [ + { "principal": "group:analysts-eu", "name": "eu_only", "where": "region = 'EU'" } + ] + } + } + } + ] + }, + "valid": true + }, + { + "description": "columnRestrictions access mask without the mask rule", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { + "columnRestrictions": [ + { "principal": "group:support", "columns": ["phone"], "access": "mask" } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/policy/authz/columnRestrictions/0", + "keyword": "required", + "detail": "mask" + } + ] + }, + { + "description": "columnRestrictions mask rule on a deny restriction", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { + "columnRestrictions": [ + { + "principal": "group:support", + "columns": ["phone"], + "access": "deny", + "mask": "last_four" + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/authz/columnRestrictions/0/access", "keyword": "const" } + ] + }, + { + "description": "columnRestrictions mask rule without an access", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { + "columnRestrictions": [ + { "principal": "group:support", "columns": ["phone"], "mask": "last_four" } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/policy/authz/columnRestrictions/0", + "keyword": "required", + "detail": "access" + } + ] + }, + { + "description": "columnRestrictions mask rule outside the recognized set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { + "columnRestrictions": [ + { + "principal": "group:support", + "columns": ["phone"], + "access": "mask", + "mask": "redact" + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/authz/columnRestrictions/0/mask", "keyword": "enum" } + ] + }, + { + "description": "rowFilters item omits the required where", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { "rowFilters": [{ "principal": "group:analysts-eu", "name": "eu_only" }] } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/policy/authz/rowFilters/0", + "keyword": "required", + "detail": "where" + } + ] + }, + { + "description": "rowFilters item omits the required principal", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { "rowFilters": [{ "name": "eu_only", "where": "region = 'EU'" }] } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/policy/authz/rowFilters/0", + "keyword": "required", + "detail": "principal" + } + ] + }, + { + "description": "rowFilters item's where is empty", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { + "rowFilters": [ + { "principal": "group:analysts-eu", "name": "eu_only", "where": "" } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/authz/rowFilters/0/where", "keyword": "minLength" } + ] + }, + { + "description": "rowFilters item's name is not a lowercase identifier", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { + "rowFilters": [ + { "principal": "group:analysts-eu", "name": "EU-only", "where": "region = 'EU'" } + ] + } + } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/policy/authz/rowFilters/0/name", "keyword": "pattern" }] + }, + { + "description": "rowFilters item carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { + "rowFilters": [ + { + "principal": "group:analysts-eu", + "name": "eu_only", + "where": "region = 'EU'", + "priority": 1 + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/policy/authz/rowFilters/0", + "keyword": "additionalProperties", + "detail": "priority" + } + ] + }, + { + "description": "privacy carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "privacy": { + "masking": [{ "column": "ssn", "strategy": "tokenize" }], + "retentionDays": 30 + } + } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/policy/privacy", "keyword": "additionalProperties", "detail": "retentionDays" }] + }, + { + "description": "masking rule omits column", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "privacy": { + "masking": [ + { + "strategy": "tokenize", + "params": { "format": "fpe" }, + "tags": ["masking-rule"], + "labels": { "applies-to": "ssn" } + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/privacy/masking/0", "keyword": "required", "detail": "column" } + ] + }, + { + "description": "masking rule omits strategy", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "privacy": { + "masking": [ + { + "column": "ssn", + "params": { "format": "fpe" }, + "tags": ["masking-rule"], + "labels": { "applies-to": "ssn" } + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/privacy/masking/0", "keyword": "required", "detail": "strategy" } + ] + }, + { + "description": "masking rule carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "privacy": { + "masking": [ + { + "column": "ssn", + "strategy": "tokenize", + "algorithm": "aes256" + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/privacy/masking/0", "keyword": "additionalProperties", "detail": "algorithm" } + ] + }, + { + "description": "masking strategy value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "privacy": { + "masking": [{ "column": "ssn", "strategy": "obfuscate" }] + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/privacy/masking/0/strategy", "keyword": "enum" } + ] + }, + { + "description": "rowLevelPolicy omits expression", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "privacy": { + "rowLevelPolicy": { + "tags": ["rls-rule"], + "labels": { "scope": "region" } + } + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/privacy/rowLevelPolicy", "keyword": "required", "detail": "expression" } + ] + }, + { + "description": "rowLevelPolicy carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "privacy": { + "rowLevelPolicy": { + "expression": "region = current_region()", + "priority": 1 + } + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/privacy/rowLevelPolicy", "keyword": "additionalProperties", "detail": "priority" } + ] + }, + { + "description": "classification value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "classification": "Secret" + } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/policy/classification", "keyword": "enum" }] + } + ] +} diff --git a/tests/0.7.7/expose.json b/tests/0.7.7/expose.json new file mode 100644 index 0000000..c7dbd2d --- /dev/null +++ b/tests/0.7.7/expose.json @@ -0,0 +1,537 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "expose", + "description": "The expose object: its four required members, its closed property set, the full breadth of the kind enum, the mcp and iceberg member shapes, exposeId against the identifier grammar, and version against semver.", + "cases": [ + { + "description": "version sits at the lowest permissible semver value", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "raw_events", + "version": "0.0.0", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "raw", "table": "events" } + }, + "contract": { "schema": [{ "name": "event_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "optional title, description, version, tags and labels populated on a view", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "customer_view", + "title": "Customer 360 View", + "description": "Denormalized customer view joining profile and orders.", + "version": "1.4.0", + "kind": "view", + "tags": ["pii", "customer-facing"], + "labels": { "team": "customer-analytics", "retention": "7-years" }, + "binding": { + "platform": "snowflake", + "format": "snowflake_view", + "location": { "database": "ANALYTICS", "schema": "PUBLIC", "table": "customer_view" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "kind is api exposed over http with an openapiRef contract", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "customers_api", + "kind": "api", + "binding": { + "platform": "aws", + "format": "http_api", + "location": { "baseUrl": "https://api.acme.com/v1/customers", "gateway": "customers-gw" } + }, + "contract": { "openapiRef": "https://api.acme.com/openapi.json" } + } + ] + }, + "valid": true + }, + { + "description": "kind is stream backed by a kafka topic binding", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "customer_events", + "kind": "stream", + "binding": { + "platform": "kafka", + "format": "kafka_topic", + "location": { "topic": "customer-events" } + }, + "contract": { "schema": [{ "name": "event_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "kind is model with an other-platform binding and a float output column", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "churn_model", + "kind": "model", + "binding": { + "platform": "other", + "format": "other", + "location": { "path": "s3://ml-models/churn/v2" } + }, + "contract": { "schema": [{ "name": "churn_probability", "type": "float" }] } + } + ] + }, + "valid": true + }, + { + "description": "exposeId is a single character", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "x", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "second expose entry carries a dotted-hyphenated exposeId and a full prerelease-plus-build version", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + }, + { + "exposeId": "sales.orders-v2", + "version": "2.1.0-beta.1+build.42", + "kind": "view", + "binding": { + "platform": "databricks", + "format": "delta_table", + "location": { "database": "analytics", "table": "orders_v2" } + }, + "contract": { "schema": [{ "name": "order_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "expose carries the mcp gateway member alongside crawler configuration", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "crawler": { "name": "profiles-crawler", "role": "arn:aws:iam::123456789012:role/glue-crawler" }, + "mcp": { + "sampling": { "maxRows": 250 }, + "classification": { "dataClass": "confidential" } + } + } + ] + }, + "valid": true + }, + { + "description": "exposeId absent from the first expose entry", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0", "keyword": "required", "detail": "exposeId" }] + }, + { + "description": "kind absent from the expose entry", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0", "keyword": "required", "detail": "kind" }] + }, + { + "description": "binding absent from the expose entry", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0", "keyword": "required", "detail": "binding" }] + }, + { + "description": "contract absent from the expose entry", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0", "keyword": "required", "detail": "contract" }] + }, + { + "description": "unknown member on the expose entry is refused because its property set is closed", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "owner": "customer-analytics", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0", "keyword": "additionalProperties", "detail": "owner" }] + }, + { + "description": "kind value is outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "queue", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/kind", "keyword": "enum" }] + }, + { + "description": "exposeId contains a space", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/exposeId", "keyword": "pattern" }] + }, + { + "description": "exposeId is an empty string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/exposeId", "keyword": "pattern" }] + }, + { + "description": "version omits the patch component", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "version": "1.0", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/version", "keyword": "pattern" }] + }, + { + "description": "the seven kind values not exercised elsewhere, one per expose entry, with an iceberg block on the last", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "raw_files", + "kind": "file", + "binding": { + "platform": "aws", + "format": "s3_file", + "location": { "bucket": "acme-raw", "path": "landing/events/" } + }, + "contract": { "schema": [{ "name": "path", "type": "string" }] } + }, + { + "exposeId": "customer_topic", + "kind": "topic", + "binding": { + "platform": "gcp", + "format": "pubsub_topic", + "location": { "topic": "customer-updates" } + }, + "contract": { "schema": [{ "name": "event_id", "type": "string" }] } + }, + { + "exposeId": "customer_features", + "kind": "feature_store", + "binding": { + "platform": "databricks", + "format": "delta_table", + "location": { "database": "features", "table": "customer_features" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + }, + { + "exposeId": "customer_embeddings", + "kind": "vector", + "binding": { + "platform": "other", + "format": "other", + "location": { "path": "s3://ml-vectors/customers" } + }, + "contract": { "schema": [{ "name": "embedding", "type": "array" }] } + }, + { + "exposeId": "customer_graph", + "kind": "graph", + "binding": { + "platform": "other", + "format": "other", + "location": { "path": "s3://graph-store/customers" } + }, + "contract": { "schema": [{ "name": "node_id", "type": "string" }] } + }, + { + "exposeId": "customer_metrics_ts", + "kind": "time_series", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "metrics", "table": "customer_ts" } + }, + "contract": { "schema": [{ "name": "ts", "type": "timestamp" }] } + }, + { + "exposeId": "legacy_export", + "kind": "other", + "binding": { + "platform": "azure", + "format": "other", + "location": { "account": "acmestorage", "path": "exports/legacy.csv" } + }, + "contract": { "schema": [{ "name": "raw", "type": "string" }] }, + "iceberg": { + "writeFormat": "parquet", + "snapshotRetention": { "maxSnapshotAgeMs": 604800000, "minSnapshotsToKeep": 5 }, + "compaction": { "enabled": true, "targetFileSizeMb": 512 } + } + } + ] + }, + "valid": true + }, + { + "description": "iceberg.writeFormat is outside its parquet/orc/avro enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "iceberg": { "writeFormat": "iceberg-native" } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/iceberg/writeFormat", "keyword": "enum" }] + } + ] +} diff --git a/tests/0.7.7/governance-access.json b/tests/0.7.7/governance-access.json new file mode 100644 index 0000000..079b554 --- /dev/null +++ b/tests/0.7.7/governance-access.json @@ -0,0 +1,673 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "governance-access", + "description": "The optional accessPolicy, governance, and retention blocks: the accessPolicy grants array (required principal, permissions enum, uniqueItems), the governance lakeFormation block (ARN-pattern admins, tag-definition value arrays), and the retention block's ISO-8601 duration fields -- plus each block's closed property set.", + "cases": [ + { + "description": "accessPolicy grant carrying only the required principal", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "accessPolicy": { + "grants": [ + { "principal": "user:analyst@company.com" } + ] + } + }, + "valid": true + }, + { + "description": "accessPolicy grant exercising the full permissions enum range plus resources and conditions", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "accessPolicy": { + "grants": [ + { + "principal": "group:data-analytics@company.com", + "permissions": ["read", "select", "query", "write", "insert", "update", "delete", "create", "admin", "manage"], + "resources": ["$.exposes[?(@.exposeId=='profiles_table')]"], + "conditions": { "ipRange": "10.0.0.0/8" } + } + ] + } + }, + "valid": true + }, + { + "description": "accessPolicy grant missing its principal", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "accessPolicy": { + "grants": [ + { "permissions": ["read"] } + ] + } + }, + "valid": false, + "errors": [{ "pointer": "/accessPolicy/grants/0", "keyword": "required", "detail": "principal" }] + }, + { + "description": "accessPolicy grant permission value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "accessPolicy": { + "grants": [ + { "principal": "user:analyst@company.com", "permissions": ["superuser"] } + ] + } + }, + "valid": false, + "errors": [{ "pointer": "/accessPolicy/grants/0/permissions/0", "keyword": "enum" }] + }, + { + "description": "accessPolicy grant permissions array with a repeated value", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "accessPolicy": { + "grants": [ + { "principal": "user:analyst@company.com", "permissions": ["read", "read"] } + ] + } + }, + "valid": false, + "errors": [{ "pointer": "/accessPolicy/grants/0/permissions", "keyword": "uniqueItems" }] + }, + { + "description": "accessPolicy grant principal given as a number instead of a string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "accessPolicy": { + "grants": [ + { "principal": 12345 } + ] + } + }, + "valid": false, + "errors": [{ "pointer": "/accessPolicy/grants/0/principal", "keyword": "type" }] + }, + { + "description": "accessPolicy grant permissions given as a bare string instead of an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "accessPolicy": { + "grants": [ + { "principal": "user:analyst@company.com", "permissions": "read" } + ] + } + }, + "valid": false, + "errors": [{ "pointer": "/accessPolicy/grants/0/permissions", "keyword": "type" }] + }, + { + "description": "accessPolicy carrying an unrecognized member alongside grants", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "accessPolicy": { + "grants": [ + { "principal": "user:analyst@company.com" } + ], + "policies": [] + } + }, + "valid": false, + "errors": [{ "pointer": "/accessPolicy", "keyword": "additionalProperties", "detail": "policies" }] + }, + { + "description": "accessPolicy grant carrying an unrecognized member alongside principal", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "accessPolicy": { + "grants": [ + { "principal": "user:analyst@company.com", "role": "viewer" } + ] + } + }, + "valid": false, + "errors": [{ "pointer": "/accessPolicy/grants/0", "keyword": "additionalProperties", "detail": "role" }] + }, + { + "description": "governance lakeFormation block with admins and tag definitions", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "governance": { + "lakeFormation": { + "admins": ["arn:aws:iam::123456789012:role/DataLakeAdmin"], + "tagDefinitions": { "confidentiality": ["public", "internal", "restricted"] } + } + } + }, + "valid": true + }, + { + "description": "governance carrying an unrecognized member instead of lakeFormation", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "governance": { + "snowflake": {} + } + }, + "valid": false, + "errors": [{ "pointer": "/governance", "keyword": "additionalProperties", "detail": "snowflake" }] + }, + { + "description": "lakeFormation block carrying an unrecognized member alongside admins", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "governance": { + "lakeFormation": { + "admins": ["arn:aws:iam::123456789012:role/DataLakeAdmin"], + "auditors": [] + } + } + }, + "valid": false, + "errors": [{ "pointer": "/governance/lakeFormation", "keyword": "additionalProperties", "detail": "auditors" }] + }, + { + "description": "lakeFormation admin entry that is not an IAM ARN", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "governance": { + "lakeFormation": { + "admins": ["not-an-arn"] + } + } + }, + "valid": false, + "errors": [{ "pointer": "/governance/lakeFormation/admins/0", "keyword": "pattern" }] + }, + { + "description": "lakeFormation admins given as a bare string instead of an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "governance": { + "lakeFormation": { + "admins": "arn:aws:iam::123456789012:role/DataLakeAdmin" + } + } + }, + "valid": false, + "errors": [{ "pointer": "/governance/lakeFormation/admins", "keyword": "type" }] + }, + { + "description": "lakeFormation admins array with a repeated ARN", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "governance": { + "lakeFormation": { + "admins": [ + "arn:aws:iam::123456789012:role/DataLakeAdmin", + "arn:aws:iam::123456789012:role/DataLakeAdmin" + ] + } + } + }, + "valid": false, + "errors": [{ "pointer": "/governance/lakeFormation/admins", "keyword": "uniqueItems" }] + }, + { + "description": "lakeFormation tag definition with no allowed values", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "governance": { + "lakeFormation": { + "tagDefinitions": { "confidentiality": [] } + } + } + }, + "valid": false, + "errors": [{ "pointer": "/governance/lakeFormation/tagDefinitions/confidentiality", "keyword": "minItems" }] + }, + { + "description": "lakeFormation tag definition value given as a string instead of an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "governance": { + "lakeFormation": { + "tagDefinitions": { "confidentiality": "public" } + } + } + }, + "valid": false, + "errors": [{ "pointer": "/governance/lakeFormation/tagDefinitions/confidentiality", "keyword": "type" }] + }, + { + "description": "lakeFormation tag definition with a repeated allowed value", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "governance": { + "lakeFormation": { + "tagDefinitions": { "confidentiality": ["public", "public"] } + } + } + }, + "valid": false, + "errors": [{ "pointer": "/governance/lakeFormation/tagDefinitions/confidentiality", "keyword": "uniqueItems" }] + }, + { + "description": "retention block setting all four durations", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "retention": { + "runState": "P30D", + "runLogs": "P90D", + "lineage": "P365D", + "dlq": "P180D" + } + }, + "valid": true + }, + { + "description": "retention block setting only runLogs with an hours-and-minutes duration", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "retention": { + "runLogs": "PT12H30M" + } + }, + "valid": true + }, + { + "description": "retention duration given as a number instead of a string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "retention": { + "runState": 30 + } + }, + "valid": false, + "errors": [{ "pointer": "/retention/runState", "keyword": "type" }] + }, + { + "description": "retention duration missing the leading ISO-8601 designator", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "retention": { + "runState": "30D" + } + }, + "valid": false, + "errors": [{ "pointer": "/retention/runState", "keyword": "pattern" }] + }, + { + "description": "retention block carrying an unrecognized member", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "retention": { + "runState": "P30D", + "sweepInterval": "P1D" + } + }, + "valid": false, + "errors": [{ "pointer": "/retention", "keyword": "additionalProperties", "detail": "sweepInterval" }] + } + ] +} diff --git a/tests/0.7.7/lifecycle-expire-bucketpolicy.json b/tests/0.7.7/lifecycle-expire-bucketpolicy.json new file mode 100644 index 0000000..1c7adac --- /dev/null +++ b/tests/0.7.7/lifecycle-expire-bucketpolicy.json @@ -0,0 +1,256 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "lifecycle-expire-bucketpolicy", + "description": "Expose-level additions in 0.7.6: lifecycle.expire (a boolean on the expose lifecycle, which keeps the closed property set of 0.7.5's lifecycle plus that one member), and lakeFormation.bucketPolicy's enum.", + "cases": [ + { + "description": "an expose lifecycle declares a retention period and opts in to expiry", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "lifecycle": { + "state": "active", + "retention": "P90D", + "expire": true + } + } + ] + }, + "valid": true + }, + { + "description": "lifecycle.expire is a string instead of a boolean", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "lifecycle": { + "retention": "P90D", + "expire": "true" + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/lifecycle/expire", + "keyword": "type" + } + ] + }, + { + "description": "the expose lifecycle carries a property outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "lifecycle": { + "retention": "P90D", + "expireAfter": "P90D" + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/lifecycle", + "keyword": "additionalProperties", + "detail": "expireAfter" + } + ] + }, + { + "description": "lakeFormation.bucketPolicy restricts the S3 bucket policy to no statements", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "registerLocation": true, + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": [ + "SELECT" + ] + } + ], + "bucketPolicy": "none" + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "lakeFormation.bucketPolicy names a scope outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "registerLocation": true, + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": [ + "SELECT" + ] + } + ], + "bucketPolicy": "same-account" + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/bucketPolicy", + "keyword": "enum" + } + ] + } + ] +} diff --git a/tests/0.7.7/lifecycle-observability.json b/tests/0.7.7/lifecycle-observability.json new file mode 100644 index 0000000..f98da2a --- /dev/null +++ b/tests/0.7.7/lifecycle-observability.json @@ -0,0 +1,1154 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "lifecycle-observability", + "description": "The lifecycle, docs, observability, defaultSLIs and exposeQoS definitions: the unified lifecycleState vocabulary, each of these objects' closed property sets, their nested enums (onBreach type/severity, alert channel kind, default SLI severity), the isoDuration and availability-percentage grammars, and the 0..1 / 0..100 numeric bounds on SLI thresholds and QoS ratios.", + "cases": [ + { + "description": "top-level lifecycle in preview carries tags and labels for automation", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "lifecycle": { + "state": "preview", + "tags": ["internal-only"], + "labels": { "stage": "beta" } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "top-level lifecycle in deprecated state carries a full deprecation policy and retention", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "lifecycle": { + "state": "deprecated", + "retention": "P2Y", + "deprecationPolicy": { + "noticePeriod": "P30D", + "contact": "data-platform@acme.com", + "replacement": "customer_profiles_v2", + "tags": ["breaking-change"], + "labels": { "migration-owner": "data-platform" } + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "top-level docs links every reference surface alongside a retired lifecycle", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "lifecycle": { "state": "retired" }, + "docs": { + "homepage": "https://internal.acme.com/data-products/customer-profiles", + "runbook": "https://internal.acme.com/runbooks/customer-profiles", + "dictionary": "https://internal.acme.com/dictionary/customer-profiles", + "changeLog": "https://internal.acme.com/changelog/customer-profiles", + "tags": ["documented"], + "labels": { "docs-owner": "customer-analytics" } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "an expose declares an active lifecycle, a full observability configuration, and boundary-value QoS targets", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "lifecycle": { "state": "active" }, + "observability": { + "metrics": [ + { "name": "row_count", "source": "warehouse.metrics.row_count", "sli": "row_count > 0" } + ], + "onBreach": [ + { + "type": "pagerduty", + "channel": "data-platform-oncall", + "target": "customer-profiles-service", + "severity": "critical", + "tags": ["p1"], + "labels": { "escalation": "immediate" } + } + ], + "defaultSLIs": { + "enabled": false, + "freshness": { + "enabled": true, + "threshold": "PT6H", + "severity": "warning", + "tags": ["freshness"], + "labels": { "sla": "gold" } + }, + "completeness": { "enabled": true, "threshold": 1, "severity": "critical" }, + "latency": { "enabled": true, "threshold": "PT5S", "percentile": 0, "severity": "info" }, + "tags": ["default-sli"], + "labels": { "generated-by": "forge-cli" } + }, + "alert": { + "channels": [ + { "kind": "webhook", "url": "https://hooks.acme.com/alerts/customer-profiles" } + ] + }, + "tags": ["observability"], + "labels": { "observed-by": "data-platform" } + }, + "qos": { + "availability": "100%", + "freshnessSLO": "PT15M", + "dataLossSLO": "0 rows", + "latencyP95": "PT5S", + "completenessTarget": 1, + "errorBudget": 0, + "tags": ["sla"], + "labels": { "tier": "gold" } + } + } + ] + }, + "valid": true + }, + { + "description": "lifecycle state uses a value outside the unified vocabulary", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "lifecycle": { "state": "sunset" }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/lifecycle/state", "keyword": "enum" }] + }, + { + "description": "lifecycle carries an owner field the closed schema does not define", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "lifecycle": { "state": "active", "owner": "data-platform" }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/lifecycle", "keyword": "additionalProperties", "detail": "owner" }] + }, + { + "description": "lifecycle retention is expressed in prose instead of an ISO-8601 duration", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "lifecycle": { "state": "active", "retention": "6 months" }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/lifecycle/retention", "keyword": "pattern" }] + }, + { + "description": "deprecation policy carries an slaHours field the closed schema does not define", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "lifecycle": { + "state": "deprecated", + "deprecationPolicy": { "noticePeriod": "P30D", "slaHours": 72 } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/lifecycle/deprecationPolicy", "keyword": "additionalProperties", "detail": "slaHours" } + ] + }, + { + "description": "docs carries a wiki field the closed schema does not define", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "docs": { + "homepage": "https://internal.acme.com/data-products/customer-profiles", + "wiki": "https://internal.acme.com/wiki/customer-profiles" + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/docs", "keyword": "additionalProperties", "detail": "wiki" }] + }, + { + "description": "expose observability carries a dashboards field the closed schema does not define", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { "dashboards": ["https://grafana.acme.com/customer-profiles"] } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability", "keyword": "additionalProperties", "detail": "dashboards" } + ] + }, + { + "description": "an observability metric entry omits its name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { "metrics": [{ "source": "warehouse.metrics.row_count" }] } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/metrics/0", "keyword": "required", "detail": "name" } + ] + }, + { + "description": "an observability metric entry omits its source", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { "metrics": [{ "name": "row_count" }] } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/metrics/0", "keyword": "required", "detail": "source" } + ] + }, + { + "description": "an onBreach notification entry omits its type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { "onBreach": [{ "channel": "data-platform-oncall" }] } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/onBreach/0", "keyword": "required", "detail": "type" } + ] + }, + { + "description": "an onBreach notification declares a delivery type outside the supported set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { "onBreach": [{ "type": "carrier-pigeon" }] } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/onBreach/0/type", "keyword": "enum" } + ] + }, + { + "description": "an onBreach notification declares a severity outside the supported set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { "onBreach": [{ "type": "slack", "severity": "catastrophic" }] } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/onBreach/0/severity", "keyword": "enum" } + ] + }, + { + "description": "an alert dispatcher channel omits its kind", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "alert": { "channels": [{ "url": "https://hooks.acme.com/alerts" }] } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/observability/alert/channels/0", + "keyword": "required", + "detail": "kind" + } + ] + }, + { + "description": "an alert dispatcher channel declares a kind outside the supported set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "alert": { "channels": [{ "kind": "carrier-pigeon" }] } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/alert/channels/0/kind", "keyword": "enum" } + ] + }, + { + "description": "the default SLI trio carries a throughput field the closed schema does not define", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { "enabled": true, "throughput": { "enabled": true } } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/observability/defaultSLIs", + "keyword": "additionalProperties", + "detail": "throughput" + } + ] + }, + { + "description": "the default freshness SLI carries a window field the closed schema does not define", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { "freshness": { "enabled": true, "window": "PT6H" } } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/observability/defaultSLIs/freshness", + "keyword": "additionalProperties", + "detail": "window" + } + ] + }, + { + "description": "the default freshness SLI declares a severity outside the supported set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { "freshness": { "severity": "catastrophic" } } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/observability/defaultSLIs/freshness/severity", + "keyword": "enum" + } + ] + }, + { + "description": "the default completeness SLI threshold is negative", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { "completeness": { "threshold": -0.1 } } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/observability/defaultSLIs/completeness/threshold", + "keyword": "minimum" + } + ] + }, + { + "description": "the default completeness SLI threshold exceeds a full ratio", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { "completeness": { "threshold": 1.5 } } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/observability/defaultSLIs/completeness/threshold", + "keyword": "maximum" + } + ] + }, + { + "description": "the default latency SLI percentile exceeds 100", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { "latency": { "percentile": 101 } } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/observability/defaultSLIs/latency/percentile", + "keyword": "maximum" + } + ] + }, + { + "description": "expose qos carries an sla field the closed schema does not define", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "qos": { "availability": "99.9%", "sla": "gold" } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/qos", "keyword": "additionalProperties", "detail": "sla" }] + }, + { + "description": "expose qos availability exceeds the percentage grammar", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "qos": { "availability": "150%" } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/qos/availability", "keyword": "pattern" }] + }, + { + "description": "expose qos completeness target exceeds a full ratio", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "qos": { "completenessTarget": 1.2 } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/qos/completenessTarget", "keyword": "maximum" }] + }, + { + "description": "expose qos error budget is negative", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "qos": { "errorBudget": -0.05 } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/qos/errorBudget", "keyword": "minimum" }] + }, + { + "description": "expose qos error budget exceeds a full ratio", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "qos": { "errorBudget": 1.5 } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/qos/errorBudget", "keyword": "maximum" }] + }, + { + "description": "expose qos completeness target is negative", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "qos": { "completenessTarget": -0.1 } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/qos/completenessTarget", "keyword": "minimum" }] + }, + { + "description": "expose qos p95 latency target is expressed in prose instead of an ISO-8601 duration", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "qos": { "latencyP95": "5 seconds" } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/qos/latencyP95", "keyword": "pattern" }] + }, + { + "description": "deprecation policy notice period is expressed in prose instead of an ISO-8601 duration", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "lifecycle": { + "state": "deprecated", + "deprecationPolicy": { "noticePeriod": "30 days" } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/lifecycle/deprecationPolicy/noticePeriod", "keyword": "pattern" }] + }, + { + "description": "the default freshness SLI threshold is expressed in prose instead of an ISO-8601 duration", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { "freshness": { "threshold": "6 hours" } } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/observability/defaultSLIs/freshness/threshold", + "keyword": "pattern" + } + ] + }, + { + "description": "the default latency SLI threshold is expressed in prose instead of an ISO-8601 duration", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { "latency": { "threshold": "5 seconds" } } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/observability/defaultSLIs/latency/threshold", + "keyword": "pattern" + } + ] + }, + { + "description": "the default latency SLI percentile is negative", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { "latency": { "percentile": -1 } } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/observability/defaultSLIs/latency/percentile", + "keyword": "minimum" + } + ] + }, + { + "description": "an observability metric entry carries a unit field the closed schema does not define", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "metrics": [ + { "name": "row_count", "source": "warehouse.metrics.row_count", "unit": "rows" } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/observability/metrics/0", + "keyword": "additionalProperties", + "detail": "unit" + } + ] + }, + { + "description": "an onBreach notification entry carries a retries field the closed schema does not define", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "onBreach": [{ "type": "slack", "retries": 3 }] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/observability/onBreach/0", + "keyword": "additionalProperties", + "detail": "retries" + } + ] + }, + { + "description": "the alert dispatcher config carries an enabled field the closed schema does not define", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "alert": { "enabled": true } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/alert", "keyword": "additionalProperties", "detail": "enabled" } + ] + }, + { + "description": "an alert dispatcher channel carries a retryPolicy field the closed schema does not define", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "alert": { + "channels": [{ "kind": "webhook", "retryPolicy": "exponential" }] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/observability/alert/channels/0", + "keyword": "additionalProperties", + "detail": "retryPolicy" + } + ] + } + ] +} diff --git a/tests/0.7.7/lineage-ml.json b/tests/0.7.7/lineage-ml.json new file mode 100644 index 0000000..7dbdbef --- /dev/null +++ b/tests/0.7.7/lineage-ml.json @@ -0,0 +1,1487 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "lineage-ml", + "description": "The optional lineage block (granularity, upstream dependency entries with their fieldMappings/transformation nesting and relationship enum, and downstream consumer entries with the impact enum), the schemaEvolution block (strategy/compatibility enums and the nested changePolicy object), and the machineLearning block (framework enum and model entries with their required fields, type enum, and nested validation object) -- including every closed object's additionalProperties:false boundary.", + "cases": [ + { + "description": "lineage, schemaEvolution and machineLearning populated to their nested edges", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "granularity": "field_level", + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "relationship": "aggregation", + "fieldMappings": [ + { + "sourceField": "event_ts", + "targetField": "profile_updated_at", + "transformation": { + "type": "calculation", + "expression": "MAX(event_ts)" + } + } + ] + } + ], + "downstream": [ + { + "consumer": "churn_model", + "impact": "high" + } + ] + }, + "schemaEvolution": { + "strategy": "semantic_versioning", + "compatibility": "backward_compatible", + "changePolicy": { + "changeWindowDays": 14, + "approvalRequired": true, + "approvers": [ + "data-governance" + ] + } + }, + "machineLearning": { + "enabled": true, + "framework": "mlflow", + "models": [ + { + "name": "churn_predictor", + "version": "1.2.0", + "type": "classification", + "algorithm": "xgboost", + "features": [ + "tenure_days", + "monthly_spend" + ], + "target": "churned", + "validation": { + "metrics": [ + "accuracy", + "auc_roc" + ], + "thresholds": { + "accuracy": 0.85 + } + } + } + ] + } + }, + "valid": true + }, + { + "description": "lineage carries an unrecognized property alongside granularity", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "granularity": "table_level", + "retentionOverride": "90d" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage", + "keyword": "additionalProperties", + "detail": "retentionOverride" + } + ] + }, + { + "description": "lineage granularity outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "granularity": "row_level" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/granularity", + "keyword": "enum" + } + ] + }, + { + "description": "upstream entry omits productId", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "exposeId": "events_table" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0", + "keyword": "required", + "detail": "productId" + } + ] + }, + { + "description": "upstream entry omits exposeId", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0", + "keyword": "required", + "detail": "exposeId" + } + ] + }, + { + "description": "upstream entry carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "sourceSystem": "kafka" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0", + "keyword": "additionalProperties", + "detail": "sourceSystem" + } + ] + }, + { + "description": "fieldMappings entry omits sourceField", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "fieldMappings": [ + { + "targetField": "profile_updated_at" + } + ] + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0/fieldMappings/0", + "keyword": "required", + "detail": "sourceField" + } + ] + }, + { + "description": "fieldMappings entry omits targetField", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "fieldMappings": [ + { + "sourceField": "event_ts" + } + ] + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0/fieldMappings/0", + "keyword": "required", + "detail": "targetField" + } + ] + }, + { + "description": "fieldMappings entry carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "fieldMappings": [ + { + "sourceField": "event_ts", + "targetField": "profile_updated_at", + "notes": "legacy mapping, do not touch" + } + ] + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0/fieldMappings/0", + "keyword": "additionalProperties", + "detail": "notes" + } + ] + }, + { + "description": "fieldMappings transformation carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "fieldMappings": [ + { + "sourceField": "event_ts", + "targetField": "profile_updated_at", + "transformation": { + "type": "calculation", + "engine": "spark" + } + } + ] + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0/fieldMappings/0/transformation", + "keyword": "additionalProperties", + "detail": "engine" + } + ] + }, + { + "description": "fieldMappings transformation type outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "fieldMappings": [ + { + "sourceField": "event_ts", + "targetField": "profile_updated_at", + "transformation": { + "type": "join" + } + } + ] + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0/fieldMappings/0/transformation/type", + "keyword": "enum" + } + ] + }, + { + "description": "upstream relationship outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "relationship": "streaming_sync" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0/relationship", + "keyword": "enum" + } + ] + }, + { + "description": "downstream entry carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "downstream": [ + { + "consumer": "churn_model", + "notifyChannel": "slack" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/downstream/0", + "keyword": "additionalProperties", + "detail": "notifyChannel" + } + ] + }, + { + "description": "downstream impact outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "downstream": [ + { + "consumer": "churn_model", + "impact": "catastrophic" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/downstream/0/impact", + "keyword": "enum" + } + ] + }, + { + "description": "schemaEvolution carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "schemaEvolution": { + "strategy": "semantic_versioning", + "migrationGuideUrl": "https://docs.acme.internal/schema-migrations" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/schemaEvolution", + "keyword": "additionalProperties", + "detail": "migrationGuideUrl" + } + ] + }, + { + "description": "schemaEvolution strategy outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "schemaEvolution": { + "strategy": "calendar_based" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/schemaEvolution/strategy", + "keyword": "enum" + } + ] + }, + { + "description": "schemaEvolution compatibility outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "schemaEvolution": { + "compatibility": "none" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/schemaEvolution/compatibility", + "keyword": "enum" + } + ] + }, + { + "description": "changePolicy carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "schemaEvolution": { + "changePolicy": { + "changeWindowDays": 14, + "escalationContact": "data-governance@acme.com" + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/schemaEvolution/changePolicy", + "keyword": "additionalProperties", + "detail": "escalationContact" + } + ] + }, + { + "description": "changePolicy changeWindowDays given as a negative number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "schemaEvolution": { + "changePolicy": { + "changeWindowDays": -5 + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/schemaEvolution/changePolicy/changeWindowDays", + "keyword": "minimum" + } + ] + }, + { + "description": "machineLearning carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "enabled": true, + "servingEndpoint": "https://models.acme.internal/churn" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning", + "keyword": "additionalProperties", + "detail": "servingEndpoint" + } + ] + }, + { + "description": "machineLearning framework outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "framework": "tensorflow_serving" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning/framework", + "keyword": "enum" + } + ] + }, + { + "description": "model entry omits name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "models": [ + { + "version": "1.0.0", + "type": "classification" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning/models/0", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "model entry omits version", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "models": [ + { + "name": "churn_predictor", + "type": "classification" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning/models/0", + "keyword": "required", + "detail": "version" + } + ] + }, + { + "description": "model entry omits type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "models": [ + { + "name": "churn_predictor", + "version": "1.0.0" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning/models/0", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "model entry carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "models": [ + { + "name": "churn_predictor", + "version": "1.0.0", + "type": "classification", + "trainingDataset": "customers_v3" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning/models/0", + "keyword": "additionalProperties", + "detail": "trainingDataset" + } + ] + }, + { + "description": "model type outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "models": [ + { + "name": "churn_predictor", + "version": "1.0.0", + "type": "forecasting" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning/models/0/type", + "keyword": "enum" + } + ] + }, + { + "description": "model validation carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "models": [ + { + "name": "churn_predictor", + "version": "1.0.0", + "type": "classification", + "validation": { + "metrics": [ + "accuracy" + ], + "reviewedBy": "data-science-lead" + } + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning/models/0/validation", + "keyword": "additionalProperties", + "detail": "reviewedBy" + } + ] + } + ] +} diff --git a/tests/0.7.7/masking-aggparams.json b/tests/0.7.7/masking-aggparams.json new file mode 100644 index 0000000..7193435 --- /dev/null +++ b/tests/0.7.7/masking-aggparams.json @@ -0,0 +1,572 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "masking-aggparams", + "description": "Typed parameters added in 0.7.6: masking params' saltEnv/keyEnv environment-variable-name pattern and keepFirst/keepLast integer bounds (0..64), which leave params open to other members as the schema states; and a semantic measure's closed aggParams object with its 0..1 percentile bounds.", + "cases": [ + { + "description": "masking params name the salt and key environment variables and keep a prefix and suffix", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "policy": { + "privacy": { + "masking": [ + { + "column": "email", + "strategy": "mask", + "params": { + "saltEnv": "FLUID_PII_HASH_SECRET", + "keyEnv": "PII_KEY", + "keepFirst": 0, + "keepLast": 64 + } + } + ] + } + } + } + ] + }, + "valid": true + }, + { + "description": "masking params.saltEnv is not an environment-variable name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "policy": { + "privacy": { + "masking": [ + { + "column": "email", + "strategy": "hash", + "params": { + "saltEnv": "pii-salt" + } + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/policy/privacy/masking/0/params/saltEnv", + "keyword": "pattern" + } + ] + }, + { + "description": "masking params.keyEnv is not an environment-variable name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "policy": { + "privacy": { + "masking": [ + { + "column": "email", + "strategy": "encrypt", + "params": { + "keyEnv": "1KEY" + } + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/policy/privacy/masking/0/params/keyEnv", + "keyword": "pattern" + } + ] + }, + { + "description": "masking params.keepFirst is negative", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "policy": { + "privacy": { + "masking": [ + { + "column": "email", + "strategy": "mask", + "params": { + "keepFirst": -1 + } + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/policy/privacy/masking/0/params/keepFirst", + "keyword": "minimum" + } + ] + }, + { + "description": "masking params.keepLast exceeds 64", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "policy": { + "privacy": { + "masking": [ + { + "column": "email", + "strategy": "mask", + "params": { + "keepLast": 65 + } + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/policy/privacy/masking/0/params/keepLast", + "keyword": "maximum" + } + ] + }, + { + "description": "masking params.keepLast is a string instead of an integer", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "policy": { + "privacy": { + "masking": [ + { + "column": "email", + "strategy": "mask", + "params": { + "keepLast": "4" + } + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/policy/privacy/masking/0/params/keepLast", + "keyword": "type" + } + ] + }, + { + "description": "a percentile measure carries aggParams at the inclusive upper bound", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "orders_model", + "measures": [ + { + "name": "p95_order_value", + "agg": "percentile", + "expr": "order_value", + "aggParams": { + "percentile": 1, + "useDiscretePercentile": true + } + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "aggParams.percentile is above 1", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "orders_model", + "measures": [ + { + "name": "p95_order_value", + "agg": "percentile", + "expr": "order_value", + "aggParams": { + "percentile": 95 + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0/aggParams/percentile", + "keyword": "maximum" + } + ] + }, + { + "description": "aggParams.percentile is negative", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "orders_model", + "measures": [ + { + "name": "p95_order_value", + "agg": "percentile", + "expr": "order_value", + "aggParams": { + "percentile": -0.5 + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0/aggParams/percentile", + "keyword": "minimum" + } + ] + }, + { + "description": "aggParams carries a property outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "orders_model", + "measures": [ + { + "name": "p95_order_value", + "agg": "percentile", + "expr": "order_value", + "aggParams": { + "percentile": 0.95, + "interpolation": "linear" + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0/aggParams", + "keyword": "additionalProperties", + "detail": "interpolation" + } + ] + } + ] +} diff --git a/tests/0.7.7/metadata.json b/tests/0.7.7/metadata.json new file mode 100644 index 0000000..c821e46 --- /dev/null +++ b/tests/0.7.7/metadata.json @@ -0,0 +1,469 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "metadata", + "description": "The metadata object: the required owner member, metadata's closed property set, the productType and classification enums, the closed owner object (team/email/slack/oncall), the closed businessContext object, and the experimental array's uniqueItems constraint. Format-assertion cases for owner.email and createdAt (format is an annotation, not an assertion, in Draft 2020-12) live in tests/optional/0.7.6/metadata.json.", + "cases": [ + { + "description": "metadata populated with every member of its closed property set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "provenance": { + "schema_version": 1, + "kind": "ContractMetadata", + "generated_at": "2024-03-01T12:00:00Z", + "generated_by": { "tool": "fluid", "version": "0.7.5", "command": "fluid init" } + }, + "layer": "Silver", + "productType": "ADP", + "classification": "confidential", + "experimental": ["acquisition"], + "owner": { + "team": "customer-analytics", + "email": "customer-analytics@acme.com", + "slack": "#customer-analytics", + "oncall": "customer-analytics-oncall" + }, + "createdAt": "2024-03-01T12:00:00Z", + "businessContext": { + "domain": "customer", + "subdomain": "profiles", + "businessCapability": "customer-360", + "valueStream": "acquisition" + }, + "tags": ["customer-data", "pii"] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "owner present as an empty object with none of its optional members set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": {} }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "owner carries slack and oncall but omits team and email", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { "slack": "#customer-analytics", "oncall": "pd-customer-analytics" } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "productType set to SDP with no layer present", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "ingestion" }, "productType": "SDP" }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "productType set to CDP with no layer present", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "analytics" }, "productType": "CDP" }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "classification set to public, the least restrictive value", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "analytics" }, "classification": "public" }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "classification set to restricted alongside empty experimental and tags arrays", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { "team": "analytics" }, + "classification": "restricted", + "experimental": [], + "tags": [] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "layer set to Bronze with no productType present", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "ingestion" }, "layer": "Bronze" }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "classification set to internal", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "analytics" }, "classification": "internal" }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "metadata is an empty object, never supplying owner", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": {}, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/metadata", "keyword": "required", "detail": "owner" }] + }, + { + "description": "metadata carries an unrecognized description property alongside owner", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" }, "description": "raw customer profile records" }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/metadata", "keyword": "additionalProperties", "detail": "description" }] + }, + { + "description": "owner carries an unrecognized phone property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics", "phone": "+1-555-0100" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/metadata/owner", "keyword": "additionalProperties", "detail": "phone" }] + }, + { + "description": "productType set to a lowercase value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "ingestion" }, "productType": "sdp" }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/metadata/productType", "keyword": "enum" }] + }, + { + "description": "classification set to a value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "analytics" }, "classification": "secret" }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/metadata/classification", "keyword": "enum" }] + }, + { + "description": "owner is a plain string rather than an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": "customer-analytics" }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/metadata/owner", "keyword": "type" }] + }, + { + "description": "metadata is a string rather than an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": "owned-by-customer-analytics", + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/metadata", "keyword": "type" }] + }, + { + "description": "businessContext carries an unrecognized region property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { "team": "customer-analytics" }, + "businessContext": { "domain": "customer", "region": "us-east" } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/metadata/businessContext", "keyword": "additionalProperties", "detail": "region" }] + }, + { + "description": "experimental repeats the same feature gate twice", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { "team": "customer-analytics" }, + "experimental": ["acquisition", "acquisition"] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/metadata/experimental", "keyword": "uniqueItems" }] + } + ] +} diff --git a/tests/0.7.7/nested-columns.json b/tests/0.7.7/nested-columns.json new file mode 100644 index 0000000..2813529 --- /dev/null +++ b/tests/0.7.7/nested-columns.json @@ -0,0 +1,1061 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "nested-columns", + "description": "Typed nested columns (0.7.7 preview): a column's fields (struct), element (array), key and value (map), each a non-empty list of columns or a nestedType whose type is required, whose property set is closed, and which nests again.", + "cases": [ + { + "description": "a struct column lists typed child columns", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "struct", + "fields": [ + { + "name": "city", + "type": "string" + }, + { + "name": "zip", + "type": "string", + "required": true + } + ] + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "an array column types its element", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "array", + "element": { + "type": "string", + "required": true + } + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "a map column types its key and value", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "map", + "key": { + "type": "string", + "required": true + }, + "value": { + "type": "int" + } + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "an array of structs, and a map of arrays, nest to any depth", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "array", + "element": { + "type": "struct", + "fields": [ + { + "name": "sku", + "type": "string" + }, + { + "name": "tags", + "type": "array", + "element": { + "type": "string" + } + } + ] + } + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "a map whose value is a map whose value is a struct", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "map", + "key": { + "type": "string", + "required": true + }, + "value": { + "type": "map", + "key": { + "type": "string", + "required": true + }, + "value": { + "type": "struct", + "fields": [ + { + "name": "a", + "type": "int" + } + ] + } + } + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "column.fields is an empty array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "struct", + "fields": [] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/fields", + "keyword": "minItems" + } + ] + }, + { + "description": "column.fields is an object, not an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "struct", + "fields": { + "name": "city", + "type": "string" + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/fields", + "keyword": "type" + } + ] + }, + { + "description": "a child column omits its type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "struct", + "fields": [ + { + "name": "city" + } + ] + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/fields/0", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "column.element is a string, not a nested type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "array", + "element": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/element", + "keyword": "type" + } + ] + }, + { + "description": "column.key is a string, not a nested type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "map", + "key": "string", + "value": { + "type": "int" + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/key", + "keyword": "type" + } + ] + }, + { + "description": "column.value is a string, not a nested type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "map", + "key": { + "type": "string" + }, + "value": "int" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/value", + "keyword": "type" + } + ] + }, + { + "description": "a nested type omits type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "array", + "element": { + "required": true + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/element", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "a nested type carries a member outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "array", + "element": { + "type": "string", + "name": "sku" + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/element", + "keyword": "additionalProperties", + "detail": "name" + } + ] + }, + { + "description": "a nested type has a type outside the column types", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "array", + "element": { + "type": "money_of_the_realm" + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/element/type", + "keyword": "anyOf" + } + ] + }, + { + "description": "a nested type's required is a string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "array", + "element": { + "type": "string", + "required": "yes" + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/element/required", + "keyword": "type" + } + ] + }, + { + "description": "a nested type's fields is an empty array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "array", + "element": { + "type": "struct", + "fields": [] + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/element/fields", + "keyword": "minItems" + } + ] + }, + { + "description": "a nested type's fields is an object, not an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "array", + "element": { + "type": "struct", + "fields": { + "name": "a", + "type": "int" + } + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/element/fields", + "keyword": "type" + } + ] + }, + { + "description": "a nested type's element is a string, not a nested type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "array", + "element": { + "type": "array", + "element": "string" + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/element/element", + "keyword": "type" + } + ] + }, + { + "description": "a nested type's key is a string, not a nested type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "array", + "element": { + "type": "map", + "key": "string", + "value": { + "type": "int" + } + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/element/key", + "keyword": "type" + } + ] + }, + { + "description": "a nested type's value is a string, not a nested type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + }, + { + "name": "details", + "type": "array", + "element": { + "type": "map", + "key": { + "type": "string" + }, + "value": "int" + } + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/contract/schema/1/element/value", + "keyword": "type" + } + ] + } + ] +} diff --git a/tests/0.7.7/orchestration.json b/tests/0.7.7/orchestration.json new file mode 100644 index 0000000..a4f64d5 --- /dev/null +++ b/tests/0.7.7/orchestration.json @@ -0,0 +1,8619 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "orchestration", + "description": "The scheduling and workflow-engine surface: the top-level orchestration block and build.execution's inline orchestration twin (engine/mode), build.execution's trigger (type, dataset references, datasetsOperator), and the Airflow-specific shapes reached through orchestration.airflow -- airflowOrchestration's dagId/dagConfig/taskDefaults/errorHandling/tasks/taskGroups, airflowTask's identifier and type-taxonomy fields plus its five conditional allOf branches (provider_action, fluid_execute, bash, python), and airflowSensor. A conformant implementation reproduces every verdict below.", + "cases": [ + { + "description": "orchestration block reaches every mounted definition -- root orchestration, build.execution's trigger and inline orchestration, airflowOrchestration's dagConfig/taskDefaults/errorHandling, every airflowTask type variant (provider_action, fluid_execute, bash, python, custom), a sensor, and a task group -- with legal values throughout", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": true + }, + { + "description": "a custom-typed task carries a legacy 'operator' alias and a params object that satisfies none of provider_action/fluid_execute/bash/python's type-specific requirements -- valid because its type/operator match none of those allOf branches", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + }, + { + "taskId": "generic_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "deliberately lacks contract_path, bash_command/command, and python_callable/callable" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": true + }, + { + "description": "a bash task additionally carries the deprecated operator alias alongside type=bash, and its params satisfy the bash param shape but neither the fluid_execute nor python shapes", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + }, + { + "taskId": "legacy_bash_alias_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": true + }, + { + "description": "build.execution carries a top-level member the schema does not define", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + }, + "scheduleHint": "run me first" + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution", + "keyword": "additionalProperties", + "detail": "scheduleHint" + } + ] + }, + { + "description": "build.execution.orchestration is present but omits engine", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/orchestration", + "keyword": "required", + "detail": "engine" + } + ] + }, + { + "description": "build.execution.orchestration.engine names an engine outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "step_functions", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/orchestration/engine", + "keyword": "enum" + } + ] + }, + { + "description": "build.execution.orchestration.mode names a mode outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "auto" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/orchestration/mode", + "keyword": "enum" + } + ] + }, + { + "description": "build.execution.trigger.type names a trigger type outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "cron_only", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/type", + "keyword": "enum" + } + ] + }, + { + "description": "a trigger dataset given as a data-product reference omits productId", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + { + "exposeId": "raw_customers_table" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/datasets/0", + "keyword": "oneOf" + } + ] + }, + { + "description": "a trigger dataset given as a data-product reference omits exposeId", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + { + "productId": "raw_customers" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/datasets/0", + "keyword": "oneOf" + } + ] + }, + { + "description": "trigger.datasetsOperator names a value outside its any/all enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "majority" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/datasetsOperator", + "keyword": "enum" + } + ] + }, + { + "description": "the top-level orchestration block omits engine", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration", + "keyword": "required", + "detail": "engine" + } + ] + }, + { + "description": "the top-level orchestration.engine names an engine outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "temporal", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/engine", + "keyword": "enum" + } + ] + }, + { + "description": "the top-level orchestration.mode names a mode outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "auto", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/mode", + "keyword": "enum" + } + ] + }, + { + "description": "orchestration.airflow is present but omits dagId", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow", + "keyword": "required", + "detail": "dagId" + } + ] + }, + { + "description": "orchestration.airflow.dagId contains characters the identifier pattern forbids", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "Customer Profiles DAG!", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/dagId", + "keyword": "pattern" + } + ] + }, + { + "description": "dagConfig.maxActiveRuns is set below its minimum of 1", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 0 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/dagConfig/maxActiveRuns", + "keyword": "minimum" + } + ] + }, + { + "description": "taskDefaults.retries is set below its minimum of 0", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": -1, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/taskDefaults/retries", + "keyword": "minimum" + } + ] + }, + { + "description": "taskDefaults.triggerRule names a rule outside Airflow's trigger-rule enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "always", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/taskDefaults/triggerRule", + "keyword": "enum" + } + ] + }, + { + "description": "taskDefaults.errorHandling.strategy names a strategy outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "immediate", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/taskDefaults/errorHandling/strategy", + "keyword": "enum" + } + ] + }, + { + "description": "an onError entry's action names a value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "page", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/taskDefaults/errorHandling/onError/0/action", + "keyword": "enum" + } + ] + }, + { + "description": "an onError entry's severity names a value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "urgent", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/taskDefaults/errorHandling/onError/0/severity", + "keyword": "enum" + } + ] + }, + { + "description": "an onError entry's when names a value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "immediately" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/taskDefaults/errorHandling/onError/0/when", + "keyword": "enum" + } + ] + }, + { + "description": "orchestration.airflow.tasks is present but empty", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks", + "keyword": "minItems" + } + ] + }, + { + "description": "a taskGroups entry omits groupId", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "description": "staging tasks, no id" + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/taskGroups/0", + "keyword": "required", + "detail": "groupId" + } + ] + }, + { + "description": "a task omits taskId", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0", + "keyword": "required", + "detail": "taskId" + } + ] + }, + { + "description": "a task's taskId contains characters the identifier pattern forbids", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "Ensure Bucket!", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/taskId", + "keyword": "pattern" + } + ] + }, + { + "description": "a task's type names a value outside the task-type taxonomy", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "spark_job", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/type", + "keyword": "enum" + } + ] + }, + { + "description": "a provider_action task names a provider outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "oracle", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/provider", + "keyword": "enum" + } + ] + }, + { + "description": "a provider_action task's action omits the required service.operation dot", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "ensurebucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/action", + "keyword": "pattern" + } + ] + }, + { + "description": "a task's produces entry omits exposeId", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/produces/0", + "keyword": "required", + "detail": "exposeId" + } + ] + }, + { + "description": "a task's produces entry names an updateStrategy outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "overwrite" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/produces/0/updateStrategy", + "keyword": "enum" + } + ] + }, + { + "description": "a task's dataProductDependencies entry omits productId", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/dataProductDependencies/0", + "keyword": "required", + "detail": "productId" + } + ] + }, + { + "description": "a task's dataProductDependencies entry omits exposeId", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/dataProductDependencies/0", + "keyword": "required", + "detail": "exposeId" + } + ] + }, + { + "description": "a task's xcom push entry names a serializer outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "yaml" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/xcom/push/0/serializer", + "keyword": "enum" + } + ] + }, + { + "description": "a provider_action task omits provider while action and params remain", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0", + "keyword": "required", + "detail": "provider" + } + ] + }, + { + "description": "a provider_action task omits action while provider and params remain", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0", + "keyword": "required", + "detail": "action" + } + ] + }, + { + "description": "a provider_action task omits params while provider and action remain", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0", + "keyword": "required", + "detail": "params" + } + ] + }, + { + "description": "a fluid_execute task's params omit contract_path", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": {} + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/1/params", + "keyword": "required", + "detail": "contract_path" + } + ] + }, + { + "description": "a bash task's params supply neither bash_command nor command", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": {} + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/2/params", + "keyword": "anyOf" + } + ] + }, + { + "description": "a python task's params supply neither python_callable nor callable", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": {} + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/3/params", + "keyword": "anyOf" + } + ] + }, + { + "description": "a sensor omits taskId", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/sensors/0", + "keyword": "required", + "detail": "taskId" + } + ] + }, + { + "description": "a sensor omits sensorType", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/sensors/0", + "keyword": "required", + "detail": "sensorType" + } + ] + }, + { + "description": "a sensor's sensorType names a value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "KafkaSensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/sensors/0/sensorType", + "keyword": "enum" + } + ] + }, + { + "description": "a sensor's pokInterval is set below its minimum of 1", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 0, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/sensors/0/pokInterval", + "keyword": "minimum" + } + ] + }, + { + "description": "a sensor's timeout is set below its minimum of 1", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 0, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/sensors/0/timeout", + "keyword": "minimum" + } + ] + }, + { + "description": "a sensor's mode names a value outside the poke/reschedule enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "async" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/sensors/0/mode", + "keyword": "enum" + } + ] + } + ] +} diff --git a/tests/0.7.7/packaging.json b/tests/0.7.7/packaging.json new file mode 100644 index 0000000..7fdd5fc --- /dev/null +++ b/tests/0.7.7/packaging.json @@ -0,0 +1,440 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "packaging", + "description": "Declarative packaging modes (0.7.6): the top-level packaging block and its per-binding override, the isolated/shared mode enum, the non-empty pool and poolManifest strings, the closed containers map and its per-kind mode enum, and packaging's own closed property set.", + "cases": [ + { + "description": "top-level packaging declares a shared pool with a per-kind containers override", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "packaging": { + "mode": "shared", + "pool": "analytics-shared", + "poolManifest": "pools/analytics-shared.yaml", + "containers": { + "bucket": "shared", + "database": "shared", + "dataset": "isolated", + "schema": "isolated", + "warehouse": "isolated", + "cluster": "shared" + } + } + }, + "valid": true + }, + { + "description": "a binding overrides the product-level packaging mode", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + }, + "packaging": { + "mode": "isolated" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "packaging.mode names a mode outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "packaging": { + "mode": "dedicated" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/packaging/mode", + "keyword": "enum" + } + ] + }, + { + "description": "packaging.pool is an empty string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "packaging": { + "mode": "shared", + "pool": "" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/packaging/pool", + "keyword": "minLength" + } + ] + }, + { + "description": "packaging.poolManifest is an empty string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "packaging": { + "mode": "shared", + "poolManifest": "" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/packaging/poolManifest", + "keyword": "minLength" + } + ] + }, + { + "description": "packaging carries a property outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "packaging": { + "mode": "shared", + "tenant": "finance" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/packaging", + "keyword": "additionalProperties", + "detail": "tenant" + } + ] + }, + { + "description": "packaging.containers names a container kind outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "packaging": { + "mode": "shared", + "containers": { + "topic": "shared" + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/packaging/containers", + "keyword": "additionalProperties", + "detail": "topic" + } + ] + }, + { + "description": "a packaging.containers entry names a mode outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "packaging": { + "mode": "shared", + "containers": { + "bucket": "pooled" + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/packaging/containers/bucket", + "keyword": "enum" + } + ] + }, + { + "description": "a binding-level packaging override names a mode outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + }, + "packaging": { + "mode": "dedicated" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/packaging/mode", + "keyword": "enum" + } + ] + } + ] +} diff --git a/tests/0.7.7/primitives.json b/tests/0.7.7/primitives.json new file mode 100644 index 0000000..92afdc2 --- /dev/null +++ b/tests/0.7.7/primitives.json @@ -0,0 +1,746 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "primitives", + "description": "The shared primitive grammars referenced throughout the document: identifier, semver, semverRange, tags, labels, isoDuration, availabilityPct and lifecycleState. Each is exercised at one or more of its real usage sites (exposeId, expose.version, qos, lifecycle, consumes), not only where the envelope group already pins it down.", + "cases": [ + { + "description": "a document exercising tags, labels, semver with prerelease and build metadata, QoS availability and freshness, lifecycle state and retention, and a consumed version constraint together", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "tags": ["customer-data", "pii"], + "labels": { "team": "customer-analytics", "cost-center": "engineering" }, + "metadata": { + "owner": { "team": "customer-analytics" }, + "tags": ["gold-tier"] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "version": "2.1.0-beta.1+exp.sha.abc123", + "tags": ["real-time"], + "labels": { "region": "us-central1" }, + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "qos": { + "availability": "99.95%", + "freshnessSLO": "PT15M" + }, + "lifecycle": { + "state": "active", + "retention": "P365D", + "deprecationPolicy": { "noticePeriod": "P30D" } + } + } + ], + "consumes": [ + { "productId": "silver.hr.people", "exposeId": "people_v2", "versionConstraint": "^1.2.0" } + ] + }, + "valid": true + }, + { + "description": "top-level tags is an empty array and top-level labels is an empty object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "tags": [], + "labels": {}, + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "QoS availability sits at the exact 100 percent upper boundary", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "qos": { "availability": "100%" } + } + ] + }, + "valid": true + }, + { + "description": "lifecycle retention combines date and time units in one ISO-8601 duration", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "lifecycle": { "retention": "P1DT12H" } + } + ] + }, + "valid": true + }, + { + "description": "a consumed product id mixes an internal dot and hyphen in the identifier's middle", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "consumes": [ + { "productId": "silver.hr.people-v2", "exposeId": "people_v2" } + ] + }, + "valid": true + }, + { + "description": "the expose id is a single underscore character", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "_", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "a consumed version constraint uses a compound range expression rather than a bare semver", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "consumes": [ + { "productId": "upstream_product", "exposeId": "upstream_expose", "versionConstraint": "^1.2.0 || ~1.4.0" } + ] + }, + "valid": true + }, + { + "description": "a top-level tag contains an uppercase letter", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "tags": ["Customer-Data"], + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/tags/0", "keyword": "pattern" }] + }, + { + "description": "the tags array repeats the same value twice", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "tags": ["pii", "pii"], + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/tags", "keyword": "uniqueItems" }] + }, + { + "description": "a top-level label value is a number instead of a string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "labels": { "team": 42 }, + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/labels/team", "keyword": "type" }] + }, + { + "description": "the QoS freshness SLO omits the ISO-8601 duration's leading P", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "qos": { "freshnessSLO": "15M" } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/qos/freshnessSLO", "keyword": "pattern" }] + }, + { + "description": "the QoS availability target exceeds 100 percent", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "qos": { "availability": "101%" } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/qos/availability", "keyword": "pattern" }] + }, + { + "description": "the expose version is prefixed with a literal v", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "version": "v1.2.3", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/version", "keyword": "pattern" }] + }, + { + "description": "the expose version's major segment carries a leading zero", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "version": "01.2.3", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/version", "keyword": "pattern" }] + }, + { + "description": "the expose lifecycle state is a value outside the vocabulary", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "lifecycle": { "state": "sunset" } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/lifecycle/state", "keyword": "enum" }] + }, + { + "description": "the expose id ends with a hyphen", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles-", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/exposeId", "keyword": "pattern" }] + }, + { + "description": "the expose id begins with a dot", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": ".profiles", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/exposeId", "keyword": "pattern" }] + }, + { + "description": "the consumed version constraint is a number instead of a string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ], + "consumes": [ + { "productId": "upstream_product", "exposeId": "upstream_expose", "versionConstraint": 42 } + ] + }, + "valid": false, + "errors": [{ "pointer": "/consumes/0/versionConstraint", "keyword": "type" }] + }, + { + "description": "the expose id contains a space in the middle", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/exposeId", "keyword": "pattern" }] + }, + { + "description": "the expose id is a single hyphen character", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "-", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/exposeId", "keyword": "pattern" }] + }, + { + "description": "the expose version omits the patch segment", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "version": "1.2", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/version", "keyword": "pattern" }] + }, + { + "description": "the expose version's prerelease segment contains an underscore", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "version": "1.2.3-beta_1", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/version", "keyword": "pattern" }] + }, + { + "description": "a top-level tag ends with a hyphen", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "tags": ["pii-"], + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/tags/0", "keyword": "pattern" }] + }, + { + "description": "top-level tags is a string instead of an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "tags": "pii", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/tags", "keyword": "type" }] + }, + { + "description": "top-level labels is an array instead of an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "labels": ["customer-analytics"], + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/labels", "keyword": "type" }] + }, + { + "description": "the QoS freshness SLO is the bare duration designator with no unit components", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "qos": { "freshnessSLO": "P" } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/qos/freshnessSLO", "keyword": "pattern" }] + }, + { + "description": "lifecycle retention lists a day component before the year component", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "lifecycle": { "retention": "P1D2Y" } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/lifecycle/retention", "keyword": "pattern" }] + }, + { + "description": "the QoS availability target omits the percent sign", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "qos": { "availability": "99.9" } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/qos/availability", "keyword": "pattern" }] + }, + { + "description": "the expose lifecycle state is set to the preview vocabulary value", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "lifecycle": { "state": "preview" } + } + ] + }, + "valid": true + } + ] +} diff --git a/tests/0.7.7/record-mapping.json b/tests/0.7.7/record-mapping.json new file mode 100644 index 0000000..f2a71af --- /dev/null +++ b/tests/0.7.7/record-mapping.json @@ -0,0 +1,3432 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "record-mapping", + "description": "How a stream's records become rows (0.7.7 preview): recordMapping's closed property set and its non-empty columns, recordExplode's path and as (required, with the dotted-path and quoted-key grammar), and recordMappingColumn's required name and from, the name and from grammars, and cast, which takes any column type.", + "cases": [ + { + "description": "a mapping with an explode and a renamed, converted column", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "items", + "as": "item" + }, + "columns": [ + { + "name": "sku", + "from": "item.sku", + "cast": "string" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a mapping with nothing in it", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": {}, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a mapping column that renames a nested field and reads a quoted key", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "address.city", + "from": "customer.address.city" + }, + { + "name": "tier", + "from": "attributes['loyalty-tier']" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a mapping column cast to a parameterised type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "amount", + "from": "amount", + "cast": "decimal(18,2)" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "an explode over a nested array path", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "order.lines", + "as": "line" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "an explode over a quoted key", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "payload['line-items']", + "as": "item" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a mapping is a string, not an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": "payment_id", + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping", + "keyword": "type" + } + ] + }, + { + "description": "a mapping carries a member outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "filter": "amount > 0" + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping", + "keyword": "additionalProperties", + "detail": "filter" + } + ] + }, + { + "description": "mapping.columns is an empty array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns", + "keyword": "minItems" + } + ] + }, + { + "description": "mapping.columns is an object, not an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": { + "name": "id", + "from": "payment_id" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns", + "keyword": "type" + } + ] + }, + { + "description": "a mapping column is a string, not an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + "id" + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0", + "keyword": "type" + } + ] + }, + { + "description": "a mapping column omits name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "from": "payment_id" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "a mapping column omits from", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "id" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0", + "keyword": "required", + "detail": "from" + } + ] + }, + { + "description": "a mapping column carries a member outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "id", + "from": "payment_id", + "default": 0 + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0", + "keyword": "additionalProperties", + "detail": "default" + } + ] + }, + { + "description": "a mapping column name is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": 7, + "from": "payment_id" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/name", + "keyword": "type" + } + ] + }, + { + "description": "a mapping column name that is not a dotted identifier path ('1id')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "1id", + "from": "payment_id" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/name", + "keyword": "pattern" + } + ] + }, + { + "description": "a mapping column name that is not a dotted identifier path ('a..b')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "a..b", + "from": "payment_id" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/name", + "keyword": "pattern" + } + ] + }, + { + "description": "a mapping column name that is not a dotted identifier path ('a.')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "a.", + "from": "payment_id" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/name", + "keyword": "pattern" + } + ] + }, + { + "description": "a mapping column name that is not a dotted identifier path (\"a['x']\")", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "a['x']", + "from": "payment_id" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/name", + "keyword": "pattern" + } + ] + }, + { + "description": "a mapping column name that is not a dotted identifier path ('my id')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "my id", + "from": "payment_id" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/name", + "keyword": "pattern" + } + ] + }, + { + "description": "a mapping column from is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "id", + "from": 7 + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/from", + "keyword": "type" + } + ] + }, + { + "description": "a mapping column from that is not a field path ('1id')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "id", + "from": "1id" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/from", + "keyword": "pattern" + } + ] + }, + { + "description": "a mapping column from that is not a field path ('a..b')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "id", + "from": "a..b" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/from", + "keyword": "pattern" + } + ] + }, + { + "description": "a mapping column from that is not a field path ('a.')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "id", + "from": "a." + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/from", + "keyword": "pattern" + } + ] + }, + { + "description": "a mapping column from that is not a field path ('a[0]')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "id", + "from": "a[0]" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/from", + "keyword": "pattern" + } + ] + }, + { + "description": "a mapping column from that is not a field path (\"a['']\")", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "id", + "from": "a['']" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/from", + "keyword": "pattern" + } + ] + }, + { + "description": "a mapping column from that is not a field path (\"a['x\\\\y']\")", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "id", + "from": "a['x\\y']" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/from", + "keyword": "pattern" + } + ] + }, + { + "description": "a mapping column from that is not a field path ('a b')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "id", + "from": "a b" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/from", + "keyword": "pattern" + } + ] + }, + { + "description": "a mapping column from that is not a field path (\"items['xxxxxxxxxxxxx\")", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "id", + "from": "items['xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx']" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/from", + "keyword": "pattern" + } + ] + }, + { + "description": "a mapping column cast to a type that is not a column type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "id", + "from": "payment_id", + "cast": "money_of_the_realm" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/columns/0/cast", + "keyword": "anyOf" + } + ] + }, + { + "description": "an explode is a string, not an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": "items" + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode", + "keyword": "type" + } + ] + }, + { + "description": "an explode omits path", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "as": "item" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode", + "keyword": "required", + "detail": "path" + } + ] + }, + { + "description": "an explode omits as", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "items" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode", + "keyword": "required", + "detail": "as" + } + ] + }, + { + "description": "an explode carries a member outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "items", + "as": "item", + "outer": true + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode", + "keyword": "additionalProperties", + "detail": "outer" + } + ] + }, + { + "description": "an explode path is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": 7, + "as": "item" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode/path", + "keyword": "type" + } + ] + }, + { + "description": "an explode path that is not a field path ('1items')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "1items", + "as": "item" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode/path", + "keyword": "pattern" + } + ] + }, + { + "description": "an explode path that is not a field path ('order..lines')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "order..lines", + "as": "item" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode/path", + "keyword": "pattern" + } + ] + }, + { + "description": "an explode path that is not a field path ('order.')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "order.", + "as": "item" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode/path", + "keyword": "pattern" + } + ] + }, + { + "description": "an explode path that is not a field path ('items[0]')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "items[0]", + "as": "item" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode/path", + "keyword": "pattern" + } + ] + }, + { + "description": "an explode path that is not a field path (\"items['']\")", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "items['']", + "as": "item" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode/path", + "keyword": "pattern" + } + ] + }, + { + "description": "an explode as is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "items", + "as": 7 + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode/as", + "keyword": "type" + } + ] + }, + { + "description": "an explode as that is not a single identifier ('1item')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "items", + "as": "1item" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode/as", + "keyword": "pattern" + } + ] + }, + { + "description": "an explode as that is not a single identifier ('it.em')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "items", + "as": "it.em" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode/as", + "keyword": "pattern" + } + ] + }, + { + "description": "an explode as that is not a single identifier ('it em')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "items", + "as": "it em" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode/as", + "keyword": "pattern" + } + ] + }, + { + "description": "an explode as that is not a single identifier ('')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "explode": { + "path": "items", + "as": "" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/mapping/explode/as", + "keyword": "pattern" + } + ] + } + ] +} diff --git a/tests/0.7.7/semantic-model.json b/tests/0.7.7/semantic-model.json new file mode 100644 index 0000000..b397156 --- /dev/null +++ b/tests/0.7.7/semantic-model.json @@ -0,0 +1,1974 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "semantic-model", + "description": "The expose-level semantic model block (expose.semantics): its own closed object type, and the entities/measures/dimensions/metrics arrays -- their required members, closed property sets, and the type/agg enums, including the nested nonAdditiveDimension and typeParams sub-objects.", + "cases": [ + { + "description": "semantic model on a table expose carries entities, measures (with a non-additive dimension), dimensions (with typeParams), and metrics", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "semantic model block carries an unrecognized top-level property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ], + "grain": "one row per customer" + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics", + "keyword": "additionalProperties", + "detail": "grain" + } + ] + }, + { + "description": "entity omits its name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/entities/0", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "entity omits its type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/entities/0", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "entity carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id", + "sensitivity": "high" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/entities/0", + "keyword": "additionalProperties", + "detail": "sensitivity" + } + ] + }, + { + "description": "entity type value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "compound", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/entities/0/type", + "keyword": "enum" + } + ] + }, + { + "description": "measure omits its name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "measure omits its aggregation function", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0", + "keyword": "required", + "detail": "agg" + } + ] + }, + { + "description": "measure carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + }, + "unit": "USD" + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0", + "keyword": "additionalProperties", + "detail": "unit" + } + ] + }, + { + "description": "measure aggregation function outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "stddev", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0/agg", + "keyword": "enum" + } + ] + }, + { + "description": "measure's non-additive dimension carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max", + "granularity": "day" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0/nonAdditiveDimension", + "keyword": "additionalProperties", + "detail": "granularity" + } + ] + }, + { + "description": "non-additive dimension windowChoice value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "avg" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0/nonAdditiveDimension/windowChoice", + "keyword": "enum" + } + ] + }, + { + "description": "dimension omits its name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/dimensions/0", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "dimension omits its type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/dimensions/0", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "dimension carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + }, + "sortOrder": "asc" + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/dimensions/0", + "keyword": "additionalProperties", + "detail": "sortOrder" + } + ] + }, + { + "description": "dimension type value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "geospatial", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/dimensions/0/type", + "keyword": "enum" + } + ] + }, + { + "description": "dimension typeParams carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day", + "format": "ISO8601" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/dimensions/0/typeParams", + "keyword": "additionalProperties", + "detail": "format" + } + ] + }, + { + "description": "dimension typeParams timeGranularity value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "fortnight" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/dimensions/0/typeParams/timeGranularity", + "keyword": "enum" + } + ] + }, + { + "description": "metric omits its name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/metrics/0", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "metric omits its type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/metrics/0", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "metric carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'", + "unit": "count" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/metrics/0", + "keyword": "additionalProperties", + "detail": "unit" + } + ] + }, + { + "description": "metric type value outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "computed", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/metrics/0/type", + "keyword": "enum" + } + ] + } + ] +} diff --git a/tests/0.7.7/sovereignty.json b/tests/0.7.7/sovereignty.json new file mode 100644 index 0000000..c433d74 --- /dev/null +++ b/tests/0.7.7/sovereignty.json @@ -0,0 +1,711 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "sovereignty", + "description": "The top-level sovereignty block: its own object type, the jurisdiction and enforcementMode enums, the allowedRegions/deniedRegions/transferMechanisms/regulatoryFramework arrays (their array type, item type/enum, and uniqueItems), the dataResidency/crossBorderTransfer/validationRequired booleans, and the block's closed property set.", + "cases": [ + { + "description": "EU jurisdiction with strict enforcement and no cross-border transfer", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "allowedRegions": ["eu-west-1", "eu-central-1"], + "deniedRegions": ["us-east-1"], + "dataResidency": true, + "crossBorderTransfer": false, + "regulatoryFramework": ["GDPR"], + "enforcementMode": "strict", + "validationRequired": true + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "Multi-Region jurisdiction permits cross-border transfer under every transferMechanisms value", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "Multi-Region", + "crossBorderTransfer": true, + "transferMechanisms": ["SCCs", "BCRs", "Adequacy", "DPF", "Consent", "Derogation"] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "CN jurisdiction with audit-only enforcement and validation not required", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "CN", + "enforcementMode": "audit", + "validationRequired": false + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "JP jurisdiction with advisory enforcement and only deniedRegions declared", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "JP", + "enforcementMode": "advisory", + "deniedRegions": ["cn-north-1"] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "regulatoryFramework carries every enum value beyond GDPR", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "US", + "regulatoryFramework": ["CCPA", "CPRA", "HIPAA", "PIPEDA", "LGPD", "PDPA", "POPIA", "DPA", "APPI"] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "sovereignty block carries its own tags and labels", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "BR", + "tags": ["pii", "cross-border"], + "labels": { "policy-owner": "legal-team" } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": true + }, + { + "description": "jurisdiction value not in the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "DE" + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/jurisdiction", "keyword": "enum" }] + }, + { + "description": "enforcementMode value not in the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "enforcementMode": "block" + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/enforcementMode", "keyword": "enum" }] + }, + { + "description": "transferMechanisms carries a mechanism outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "crossBorderTransfer": true, + "transferMechanisms": ["Handshake"] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/transferMechanisms/0", "keyword": "enum" }] + }, + { + "description": "regulatoryFramework carries a framework outside the enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "US", + "regulatoryFramework": ["SOX"] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/regulatoryFramework/0", "keyword": "enum" }] + }, + { + "description": "dataResidency given as a string instead of a boolean", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "dataResidency": "true" + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/dataResidency", "keyword": "type" }] + }, + { + "description": "crossBorderTransfer given as a number instead of a boolean", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "crossBorderTransfer": 0 + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/crossBorderTransfer", "keyword": "type" }] + }, + { + "description": "validationRequired given as a string instead of a boolean", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "validationRequired": "yes" + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/validationRequired", "keyword": "type" }] + }, + { + "description": "allowedRegions repeats the same region twice", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "allowedRegions": ["eu-west-1", "eu-west-1"] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/allowedRegions", "keyword": "uniqueItems" }] + }, + { + "description": "deniedRegions repeats the same region twice", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "deniedRegions": ["us-east-1", "us-east-1"] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/deniedRegions", "keyword": "uniqueItems" }] + }, + { + "description": "transferMechanisms repeats the same mechanism twice", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "crossBorderTransfer": true, + "transferMechanisms": ["SCCs", "SCCs"] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/transferMechanisms", "keyword": "uniqueItems" }] + }, + { + "description": "regulatoryFramework repeats the same framework twice", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "US", + "regulatoryFramework": ["CCPA", "CCPA"] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/regulatoryFramework", "keyword": "uniqueItems" }] + }, + { + "description": "sovereignty carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "prohibitTransferTo": ["CN", "RU"] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty", "keyword": "additionalProperties", "detail": "prohibitTransferTo" }] + }, + { + "description": "sovereignty given as an array instead of an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": ["EU"], + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty", "keyword": "type" }] + }, + { + "description": "allowedRegions given as a string instead of an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "allowedRegions": "eu-west-1" + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/allowedRegions", "keyword": "type" }] + }, + { + "description": "allowedRegions item given as a number instead of a string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "allowedRegions": [123] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/allowedRegions/0", "keyword": "type" }] + }, + { + "description": "deniedRegions given as a string instead of an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "deniedRegions": "us-east-1" + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/deniedRegions", "keyword": "type" }] + }, + { + "description": "deniedRegions item given as a boolean instead of a string", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "deniedRegions": [true] + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/deniedRegions/0", "keyword": "type" }] + }, + { + "description": "transferMechanisms given as a string instead of an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "EU", + "crossBorderTransfer": true, + "transferMechanisms": "SCCs" + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/transferMechanisms", "keyword": "type" }] + }, + { + "description": "regulatoryFramework given as a string instead of an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "sovereignty": { + "jurisdiction": "US", + "regulatoryFramework": "GDPR" + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/sovereignty/regulatoryFramework", "keyword": "type" }] + } + ] +} diff --git a/tests/0.7.7/stream-source.json b/tests/0.7.7/stream-source.json new file mode 100644 index 0000000..df7390c --- /dev/null +++ b/tests/0.7.7/stream-source.json @@ -0,0 +1,4091 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "stream-source", + "description": "A streaming build's stream (0.7.7 preview): embedded-logic's stream and mapping members; sql optional only when a managed Flink build (parameters.flink.management) reads a stream; mapping requires stream; and streamSource's required members, closed property set, kind, topic grammar, format enum and non-empty lists, plus streamKey and streamMetadataColumn.", + "cases": [ + { + "description": "a managed Flink build reads a stream and has no sql", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a build with its own sql and a stream that describes what the sql reads", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_clean SELECT * FROM payments", + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + } + } + } + }, + "valid": true + }, + { + "description": "a stream with every member: key, Kafka metadata columns and avro-confluent values", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "avro-confluent", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "key": { + "format": "raw", + "schema": [ + { + "name": "payment_key", + "type": "string" + } + ], + "prefix": "key_" + }, + "metadata": [ + { + "name": "event_time", + "from": "timestamp" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a metadata column read from topic", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [ + { + "name": "meta_topic", + "from": "topic" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a metadata column read from partition", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [ + { + "name": "meta_partition", + "from": "partition" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a metadata column read from offset", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [ + { + "name": "meta_offset", + "from": "offset" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a metadata column read from timestamp", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [ + { + "name": "meta_timestamp", + "from": "timestamp" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a metadata column read from timestamp-type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [ + { + "name": "meta_timestamp_type", + "from": "timestamp-type" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a metadata column read from headers", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [ + { + "name": "meta_headers", + "from": "headers" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a metadata column read from leader-epoch", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [ + { + "name": "meta_leader_epoch", + "from": "leader-epoch" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a stream key encoded as raw", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "key": { + "format": "raw", + "schema": [ + { + "name": "payment_key", + "type": "string" + } + ] + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a stream key encoded as json", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "key": { + "format": "json", + "schema": [ + { + "name": "payment_key", + "type": "string" + } + ] + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a stream key encoded as avro-confluent", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "key": { + "format": "avro-confluent", + "schema": [ + { + "name": "payment_key", + "type": "string" + } + ] + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a build with neither sql nor a stream", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties", + "keyword": "required", + "detail": "sql" + } + ] + }, + { + "description": "a build with a stream but no sql and no managed Flink parameters", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties", + "keyword": "required", + "detail": "sql" + } + ] + }, + { + "description": "a build with a stream and Flink parameters that declare no management", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "parallelism": 2 + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties", + "keyword": "required", + "detail": "sql" + } + ] + }, + { + "description": "a build with a stream and parameters that declare no flink block", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "parallelism": 2 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties", + "keyword": "required", + "detail": "sql" + } + ] + }, + { + "description": "a build with managed Flink parameters and no stream or sql", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties", + "keyword": "required", + "detail": "sql" + } + ] + }, + { + "description": "a build with a stream and no parameters at all", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties", + "keyword": "required", + "detail": "sql" + } + ] + }, + { + "description": "a stream is a string, not an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": "payments.v1", + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream", + "keyword": "type" + } + ] + }, + { + "description": "a stream omits the required kind", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream", + "keyword": "required", + "detail": "kind" + } + ] + }, + { + "description": "a stream omits the required topic", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream", + "keyword": "required", + "detail": "topic" + } + ] + }, + { + "description": "a stream omits the required format", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream", + "keyword": "required", + "detail": "format" + } + ] + }, + { + "description": "a stream omits the required schema", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json" + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream", + "keyword": "required", + "detail": "schema" + } + ] + }, + { + "description": "a stream carries a member outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "brokers": "kafka:9092" + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream", + "keyword": "additionalProperties", + "detail": "brokers" + } + ] + }, + { + "description": "a stream kind other than kafka", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "pulsar", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/kind", + "keyword": "const" + } + ] + }, + { + "description": "a stream format outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "csv", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/format", + "keyword": "enum" + } + ] + }, + { + "description": "a stream topic is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": 7, + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/topic", + "keyword": "type" + } + ] + }, + { + "description": "a stream topic that is not one legal Kafka topic name ('payments v1')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/topic", + "keyword": "pattern" + } + ] + }, + { + "description": "a stream topic that is not one legal Kafka topic name ('.')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": ".", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/topic", + "keyword": "pattern" + } + ] + }, + { + "description": "a stream topic that is not one legal Kafka topic name ('..')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "..", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/topic", + "keyword": "pattern" + } + ] + }, + { + "description": "a stream topic that is not one legal Kafka topic name ('aaaaaaaaaaaa'...)", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/topic", + "keyword": "pattern" + } + ] + }, + { + "description": "a stream topic that is not one legal Kafka topic name ('pay*')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "pay*", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/topic", + "keyword": "pattern" + } + ] + }, + { + "description": "a stream topic that is not one legal Kafka topic name ('')", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/topic", + "keyword": "pattern" + } + ] + }, + { + "description": "a stream topic of 249 characters, with dots, dashes and underscores", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa._-0", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a stream schema is an empty array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/schema", + "keyword": "minItems" + } + ] + }, + { + "description": "a stream schema is an object, not an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": { + "name": "a", + "type": "string" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/schema", + "keyword": "type" + } + ] + }, + { + "description": "a stream schema column omits its type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/schema/0", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "stream metadata is an empty array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/metadata", + "keyword": "minItems" + } + ] + }, + { + "description": "stream metadata is an object, not an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": { + "name": "t", + "from": "topic" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/metadata", + "keyword": "type" + } + ] + }, + { + "description": "a stream metadata column is a string, not an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [ + "topic" + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/metadata/0", + "keyword": "type" + } + ] + }, + { + "description": "a stream metadata column omits name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [ + { + "from": "topic" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/metadata/0", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "a stream metadata column omits from", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [ + { + "name": "t" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/metadata/0", + "keyword": "required", + "detail": "from" + } + ] + }, + { + "description": "a stream metadata column carries a member outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [ + { + "name": "t", + "from": "topic", + "cast": "string" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/metadata/0", + "keyword": "additionalProperties", + "detail": "cast" + } + ] + }, + { + "description": "a stream metadata column has an empty name", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [ + { + "name": "", + "from": "topic" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/metadata/0/name", + "keyword": "minLength" + } + ] + }, + { + "description": "a stream metadata column name is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [ + { + "name": 7, + "from": "topic" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/metadata/0/name", + "keyword": "type" + } + ] + }, + { + "description": "a stream metadata column reads from outside the Kafka metadata enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "metadata": [ + { + "name": "t", + "from": "key" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/metadata/0/from", + "keyword": "enum" + } + ] + }, + { + "description": "a stream key is a string, not an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "key": "payment_key" + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/key", + "keyword": "type" + } + ] + }, + { + "description": "a stream key omits format", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "key": { + "schema": [ + { + "name": "payment_key", + "type": "string" + } + ] + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/key", + "keyword": "required", + "detail": "format" + } + ] + }, + { + "description": "a stream key omits schema", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "key": { + "format": "json" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/key", + "keyword": "required", + "detail": "schema" + } + ] + }, + { + "description": "a stream key carries a member outside its closed set", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "key": { + "format": "json", + "schema": [ + { + "name": "payment_key", + "type": "string" + } + ], + "compression": "gzip" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/key", + "keyword": "additionalProperties", + "detail": "compression" + } + ] + }, + { + "description": "a stream key format outside its enum", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "key": { + "format": "xml", + "schema": [ + { + "name": "payment_key", + "type": "string" + } + ] + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/key/format", + "keyword": "enum" + } + ] + }, + { + "description": "a stream key schema is an empty array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "key": { + "format": "json", + "schema": [] + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/key/schema", + "keyword": "minItems" + } + ] + }, + { + "description": "a stream key schema is an object, not an array", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "key": { + "format": "json", + "schema": { + "name": "a", + "type": "string" + } + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/key/schema", + "keyword": "type" + } + ] + }, + { + "description": "a stream key prefix does not start with a letter or underscore", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "key": { + "format": "json", + "schema": [ + { + "name": "payment_key", + "type": "string" + } + ], + "prefix": "1key_" + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/key/prefix", + "keyword": "pattern" + } + ] + }, + { + "description": "a stream key prefix is a number", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ], + "key": { + "format": "json", + "schema": [ + { + "name": "payment_key", + "type": "string" + } + ], + "prefix": 7 + } + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/stream/key/prefix", + "keyword": "type" + } + ] + }, + { + "description": "a mapping next to the stream it reads", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "stream": { + "kind": "kafka", + "topic": "payments.v1", + "format": "json", + "schema": [ + { + "name": "payment_id", + "type": "string" + }, + { + "name": "amount", + "type": "decimal" + } + ] + }, + "mapping": { + "columns": [ + { + "name": "id", + "from": "payment_id" + } + ] + }, + "parameters": { + "flink": { + "management": "operator" + } + } + } + } + }, + "valid": true + }, + { + "description": "a mapping without a stream", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "SELECT 1", + "mapping": { + "columns": [ + { + "name": "id", + "from": "payment_id" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties", + "keyword": "dependentRequired" + } + ] + } + ] +} diff --git a/tests/0.7.7/trigger-continuous.json b/tests/0.7.7/trigger-continuous.json new file mode 100644 index 0000000..2fd63ee --- /dev/null +++ b/tests/0.7.7/trigger-continuous.json @@ -0,0 +1,1175 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "trigger-continuous", + "description": "build.execution.trigger.type continuous (0.7.6): a long-lived job whose platform keeps it running, so its trigger takes no other trigger. cron, schedule, datasets, event and timetable are each refused next to it; every other trigger type keeps accepting them; a trigger that is not an object gets exactly one error; and each trigger member keeps its own type.", + "cases": [ + { + "description": "a continuous trigger names the type and nothing else", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuous" + } + } + } + }, + "valid": true + }, + { + "description": "a continuous trigger may carry the members that are not triggers (condition, datasetsOperator, timezone)", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuous", + "condition": "always", + "datasetsOperator": "all", + "timezone": "UTC" + } + } + } + }, + "valid": true + }, + { + "description": "a continuous trigger also carries cron", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuous", + "cron": "0 * * * *" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger", + "keyword": "not" + } + ] + }, + { + "description": "a continuous trigger also carries schedule", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuous", + "schedule": "@hourly" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger", + "keyword": "not" + } + ] + }, + { + "description": "a continuous trigger also carries datasets", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuous", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers" + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger", + "keyword": "not" + } + ] + }, + { + "description": "a continuous trigger also carries event", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuous", + "event": "payments.landed" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger", + "keyword": "not" + } + ] + }, + { + "description": "a continuous trigger also carries timetable", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuous", + "timetable": { + "class": "acme.Timetable" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger", + "keyword": "not" + } + ] + }, + { + "description": "a schedule trigger keeps carrying cron and schedule", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "cron": "0 * * * *", + "schedule": "@hourly", + "timezone": "UTC" + } + } + } + }, + "valid": true + }, + { + "description": "a dataset trigger keeps carrying datasets, and an event trigger keeps carrying event", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers" + ], + "event": "payments.landed" + } + } + } + }, + "valid": true + }, + { + "description": "a timetable trigger keeps carrying timetable", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "timetable", + "timetable": { + "class": "acme.Timetable" + } + } + } + } + }, + "valid": true + }, + { + "description": "a trigger with no type is not a continuous trigger, so it may carry cron", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "cron": "0 * * * *" + } + } + } + }, + "valid": true + }, + { + "description": "a trigger that is a string, not an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": "continuous" + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger", + "keyword": "type" + } + ] + }, + { + "description": "a trigger that is an array, not an object", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": [ + "continuous" + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger", + "keyword": "type" + } + ] + }, + { + "description": "a trigger type outside the enum, near continuous", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "continuously" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/type", + "keyword": "enum" + } + ] + }, + { + "description": "trigger.schedule has the wrong type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "schedule": 5 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/schedule", + "keyword": "type" + } + ] + }, + { + "description": "trigger.event has the wrong type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "event": 5 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/event", + "keyword": "type" + } + ] + }, + { + "description": "trigger.condition has the wrong type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "condition": 5 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/condition", + "keyword": "type" + } + ] + }, + { + "description": "trigger.datasets has the wrong type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "datasets": "dataset://gcp/acme-prod/customers/raw_customers" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/datasets", + "keyword": "type" + } + ] + }, + { + "description": "trigger.cron has the wrong type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "cron": 5 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/cron", + "keyword": "type" + } + ] + }, + { + "description": "trigger.timezone has the wrong type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "timezone": 5 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/timezone", + "keyword": "type" + } + ] + }, + { + "description": "trigger.timetable has the wrong type", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "embedded-logic", + "engine": "flink", + "properties": { + "sql": "INSERT INTO payments_reconciled SELECT * FROM payments" + }, + "execution": { + "trigger": { + "type": "schedule", + "timetable": "acme.Timetable" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/timetable", + "keyword": "type" + } + ] + } + ] +} diff --git a/tests/README.md b/tests/README.md index 9be47f9..de126d6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -28,10 +28,11 @@ tests/ Each published schema version that has a directory here is run by `conformance/run.py`; versions without one are not part of the corpus. Today -that is 0.7.5, the latest stable version, and 0.7.6, a preview (see +that is 0.7.5, 0.7.6, the latest stable version, and 0.7.7, a preview (see `scripts/schema-versions.json`). The 0.7.6 directory carries every 0.7.5 group, ported unchanged apart from the version strings, plus groups for what 0.7.6 -adds. +adds; the 0.7.7 directory carries every 0.7.6 group the same way, plus groups +for what 0.7.7 adds. The core/optional split follows [JSON-Schema-Test-Suite](https://github.com/json-schema-org/JSON-Schema-Test-Suite). @@ -194,7 +195,7 @@ decision. Both were observed with `jsonschema` 4.26 (Draft 2020-12). ### Airflow task rules apply to every task that has `params` -In 0.7.1 through 0.7.6, each `allOf` rule in `$defs/airflowTask` (fluid_execute, +In 0.7.1 through 0.7.7, each `allOf` rule in `$defs/airflowTask` (fluid_execute, bash, python) has an `if` of the form `anyOf: [{properties: {type: {const: ...}}}, {properties: {operator: {pattern: ...}}}]`. A `properties` subschema is true when its member is absent, so a task that omits `operator` (or `type`) matches @@ -232,7 +233,8 @@ from the denominator; counting them would blame the corpus for a gap it cannot close, and they are findings about the *schema* instead. Today: **41.0%** for 0.7.5 (432 of 1053 falsifiable constraints, 123 -redundant) and **43.1%** for 0.7.6 (518 of 1201, 130 redundant). The gate +redundant), **45.7%** for 0.7.6 (570 of 1248, 135 redundant) and **48.2%** +for 0.7.7 (632 of 1310, 135 redundant). The gate applies to the lowest. Most of what remains is `type` constraints, which need a wrong-typed value for every property in the schema; `--format json` lists the unpinned enums, required members, closed objects and bounds. @@ -266,3 +268,12 @@ published here as a pre-release copy, and replacing it with the stable one added the pgvector platform, the pgvector_table format, binding.vectorConfig and new location fields that no case pinned, which took coverage below the CI floor until the cases in `tests/0.7.5/binding.json` were added. + +0.7.6 was re-vendored again from forge-cli v0.27.0, with the `continuous` +trigger, the team roster, the object `description`, contract ids, Kafka +serialization and column `quality` added after the copy first published. Without +new cases its coverage fell to 41.7%, and the 0.7.7 preview, added in the same +change, would have been at 39.7%, under the floor. The cases are in +`tests/0.7.6/trigger-continuous.json` and `tests/0.7.6/bitol-fields.json`, and +for 0.7.7 in `tests/0.7.7/stream-source.json`, `record-mapping.json` and +`nested-columns.json`. diff --git a/tests/meta_test.py b/tests/meta_test.py index 14eb0b6..61a8396 100644 --- a/tests/meta_test.py +++ b/tests/meta_test.py @@ -557,12 +557,12 @@ def gate() -> subprocess.CompletedProcess: gone.write_bytes(saved) # 5. a schema that is neither synced nor frozen - stray = root / "schema" / "fluid-schema-0.7.7.json" + stray = root / "schema" / "fluid-schema-0.7.8.json" shutil.copy(root / "schema" / "fluid-schema-0.7.6.json", stray) r = gate() check( "rejects a schema that is neither synced nor frozen", - r.returncode == 1 and "fluid-schema-0.7.7.json" in r.stderr, + r.returncode == 1 and "fluid-schema-0.7.8.json" in r.stderr, f"exit={r.returncode}", ) stray.unlink() diff --git a/tests/optional/0.7.7/metadata.json b/tests/optional/0.7.7/metadata.json new file mode 100644 index 0000000..4c9370b --- /dev/null +++ b/tests/optional/0.7.7/metadata.json @@ -0,0 +1,56 @@ +{ + "corpusVersion": "0.7.7", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", + "group": "metadata", + "description": "Format-assertion cases for the metadata object: `format` is an annotation, not an assertion, in JSON Schema Draft 2020-12, so an implementation may legitimately accept these documents. These cases exist for implementations that opt into asserting owner.email and metadata.createdAt.", + "cases": [ + { + "description": "owner email is not a well-formed email address", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics", "email": "not-an-email" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/metadata/owner/email", "keyword": "format" }] + }, + { + "description": "createdAt holds a bare date with no time component", + "contract": { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" }, "createdAt": "2024-03-01" }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/metadata/createdAt", "keyword": "format" }] + } + ] +}