diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 3a8212e..f3cfb9a 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -15,7 +15,8 @@ example README, package README and user-facing command text.
`from` and `to`; never paste sample YAML into a page. `` shows
part of a file in `ci-templates/` the same way. What a sample uses comes from resolving it; never hand-write it.
- Facts the code knows (settings, defaults, connector facts, error codes,
- document fields, CLI commands) are rendered by components in
+ document fields, CLI commands, what each pack does with intent, and the
+ published standards' files) are rendered by components in
`components/reference/`. Change the code, not the page.
- Every page belongs to one of seven areas, each a folder whose `meta.json`
says `"root": true`. A folder in brackets, such as `how-to/(build)/`,
diff --git a/app/blocks.css b/app/blocks.css
index 315bcd1..8fca0a4 100644
--- a/app/blocks.css
+++ b/app/blocks.css
@@ -205,3 +205,55 @@ figure[data-kind="run"] .line.cmd::before {
color: var(--brand-signal);
user-select: none;
}
+
+/* A value still REPLACE_ME in a kit's file (components/replace-me.tsx): a
+ dashed underline and its note's number mark it, not colour alone, with a
+ light tint. Its tooltip sits above the page; touch screens, which cannot
+ hover, show the notes below the block instead. */
+.replace-me {
+ text-decoration: underline dashed 1px;
+ text-underline-offset: 3px;
+ background: color-mix(in srgb, var(--docs-link) 12%, transparent);
+ border-radius: 2px;
+ cursor: help;
+}
+.replace-me::after {
+ content: attr(data-note);
+ margin-left: 0.15em;
+ font-size: 0.7em;
+ vertical-align: super;
+ line-height: 0;
+}
+.replace-me:focus-visible {
+ outline: 2px solid var(--docs-link);
+ outline-offset: 2px;
+}
+.replace-me-tip {
+ position: fixed;
+ z-index: 60;
+ max-width: min(24rem, calc(100vw - 16px));
+ padding: 0.6rem 0.75rem;
+ border: 1px solid var(--color-fd-border);
+ border-radius: 0.5rem;
+ background: var(--color-fd-popover);
+ color: var(--color-fd-popover-foreground);
+ box-shadow: 0 6px 20px rgb(0 0 0 / 0.18);
+ font-size: 0.85rem;
+ line-height: 1.5;
+}
+.replace-me-line {
+ display: block;
+ margin-top: 0.3rem;
+}
+.prose .replace-me-notes {
+ display: none;
+ font-size: 0.9rem;
+}
+.replace-me-where {
+ color: var(--color-fd-muted-foreground);
+}
+@media (hover: none), print {
+ .prose .replace-me-notes {
+ display: block;
+ }
+}
diff --git a/app/content.css b/app/content.css
index c6d422f..81f7de8 100644
--- a/app/content.css
+++ b/app/content.css
@@ -219,3 +219,9 @@ pre code > .line {
height: 0;
scroll-margin-top: 7rem;
}
+
+/* Where to find a field's value, and a suggestion (components/reference/field-rows.tsx): each on a line of its own. */
+.field-help {
+ display: block;
+ margin-top: 0.3rem;
+}
diff --git a/ci-templates/azure-devops/azure-pipelines.yml b/ci-templates/azure-devops/azure-pipelines.yml
index 10eed32..358be6f 100644
--- a/ci-templates/azure-devops/azure-pipelines.yml
+++ b/ci-templates/azure-devops/azure-pipelines.yml
@@ -33,7 +33,7 @@ stages:
version: 22.x
# An exact version: change it when you upgrade, as
# https://docs.ingestron.io/docs/how-to/upgrade#choose-the-version-you-run says.
- - script: npm install -g ingestron@2.2.0
+ - script: npm install -g ingestron@2.3.0
displayName: Install Ingestron
- script: ingestron check
displayName: ingestron check
diff --git a/ci-templates/azure-devops/ingestron-check.yml b/ci-templates/azure-devops/ingestron-check.yml
index a32ac45..0df120e 100644
--- a/ci-templates/azure-devops/ingestron-check.yml
+++ b/ci-templates/azure-devops/ingestron-check.yml
@@ -15,7 +15,7 @@ steps:
version: 22.x
# An exact version: change it when you upgrade, as
# https://docs.ingestron.io/docs/how-to/upgrade#choose-the-version-you-run says.
- - script: npm install -g ingestron@2.2.0
+ - script: npm install -g ingestron@2.3.0
displayName: Install Ingestron
- script: ingestron check
displayName: ingestron check
diff --git a/ci-templates/github-actions/deploy.yml b/ci-templates/github-actions/deploy.yml
index fec6a14..922bf20 100644
--- a/ci-templates/github-actions/deploy.yml
+++ b/ci-templates/github-actions/deploy.yml
@@ -29,7 +29,7 @@ jobs:
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: 22
- - run: npm install -g ingestron@2.2.0
+ - run: npm install -g ingestron@2.3.0
- run: ingestron check
build:
diff --git a/ci-templates/github-actions/ingestron-check.yml b/ci-templates/github-actions/ingestron-check.yml
index d01ca94..ea7efe5 100644
--- a/ci-templates/github-actions/ingestron-check.yml
+++ b/ci-templates/github-actions/ingestron-check.yml
@@ -18,5 +18,5 @@ jobs:
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: 22
- - run: npm install -g ingestron@2.2.0
+ - run: npm install -g ingestron@2.3.0
- run: ingestron check
diff --git a/components/mdx.tsx b/components/mdx.tsx
index 57f66be..a344809 100644
--- a/components/mdx.tsx
+++ b/components/mdx.tsx
@@ -23,7 +23,20 @@ import { ContractUseTable } from "@/components/reference/contract-use";
import { PackCapabilities } from "@/components/reference/pack-capabilities";
import { DefaultLayers, PackNaming } from "@/components/reference/pack-defaults";
import { SamplesCatalogue } from "@/components/reference/samples-catalogue";
-import { ContractFields, DocumentFields } from "@/components/reference/document-fields";
+import {
+ ContractFields,
+ DocumentFields,
+ PlatformSection,
+} from "@/components/reference/document-fields";
+import {
+ IntentAliases,
+ IntentOverview,
+ IntentSupport,
+ IntentVocabulary,
+ QualityEngines,
+ ZonesNotApplied,
+} from "@/components/reference/pack-intent";
+import { KitFacts, KitFile, KitIntent, KitList } from "@/components/reference/kits";
import { ErrorCodes, LoadingCodes } from "@/components/reference/error-codes";
import { ExtensionCommands } from "@/components/reference/extension-commands";
import { ExamplesList } from "@/components/reference/examples-list";
@@ -82,6 +95,17 @@ export function getMDXComponents(components?: MDXComponents) {
LoadingCodes,
DocumentFields,
ContractFields,
+ PlatformSection,
+ IntentSupport,
+ IntentOverview,
+ QualityEngines,
+ ZonesNotApplied,
+ IntentVocabulary,
+ IntentAliases,
+ KitList,
+ KitFacts,
+ KitFile,
+ KitIntent,
CliCommands,
BuiltInTemplates,
ProjectSources,
diff --git a/components/reference/connector-settings.tsx b/components/reference/connector-settings.tsx
index 5910fed..21923d6 100644
--- a/components/reference/connector-settings.tsx
+++ b/components/reference/connector-settings.tsx
@@ -3,6 +3,7 @@ import type { Connector } from "@ingestron/definitions";
import { parse } from "yaml";
import { readExampleFile } from "@/lib/examples";
import { connectorEntries, shown } from "@/lib/reference";
+import { FindAndSuggest } from "./field-rows";
import { Inline } from "./inline";
interface Props {
@@ -84,6 +85,11 @@ function ConnectionField({
+ {(setting.find || setting.suggest) && (
+
+
+
+ )}
{example !== undefined && (
In the example: {shown(example)}.
diff --git a/components/reference/document-fields.tsx b/components/reference/document-fields.tsx
index 7967615..49582b7 100644
--- a/components/reference/document-fields.tsx
+++ b/components/reference/document-fields.tsx
@@ -1,6 +1,6 @@
// A document kind's fields at every depth, from its JSON Schema.
import { contractReads, type FieldInfo } from "@ingestron/definitions";
-import { documentFields } from "@/lib/reference";
+import { documentFields, packName, platformSectionFields } from "@/lib/reference";
import { Meaning, Name } from "./field-rows";
import { TableFrame } from "@/components/table-frame";
@@ -29,7 +29,16 @@ export function ContractFields() {
);
}
-function FieldTable({ label, fields }: { label: string; fields: FieldInfo[] }) {
+/** The fields of a pack's own section of the Platform, written under the pack's id.
+ * @example
+ */
+export function PlatformSection({ pack }: { pack: string }) {
+ const fields = platformSectionFields(pack);
+ if (fields.length === 0) throw new Error(`The ${pack} pack's Platform section has no fields.`);
+ return ;
+}
+
+export function FieldTable({ label, fields }: { label: string; fields: FieldInfo[] }) {
return (
diff --git a/components/reference/error-codes.tsx b/components/reference/error-codes.tsx
index 03e6658..df786ab 100644
--- a/components/reference/error-codes.tsx
+++ b/components/reference/error-codes.tsx
@@ -1,5 +1,6 @@
-// Every code Ingestron reports, with its meaning and fix from the code, each
-// with an anchor that problems in the editor link to.
+// Every code Ingestron reports, with its meaning, the levels it is reported
+// at and its fix from the code, each with an anchor that problems in the
+// editor link to.
import { codeHelp, loadingCodeHelp } from "@ingestron/definitions";
import { Inline } from "./inline";
@@ -24,11 +25,19 @@ export function LoadingCodes() {
);
}
+/** The levels a code is reported at, in words: "a warning", "an error or a warning". */
+export function levelWords(severities: readonly string[]): string {
+ const words = severities.map((severity) =>
+ severity === "error" ? "an error" : severity === "warning" ? "a warning" : "information",
+ );
+ return words.length > 1 ? `${words.slice(0, -1).join(", ")} or ${words.at(-1)}` : words[0]!;
+}
+
function Codes({
entries,
level: Heading,
}: {
- entries: [string, { meaning: string; fix: string }][];
+ entries: [string, { meaning: string; fix: string; severities: readonly string[] }][];
level: "h3" | "h4";
}) {
return (
@@ -37,10 +46,16 @@ function Codes({
{code}
+ {help.severities.length === 0 && " (removed)"}
+ {help.severities.length > 0 && (
+
+ Reported as: {levelWords(help.severities)}.
+
+ )}
Fix:
diff --git a/components/reference/field-rows.tsx b/components/reference/field-rows.tsx
index dd82da3..51b7aab 100644
--- a/components/reference/field-rows.tsx
+++ b/components/reference/field-rows.tsx
@@ -1,10 +1,11 @@
// Rows of described fields: what each is, its values with their meanings, its
-// default and an example. Each row has the anchor the editor links to.
+// default, an example, and where to find a value with a suggestion, as the
+// editor's hover says them. Each row has the anchor the editor links to.
import type { FieldInfo } from "@ingestron/definitions";
import { anchorId, fieldPath, shown } from "@/lib/reference";
import { Inline } from "./inline";
-/** The meaning column: description, allowed values with meanings, example. */
+/** The meaning column: description, allowed values with meanings, example, where to find it and a suggestion. */
export function Meaning({ field }: { field: FieldInfo }) {
const described = field.values ?? [];
return (
@@ -36,6 +37,25 @@ export function Meaning({ field }: { field: FieldInfo }) {
For example: {shown(field.example)}.
>
)}
+
+ >
+ );
+}
+
+/** Where the owner finds a value, and what it could look like, each on a line of its own. */
+export function FindAndSuggest({ find, suggest }: { find?: string; suggest?: string }) {
+ return (
+ <>
+ {find && (
+
+ Where to find it:
+
+ )}
+ {suggest && (
+
+ Suggested: {suggest}
+
+ )}
>
);
}
diff --git a/components/reference/kits.tsx b/components/reference/kits.tsx
new file mode 100644
index 0000000..8046218
--- /dev/null
+++ b/components/reference/kits.tsx
@@ -0,0 +1,291 @@
+// Published standards (kits), from the installed @ingestron/standards
+// package: the list to choose from, one kit's facts, a file of it with each
+// REPLACE_ME explained, and how each of its tables resolves.
+import type { ReactNode } from "react";
+import { ServerCodeBlock, type ServerCodeBlockProps } from "fumadocs-ui/components/codeblock.rsc";
+import { KindBlock } from "@/components/code-block";
+import { ReplaceMe, TipText } from "@/components/replace-me";
+import { TableFrame } from "@/components/table-frame";
+import { codeThemes } from "@/lib/code-themes";
+import { pickPath } from "@/lib/extract";
+import {
+ kitBlanks,
+ kitById,
+ kitFile,
+ kitIntent,
+ kitPage,
+ latestKits,
+ type Blank,
+} from "@/lib/kits";
+import { packName } from "@/lib/reference";
+import { Inline } from "./inline";
+
+const levels = { ga: "Generally available features only", preview: "Uses preview features" };
+
+/** The pack ids of a kit's `packs`, such as `databricks` for `databricks@2`. */
+const packIds = (packs: string[]) => packs.map((pack) => pack.split("@")[0]!);
+
+/**
+ * Every published standard installed with Ingestron, newest version of each,
+ * linked to its page.
+ * @example
+ */
+export function KitList() {
+ return (
+
+
+
+ );
+}
+
+/**
+ * One kit's facts, from its kit.yaml: what it builds, its version, packs,
+ * release level and the Ingestron it needs.
+ * @example
+ */
+export function KitFacts({ id }: { id: string }) {
+ const kit = kitById(id);
+ return (
+
+ ))}
+
+ )}
+ >
+ );
+}
+
+/** Where an intent value came from, in the words the editor's hover uses. */
+function from(
+ origin: { layer: string; file?: string; line?: number; alias?: string },
+ kind: string,
+) {
+ const layer = {
+ pack: "the pack's default",
+ standard: "the standard",
+ platform: "the platform",
+ layer: "the table's layer in the standard",
+ rule: "a standard rule",
+ definition: kind === "Flow" ? "the flow" : "the transform",
+ table: "this table",
+ fixed: "the standard, fixed",
+ }[origin.layer];
+ const where = origin.file ? ` (${origin.file}:${origin.line})` : "";
+ return origin.alias ? `the setting "${origin.alias}" in ${layer}${where}` : `${layer}${where}`;
+}
+
+/**
+ * Each of the kit's tables, its intent and where each value came from, as
+ * `plan` resolves the kit and the editor's hover shows it.
+ * @example
+ */
+export function KitIntent({ id }: { id: string }) {
+ const kit = kitById(id);
+ return (
+
+
diff --git a/components/reference/pack-intent.tsx b/components/reference/pack-intent.tsx
new file mode 100644
index 0000000..3628d3d
--- /dev/null
+++ b/components/reference/pack-intent.tsx
@@ -0,0 +1,464 @@
+// What each pack does with each intent path and value, from the pack's own
+// `intent` declarations: the support matrix on each pack's page, the methods
+// that differ from their pack, and an overview of every pack for the intent
+// reference. Nothing here is written by hand.
+import type { FeatureStatus, IntentUse } from "@ingestron/definitions";
+import { intentPaths, intentUseOf } from "@ingestron/definitions";
+import {
+ aliasTexts,
+ intentRows,
+ intentVocabulary,
+ methodIntentRows,
+ packName,
+ packOf,
+ packs,
+} from "@/lib/reference";
+import { Inline } from "./inline";
+import { TableFrame } from "@/components/table-frame";
+
+const uses = {
+ used: "Used",
+ "not-yet": "Not yet",
+ elsewhere: "Elsewhere",
+ "not-applicable": "Not applicable",
+} as const;
+
+/** What the entry says: what the pack does, or why it does not. */
+function said(use: IntentUse): string {
+ return use.use === "used" ? use.does : use.reason;
+}
+
+/**
+ * Where a used entry applies only in some layers, and the time zones it
+ * cannot convert, as sentences after what it does.
+ */
+function limits(use: IntentUse, shown?: string): string[] {
+ if (use.use !== "used") return [];
+ const found: string[] = [];
+ if (use.onlyWhere === shown) {
+ // Said once for the whole table.
+ } else if (use.onlyWhere === "check")
+ found.push("Only in a layer whose `does` lists `check`; a `raw` layer only counts.");
+ else if (use.onlyWhere) found.push(`Only in a layer whose \`does\` lists \`${use.onlyWhere}\`.`);
+ const zones = Object.keys(use.zonesNotApplied ?? {});
+ if (zones.length > 0)
+ found.push(`Not from or to ${zones.map((zone) => `\`${zone}\``).join(", ")}.`);
+ return found;
+}
+
+/**
+ * What the entry says, with the layers it applies in and the zones it cannot
+ * convert; `shown` is a layer duty a table has already said its entries need.
+ */
+function Said({ use, shown }: { use: IntentUse; shown?: string }) {
+ return ;
+}
+
+/** The quality and cleansing paths, which each pack's page lists apart from the rest of intent. */
+export function isQualityPath(path: string): boolean {
+ return path.startsWith("quality.") || path.startsWith("cleanse.") || path === "deduplicate";
+}
+
+/** The access paths (grants, exact grants, masks and row filters), which each pack's page also lists apart. */
+export function isAccessPath(path: string): boolean {
+ return path.startsWith("access.");
+}
+
+/** The part of a pack's page an intent path is listed in. */
+function partOf(path: string): "quality" | "access" | undefined {
+ return isQualityPath(path) ? "quality" : isAccessPath(path) ? "access" : undefined;
+}
+
+/** What a part's values are called in its table's label. */
+const partWords = {
+ quality: { value: "quality and cleansing value", differs: " in quality and cleansing" },
+ access: { value: "access value", differs: " in access" },
+} as const;
+
+/** A value as the vocabulary writes it; `*` is any text, such as a column's name. */
+function Values({ values }: { values: string[] }) {
+ if (values.length === 1 && values[0] === "*") return <>any>;
+ return (
+ <>
+ {values.map((value, index) => (
+
+ {index > 0 && ", "}
+ {value}
+
+ ))}
+ >
+ );
+}
+
+/** The platform feature's release level, linked to the page that says so. */
+function Release({ status }: { status?: FeatureStatus }) {
+ if (!status) return null;
+ const words = status.level === "ga" ? "Generally available" : (status.label ?? "In preview");
+ return {words};
+}
+
+/**
+ * The pack's support matrix: each intent path and value, whether the pack
+ * uses it, what it does or why not, the layers it applies in, and, where it is
+ * used, the platform feature's release level, linked to the page that says so,
+ * and the date it was checked. Then each method whose own entries differ from
+ * the pack's. `part="quality"` keeps only the quality and cleansing paths, and
+ * `part="access"` only the access paths; without it, both are left out, for
+ * the page's sections of their own.
+ * @example
+ * @example
+ * @example
+ */
+export function IntentSupport({ pack, part }: { pack: string; part?: "quality" | "access" }) {
+ const name = packName(pack);
+ const kept = (path: string) => partOf(path) === part;
+ const methods = methodIntentRows(pack).filter((row) => kept(row.path));
+ const what = part ? partWords[part].value : "intent value";
+ return (
+ <>
+
+
+
+ );
+}
+
+/**
+ * The intent vocabulary: each path, what it means, its values with their
+ * meanings, and where it can be written, from the documents' own fields.
+ * @example
+ */
+export function IntentVocabulary() {
+ return (
+
+
+
+ );
+}
+
+/**
+ * The time zones a pack cannot convert from or to, each with its reason, from
+ * its `cleanse.timezone` entry. Throws for a pack that lists none.
+ * @example
+ */
+export function ZonesNotApplied({ pack }: { pack: string }) {
+ const use = intentUseOf(packOf(pack), "cleanse.timezone", "true");
+ const zones = Object.entries((use?.use === "used" && use.zonesNotApplied) || {});
+ if (zones.length === 0) throw new Error(`The ${pack} pack converts every time zone.`);
+ return (
+
+
+
+
+
Zone
+
Why
+
+
+
+ {zones.map(([zone, reason]) => (
+
+
+ {zone}
+
+
+
+
+
+ ))}
+
+
+
+ );
+}
+
+/** The rows of the engine table: what a reader asks, and the intent path and value that answers it. */
+const engineRows: { label: string; path: string; value: string }[] = [
+ { label: "Where checks run", path: "quality.rules", value: "notNull" },
+ { label: "`warn`", path: "quality.actions", value: "warn" },
+ { label: "`drop`", path: "quality.actions", value: "drop" },
+ { label: "`fail`", path: "quality.actions", value: "fail" },
+ { label: "Table checks", path: "quality.table-checks", value: "true" },
+ { label: "`pattern` rules", path: "quality.rules", value: "pattern" },
+ { label: "`items` rules", path: "quality.rules", value: "items" },
+ { label: "Quarantine", path: "quality.quarantine", value: "true" },
+ { label: "Results", path: "quality.results", value: "true" },
+];
+
+/**
+ * What each engine does with a table's checks, from each pack's own intent:
+ * where they run, each action, table checks, the rule kinds an engine cannot
+ * apply, quarantine and results. A value a pack does not apply says why.
+ * @example
+ */
+export function QualityEngines({
+ packs: shown = ["databricks", "duckdb", "synapse"],
+}: {
+ packs?: string[];
+}) {
+ return (
+
+
+ {shown.map((pack) => {
+ const use = intentUseOf(packOf(pack), path, value);
+ if (!use) throw new Error(`The ${pack} pack says nothing about ${path}: ${value}.`);
+ return (
+
+ {use.use === "used" ? (
+ // The first row says where checks run; the others need not repeat it.
+
+ ) : (
+ <>
+ Not applied:
+ >
+ )}
+
+ );
+ })}
+
+ ))}
+
+
+
+ );
+}
diff --git a/components/reference/pack-methods.tsx b/components/reference/pack-methods.tsx
index d339cd5..9c47507 100644
--- a/components/reference/pack-methods.tsx
+++ b/components/reference/pack-methods.tsx
@@ -1,28 +1,48 @@
-// Every method of a pack: what it does, what it reads and hands over, its settings.
-import { packs } from "@/lib/reference";
+// Every method of a pack: what it does, what it reads and hands over, the
+// intent it gives and the settings that are other names for intent, and its
+// settings.
+import { aliasTexts, intentPairs, packOf } from "@/lib/reference";
+import { Inline } from "./inline";
import { SettingsTable } from "./settings-table";
/** One section per method, with an id the connector pages link to. */
export function PackMethods({ pack }: { pack: string }) {
- const found = packs.find((candidate) => candidate.id === pack);
- if (!found) throw new Error(`No pack "${pack}".`);
+ const found = packOf(pack);
return (
<>
- {Object.entries(found.methods).map(([id, method]) => (
-
-
+
+
+ );
+ })}
>
);
}
diff --git a/components/reference/versions.tsx b/components/reference/versions.tsx
index 40096f0..e082d91 100644
--- a/components/reference/versions.tsx
+++ b/components/reference/versions.tsx
@@ -33,6 +33,10 @@ export function PackageVersions() {
["@ingestron/core", "the API the command is built on"],
["@ingestron/definitions", "loading and resolving definitions"],
["@ingestron/authoring", "editor features"],
+ [
+ "@ingestron/standards",
+ "the [published standards](/docs/how-to/choose-a-published-standard) installed with the command",
+ ],
...packs.map((pack): [string, string] => [
`@ingestron/pack-${pack.id}`,
`${packName(pack.id)} pack, \`${pack.id}@${pack.version}\``,
diff --git a/components/reference/vscode-extension.json b/components/reference/vscode-extension.json
index 3c04cdb..bf38d5b 100644
--- a/components/reference/vscode-extension.json
+++ b/components/reference/vscode-extension.json
@@ -1,3 +1,3 @@
{
- "version": "2.1.4"
+ "version": "2.2.0"
}
diff --git a/components/replace-me.tsx b/components/replace-me.tsx
new file mode 100644
index 0000000..2d47b56
--- /dev/null
+++ b/components/replace-me.tsx
@@ -0,0 +1,135 @@
+"use client";
+// A value still REPLACE_ME in a kit's file (components/reference/kits.tsx):
+// marked by a dashed underline, a light tint and its note's number, not by
+// colour alone, and focusable. On hover or focus it shows the field, where to find the value and
+// a suggestion, in a tooltip drawn above the page, so the code block's
+// scrolling frame does not clip it and Copy does not copy it. Escape closes
+// it, and the pointer can move onto it. The same text is the note the mark's
+// aria-describedby names, which touch screens show below the block.
+import { useEffect, useLayoutEffect, useRef, useState, type ReactNode } from "react";
+import { createPortal } from "react-dom";
+import { Inline } from "./reference/inline";
+
+/** The help one mark gives, as the editor's hover gives it. */
+export interface ReplaceMeTip {
+ field: string;
+ description?: string;
+ find?: string;
+ suggest?: string;
+}
+
+/** One REPLACE_ME, with the id of its note and the note's number. */
+export function ReplaceMe({
+ note,
+ number,
+ tip,
+ children,
+}: {
+ note: string;
+ number: number;
+ tip?: ReplaceMeTip;
+ children: ReactNode;
+}) {
+ const mark = useRef(null);
+ const box = useRef(null);
+ const closing = useRef(undefined);
+ const [open, setOpen] = useState(false);
+ const [place, setPlace] = useState<{ top: number; left: number } | undefined>();
+
+ const show = () => {
+ window.clearTimeout(closing.current);
+ setOpen(true);
+ };
+ // A moment's grace, so the pointer can move from the mark onto the tooltip.
+ const hide = () => {
+ window.clearTimeout(closing.current);
+ closing.current = window.setTimeout(() => setOpen(false), 150);
+ };
+
+ // Below the mark, or above it when there is no room, and inside the window.
+ useLayoutEffect(() => {
+ if (!open || !mark.current || !box.current) return;
+ const at = mark.current.getBoundingClientRect();
+ const size = box.current.getBoundingClientRect();
+ const below = at.bottom + 6;
+ const top = below + size.height > window.innerHeight ? at.top - 6 - size.height : below;
+ const left = Math.max(8, Math.min(at.left, window.innerWidth - size.width - 8));
+ setPlace({ top: Math.max(8, top), left });
+ }, [open]);
+
+ useEffect(() => {
+ if (!open) return;
+ const close = () => setOpen(false);
+ const escape = (event: KeyboardEvent) => {
+ if (event.key === "Escape") close();
+ };
+ window.addEventListener("scroll", close, true);
+ window.addEventListener("resize", close);
+ window.addEventListener("keydown", escape);
+ return () => {
+ window.removeEventListener("scroll", close, true);
+ window.removeEventListener("resize", close);
+ window.removeEventListener("keydown", escape);
+ };
+ }, [open]);
+
+ useEffect(() => () => window.clearTimeout(closing.current), []);
+
+ return (
+ setOpen(false)}
+ >
+ {children}
+ {open &&
+ tip &&
+ createPortal(
+
+
+
,
+ document.body,
+ )}
+
+ );
+}
+
+/** The help as text: the field, then what it is, where to find it and a suggestion. */
+export function TipText({ tip }: { tip: ReplaceMeTip }) {
+ return (
+ <>
+
+ {tip.field}
+ {" "}
+ still needs a value.
+ {tip.description && (
+
+
+
+ )}
+ {tip.find && (
+
+ Where to find it:
+
+ )}
+ {tip.suggest && (
+
+ Suggested: {tip.suggest}
+
+ )}
+ >
+ );
+}
diff --git a/components/screenshot.tsx b/components/screenshot.tsx
index 8a3ff26..772d048 100644
--- a/components/screenshot.tsx
+++ b/components/screenshot.tsx
@@ -56,7 +56,7 @@ type Captures =
/**
* One framed screenshot.
- * @example
+ * @example
* @example
*/
export function Screenshot({
diff --git a/content/docs/(reference)/reference/cli.mdx b/content/docs/(reference)/reference/cli.mdx
index 3f1b7a9..70f78f2 100644
--- a/content/docs/(reference)/reference/cli.mdx
+++ b/content/docs/(reference)/reference/cli.mdx
@@ -5,7 +5,8 @@ description: The ingestron command's commands, options, output and exit codes.
The `ingestron` command reads a project folder and writes each platform's
files, for every environment. [Install it](/docs/start/install) first. `ingestron init` creates a
-project; see [Create a project](/docs/how-to/start-from-a-template). Then
+project; see [Create a project](/docs/how-to/start-from-a-template), or
+[Published standards](#standard) below. Then
`ingestron plan` and `ingestron apply` are the everyday loop; see
[Plan, apply and review](/docs/how-to/plan-and-generate). In CI, run
`ingestron check`; see [Check generated files in CI](/docs/how-to/check-in-ci).
@@ -36,6 +37,31 @@ ingestron check --definitions shop --out shop/build
With `--definitions shop` alone, `apply` would write `build` in the folder you
ran it from, not in `shop`.
+## Published standards [#standard]
+
+`ingestron init --kit @` creates a project from a
+[published standard](/docs/how-to/choose-a-published-standard). `--kit` takes
+the name and version, such as `duckdb/local-medallion@2026-11-10`; the name
+alone, for the newest version that is not a preview among the installed
+versions and the catalogue's tags (the newest installed one, with a note,
+when the catalogue cannot be reached); or
+`#@`, for one kept in a Git repository. It is not
+combined with `--template` or `--standard`. See
+[Start from a published standard](/docs/how-to/start-from-a-published-standard).
+
+`ingestron standard` takes one of two commands:
+
+- `ingestron standard install ` replaces the project's standard with the
+ published standard's, keeping its path and `id`, and lists the published
+ standard's other files that changed; it refuses a standard file with
+ changes that are not committed. It exits `0` once the file is written, and
+ `1` when it writes nothing. See
+ [Move to a newer published standard](/docs/how-to/upgrade-a-standard).
+- `ingestron standard list` lists the versions installed with the command,
+ then those only the public catalogue has. `--platform ` keeps one
+ platform's. It always exits `0`. See
+ [List them](/docs/how-to/choose-a-published-standard#list-them).
+
## Output
`plan` and `apply` print one block for each output folder under `--out`, such
diff --git a/content/docs/(reference)/reference/documents.mdx b/content/docs/(reference)/reference/documents.mdx
index 117c2cf..9bd78ba 100644
--- a/content/docs/(reference)/reference/documents.mdx
+++ b/content/docs/(reference)/reference/documents.mdx
@@ -4,7 +4,9 @@ description: Every field of project.yaml and the Standard, Platform, Connection,
---
Each document starts with `apiVersion: ingestron.io/v1`, its `kind` and an
-`id`. The fields below come from the documents' schemas. For how the
+`id`. The fields below come from the documents' schemas, with where to find a
+value and a suggestion for each field that has them, as the editor's hover
+gives them. For how the
documents work together, see [Definitions](/docs/concepts/definitions).
Your editor can check these as you type: run `ingestron schemas` and follow
@@ -24,28 +26,58 @@ documents read with `${values.…}`, and the files to load. It has no
See [Standards and packs](/docs/concepts/standards-and-packs),
-[Which setting wins](/docs/concepts/settings) and [Name tables and resources](/docs/how-to/name-things-your-way).
+[Which setting wins](/docs/concepts/settings), [Intent](/docs/reference/intent)
+for `layers`, `load`, `audit`, `operations` and the other intent,
+[What a layer does](/docs/concepts/layers#what-a-layer-does) for `does`,
+[Check data quality](/docs/how-to/add-quality-checks) for `quality`,
+[Cleanse data in the right layer](/docs/how-to/cleanse-data) for `cleanse`,
+[Define a data product](/docs/how-to/define-a-data-product) for `governance`'s
+rules, [Grant access, mask columns and filter rows](/docs/how-to/grant-access)
+for a layer's `access` and `governance`'s `access.grants`, `masks` and
+`filters`, and [Name tables and resources](/docs/how-to/name-things-your-way).
## Platform
+Each pack can have a section of its own, under its id, such as `databricks:`;
+its fields are on the pack's page, as for
+[the `databricks:` section](/docs/reference/packs/databricks#platform) and
+[the `synapse:` section](/docs/reference/packs/synapse#platform). The
+Databricks fields below, `workspace`, `mode`, `run_as`, `compute`,
+`permissions`, `manage` and `grants`, are read at the top of the platform as
+before, with a
+[`PLATFORM_FIELD_MOVED`](/docs/reference/errors#PLATFORM_FIELD_MOVED) warning,
+and `notifications` with one that moves it to `alerts`.
+
-See [Environments and platform](/docs/concepts/environments-and-platform).
+See [Environments and platform](/docs/concepts/environments-and-platform),
+[Accept a warning](/docs/how-to/accept-a-warning) for `accept`, and
+[Grant access, mask columns and filter rows](/docs/how-to/grant-access#roles)
+for `principals`.
## Connection
See [Connectors](/docs/connectors) and [Reachability](/docs/start/choose-a-route#reachability).
+A connection's `timezone` is the zone its timestamps are converted from; see
+[Cleanse data in the right layer](/docs/how-to/cleanse-data#add-the-columns-to-cleanse).
## Flow
+See [Check data quality](/docs/how-to/add-quality-checks) for `expectations`
+and `quality`, and [Cleanse data in the right layer](/docs/how-to/cleanse-data)
+for `cleanse`.
+
## Transform
+A transform with one input may leave out `code`; see
+[Leave out the code](/docs/how-to/write-a-transform#without-code).
+
## Data contract
Contracts use the
diff --git a/content/docs/(reference)/reference/errors.mdx b/content/docs/(reference)/reference/errors.mdx
index 88f88d0..2957a17 100644
--- a/content/docs/(reference)/reference/errors.mdx
+++ b/content/docs/(reference)/reference/errors.mdx
@@ -9,6 +9,13 @@ whole project, such as a pack's notice, is reported as `(project)`.
A warning, such as a value still [`REPLACE_ME`](#REPLACE_ME), does not stop
`plan` or `apply`; `check` reports a value still `REPLACE_ME` as an error.
+Each code below says how it is reported: as an error, a warning or
+information. A feature the engine that keeps a table cannot apply is
+[`NOT_APPLIED`](#NOT_APPLIED), listed apart under `Not applied on this platform`;
+see [Not applied here](/docs/concepts/not-applied). A warning you have decided
+to live with can be accepted in `platform.yaml`, and is then counted, not
+shown; see [Accept a warning](/docs/how-to/accept-a-warning). A code marked
+removed is no longer reported, and its help says what replaced it.
A pack that cannot be loaded stops every command before it reads the
definitions: the command prints [`PACK_LOAD`](#PACK_LOAD) alone and exits with
diff --git a/content/docs/(reference)/reference/intent.mdx b/content/docs/(reference)/reference/intent.mdx
new file mode 100644
index 0000000..31fcb4b
--- /dev/null
+++ b/content/docs/(reference)/reference/intent.mdx
@@ -0,0 +1,213 @@
+---
+title: Intent
+description: The words a standard, flow or transform uses for how data is loaded, kept and run, which value wins, where each came from, and what each pack does with it.
+---
+
+Intent says how a table's data is loaded, kept and run, in words that belong
+to no platform: `load: files`, `history: latest`, `audit: [loaded_at]`. Each
+pack turns it into its platform's terms, and says what it cannot do. Every
+part is optional: a project that writes none builds as it did before.
+
+This is the standard of the
+[local medallion](/docs/how-to/kits/duckdb-local-medallion) published
+standard. Its `layers`, with what each layer `does`, `load`, `cleanse`,
+`quality` and `features` are intent; `ingest` is a method setting:
+
+
+
+## Paths and values [#paths]
+
+Each path, what it means, its values, and the places that can write it.
+Follow a path's link for every field that writes it in
+[Documents](/docs/reference/documents).
+
+
+
+A duration is a whole number and a unit, `s`, `m`, `h` or `d`, such as `30d`.
+An audit value is a column after the contract's columns, named `_`,
+such as `_loaded_at`; `{ value: loaded_at, as: _ingested_at }` names it
+yourself. Two audit columns with one name, or one with a contract column's
+name, is [`BAD_SETTING`](/docs/reference/errors#BAD_SETTING).
+
+## Where to write it [#where]
+
+- **The standard**, for every table: `load` (`default`, `deletes` and `when`
+ rules), `audit`, `rescued`, `retention`, `storage`, `quality` (`required`,
+ `default`, `quarantine` and `results`), `cleanse` (`text` and `timestamps`)
+ and `operations`. `load.when` rules match a table's `connector`,
+ `connectivity` and `layer`; the first that matches chooses its load mode.
+- **A layer** of the standard, for every table in it: write the layer as an
+ object, `{ name: silver, history: latest }`, with `does`, `history`,
+ `audit`, `rescued`, `retention` and `quality`. A layer written as a name
+ gives nothing. Each layer is listed once; a second is
+ [`DUPLICATE_LAYER`](/docs/reference/errors#DUPLICATE_LAYER).
+- **A flow**, for its tables, and **a flow table**, for itself: `load`
+ (`mode`, `cursor` and `deletes`), `history`, `sequence` and `audit`. A flow
+ table also gives `cleanse` (`columns` and `deduplicate`), `quality`
+ (`quarantine` and `results`) and its `expectations`' actions.
+- **A transform**, for its table: `history`, `sequence`, `audit`, `cleanse`,
+ `quality` and its `expectations`' actions. A transform's table reads no
+ source, so it has no `load`, and the audit values only a source gives, such
+ as `source_file`, are
+ [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) there.
+- **The contract**: its rules, which become the checks judged as
+ `quality.rules` and `quality.table-checks`; see
+ [Describe a table with a contract](/docs/how-to/write-a-contract#rules-about-the-data).
+- **A connection**: `timezone`, the zone its timestamps are written in, which
+ `cleanse.timezone` converts from.
+- **The standard's `fixed`**: any of these that no layer, flow or transform
+ may change, such as `fixed: { history: latest }`. A definition that writes
+ another value is [`FIXED_SETTING`](/docs/reference/errors#FIXED_SETTING).
+
+This flow of the
+[streaming lakehouse](/docs/how-to/kits/databricks-lakehouse-streaming)
+published standard says which change to an order is newest, and which change
+is a delete:
+
+
+
+## Which value wins [#precedence]
+
+Each table's intent is resolved before its methods are chosen, from these
+places, a later one winning:
+
+1. the standard: `load.default`, `load.deletes`, `audit`, `rescued`,
+ `retention` and `storage`;
+2. the table's layer in the standard;
+3. the first `load.when` rule of the standard that matches the table;
+4. the flow or the transform;
+5. the flow table;
+6. the standard's `fixed`.
+
+Mappings merge key by key, so a flow's `load: { cursor: updated_at }` keeps
+the load mode the standard gave. A list, such as `audit`, is replaced whole.
+
+`quality` merges the same way. The actions come from the pack's defaults,
+the standard's `quality`, then the layer's, and an expectation's own `action`
+wins over all of them; in a layer whose `does` leaves out `check`, every
+action is then `warn`. `quarantine` and `results` come from the standard, the
+layer, then the flow table or the transform.
+
+Then the methods are chosen, and what is still unset comes from the chosen
+method's own default: `streaming-table` keeps every row as it arrived,
+`history: none`, unless something says otherwise. Each method's defaults are
+on its pack's page, under the method. A `${envvar:…}` value cannot decide
+what is generated, so in `load.mode`, `load.deletes.mode`, `history` or
+`storage.features` it sets nothing, and is
+[`DEPLOY_VALUE`](/docs/reference/errors#DEPLOY_VALUE) when deploying.
+
+## Where a value came from [#origins]
+
+Each value of a table's intent remembers where it was written. In VS Code,
+hover a flow table or a transform: the hover lists the table's intent, each
+value with where it came from, in these words:
+
+- **the standard**, **the table's layer in the standard**, **a standard rule**
+ or **the standard, fixed**, with the file and line;
+- **the flow** or **the transform**, and **this table**, with the file and
+ line;
+- **the setting "…"** in one of those places, for a
+ [setting that is intent](#settings-that-are-intent);
+- **the platform**, for such a setting in the platform's `settings`;
+- **the pack's default**, with no file.
+
+This is how each table of the streaming lakehouse resolves:
+
+
+
+## Settings that are intent [#settings-that-are-intent]
+
+Some method settings mean the same as intent. They still work, as intent
+written at the place the setting is: a flow's `store.with.mode: append`
+wins over a layer's `history: latest`, as the flow's own `history: none`
+would.
+
+
+
+`plan` reports each such setting as
+[`SETTING_MOVED`](/docs/reference/errors#SETTING_MOVED). In a flow's or a
+flow table's `with`, its fix, **Write as intent**, writes the intent there and
+removes the setting. Anywhere else, such as the standard's settings or a
+rule's `with`, write the intent by hand where it should apply: written for a
+whole standard or layer, intent also reaches tables and methods the setting
+did not. A setting in the standard's `fixed.` is not reported: it
+fixes that intent for the method's tables.
+
+A setting and intent at the same place that disagree, such as
+`mode: append` beside `history: latest` on one table, is
+[`INTENT_CONFLICT`](/docs/reference/errors#INTENT_CONFLICT). A setting a
+pack's own rule writes is kept as the rule wrote it, and intent that it
+overrides is `NOT_APPLIED`.
+
+## What each pack does [#support]
+
+Every pack says what it does with each value: **Used**, with how; **Not yet**;
+**Elsewhere**, done by another step or pack, such as the pack that lands the
+files; or **Not applicable** on its platform. A value its pack does not apply
+is a [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) warning where it is
+written, with the pack's reason; **Elsewhere** never warns. Each pack's name
+links to its full table, with what it does and the platform feature's release
+level.
+
+
+
+## Quality and cleansing [#quality]
+
+The quality and cleansing paths, from `quality.rules` to `deduplicate`, are
+judged for each check and each column a table cleanses, by the step that keeps
+the table: a rule's kind, its action, each table check, each operation and
+quarantine. A pack can apply one only in a layer whose `does` lists a duty,
+such as DuckDB's checks, which run only where a layer checks. Each pack's page
+lists them under **Quality and cleansing**, and a value a pack does not apply
+is [not applied here](/docs/concepts/not-applied). See
+[Check data quality](/docs/how-to/add-quality-checks) and
+[Cleanse data in the right layer](/docs/how-to/cleanse-data).
+
+## Access [#access]
+
+The access paths say who may use a table and what of it is hidden:
+`access.grants` for each role granted on it, `read` or `write`;
+`access.exact` when the standard's grants are exact; `access.masks` for each
+masked column, by its mask; and `access.filters` for each row filter. They
+are not written as intent: they come from the contracts' `roles` and
+classifications, the layers' `access`, and the standard's `governance`. Each
+item is judged by the step that keeps the table, and again by a step that
+serves it from another pack, such as a Synapse view over a Databricks table.
+
+An item is applied only when the pack uses it and the platform turns it on in
+the pack's section, such as `databricks.manage.grants`. One the pack does not
+apply is [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED), and one whose
+switch is off is [`CAPABILITY_OFF`](/docs/reference/errors#CAPABILITY_OFF),
+each at the line that declares it, once for all its tables. Each pack's page
+lists them under **Access**. See
+[Grant access, mask columns and filter rows](/docs/how-to/grant-access).
+
+## Release level of features [#features]
+
+Some values use platform features in preview, such as Databricks' run
+duration alert. The standard's `features` says which a project may use:
+
+- `features: ga`: only generally available features. A value that needs one
+ in preview is [`PREVIEW_FEATURE`](/docs/reference/errors#PREVIEW_FEATURE),
+ a warning that names the feature's release level and the date it was
+ checked, with a fix that allows preview features.
+- `features: preview`: preview features as well; they may change before they
+ are generally available.
+
+Without `features`, nothing is reported. Published standards say
+`features: ga` unless their version ends `-preview`.
+
+## Operations [#operations]
+
+`operations` is the policy for running what is generated: one schedule for
+the project (`schedule: { per: project }`), what starts a run (`trigger`),
+`retries`, a `timeout`, `alerts` and `concurrency`. Without it, the project
+runs on one schedule, and the platform's alerts are sent when a run fails. Values that differ per
+environment belong in the platform: the cron and time zone in `schedule`, who
+is told in `alerts`, and the tags in `tags`. `require_tags` names tags the
+platform must give; one missing is
+[`BAD_SETTING`](/docs/reference/errors#BAD_SETTING) at the platform's `tags`.
+
+The first pack in the standard's list that runs jobs applies `operations`:
+the Databricks refresh job, or the Data Factory triggers and copies. When no
+pack of the standard runs jobs, each value is `NOT_APPLIED`.
diff --git a/content/docs/(reference)/reference/meta.json b/content/docs/(reference)/reference/meta.json
index ac1125e..a887259 100644
--- a/content/docs/(reference)/reference/meta.json
+++ b/content/docs/(reference)/reference/meta.json
@@ -1,4 +1,4 @@
{
"title": "Reference",
- "pages": ["documents", "packs", "errors", "cli", "templates", "versions"]
+ "pages": ["documents", "intent", "packs", "errors", "cli", "templates", "versions"]
}
diff --git a/content/docs/(reference)/reference/packs/adf.mdx b/content/docs/(reference)/reference/packs/adf.mdx
index 6bb235c..fe10cff 100644
--- a/content/docs/(reference)/reference/packs/adf.mdx
+++ b/content/docs/(reference)/reference/packs/adf.mdx
@@ -45,13 +45,47 @@ once it is deployed.
## Contract fields
-What this pack does with each field of a table's contract. When a contract
-writes a field that no pack of the standard uses yet, Ingestron warns at that
-field with the reason below; **Elsewhere** fields are handled by another pack
-or step and never warn.
+What this pack does with each field of a table's contract. A field is judged
+for each table, against the pack that keeps it. When a contract writes a field
+this pack does not apply yet, on a table this pack keeps, Ingestron warns at
+that field with [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) and the
+reason below; **Elsewhere** fields are handled by another pack or step and
+never warn.
+## Intent [#intent]
+
+What this pack does with each value of [intent](/docs/reference/intent), from
+the pack itself. ADF lands files, so what a table keeps is the store pack's. A value the pack does not apply, **Not yet** or
+**Not applicable**, is a [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED)
+warning where it is written, with the reason below. **Elsewhere** values are
+applied by another pack or step and never warn.
+
+
+
+Its checks, cleansing and quarantine are in
+[Quality and cleansing](#quality-and-cleansing).
+
+## Quality and cleansing [#quality-and-cleansing]
+
+ADF lands the files as the source gives them, and checks and cleanses nothing.
+On a table stored by another pack, that pack does the work. On a table that
+ends at the landed files, each check, cleansing operation and quarantine is
+[not applied here](/docs/concepts/not-applied), with the hint to add a store
+step.
+
+
+
+## Access [#access]
+
+ADF lands files and keeps no table, so it grants, masks and filters nothing.
+Access to the landed files is set on the storage account. A table another
+pack stores is judged by that pack; a table that ends at the landed files has
+no access to judge, unless a view serves it.
+
+
+
## Native configuration sources
diff --git a/content/docs/(reference)/reference/packs/databricks.mdx b/content/docs/(reference)/reference/packs/databricks.mdx
index 8688cc9..62003f4 100644
--- a/content/docs/(reference)/reference/packs/databricks.mdx
+++ b/content/docs/(reference)/reference/packs/databricks.mdx
@@ -19,7 +19,8 @@ then [deploy the generated bundle](/docs/platforms/databricks-deploy#deploy).
Without a platform document, or where it leaves a setting out, the Databricks
pack uses these defaults. Each target of the bundle gets its environment's
-`mode`, workspace, `run_as` and `permissions`. Without `run_as`, the pipelines and jobs run as the
+`mode`, workspace, `run_as` and `permissions`, from the platform's
+`databricks:` section. Without `run_as`, the pipelines and jobs run as the
identity that deploys the bundle, and without `schedule`, nothing runs on a
schedule.
@@ -27,30 +28,165 @@ schedule.
-Only enable management of shared resources when this project owns them. The
-following capability reference describes each switch and its prerequisites.
+`schedule`, `tags` and `alerts` stay at the top of the platform, where every
+pack that runs jobs reads them. Only enable management of shared resources
+when this project owns them. The following capability reference describes
+each switch and its prerequisites.
+
+### The `databricks:` section [#platform]
+
+What only Databricks reads goes in the platform's `databricks:` section. Each
+field says where to find its value, and what it could look like; the editor's
+hover says the same.
+
+
+
+Written at the top of the platform, as before 2.3, `workspace`, `mode`,
+`run_as`, `compute`, `permissions`, `manage` and `grants` still work, with a
+[`PLATFORM_FIELD_MOVED`](/docs/reference/errors#PLATFORM_FIELD_MOVED) warning
+whose fix moves each into the section. `notifications.on_failure` becomes an
+email alert in `alerts`.
## Platform prerequisites
Some features change shared Unity Catalog objects, so the platform turns each
-on with a `manage` switch. While a switch is off, a definition or contract that
+on with a `databricks.manage` switch. While a switch is off, a definition or contract that
uses the feature gets a `CAPABILITY_OFF` warning with a quick fix that turns it
on. When a switch is on, the bundle has a `prerequisites/.sql` for
each environment, which lists the grants to run once there as an
administrator; it is not deployed.
-Tags, column comments and table checks run in an `apply_contract` task after
-the refresh, on serverless jobs compute: Lakeflow pipelines cannot set tags or
-column comments, nor compare counts over all rows. With no schedule, the job
-is still generated, to run on demand.
+Tags, column comments, table checks, grants, masks and row filters run in an
+`apply_contract` task after the refresh, on serverless jobs compute, as the
+`run_as` identity: Lakeflow pipelines cannot set tags or column comments, nor
+compare counts over all rows. With no schedule, the job is still generated,
+to run on demand. Masks and row filters match columns by their tags, so they
+need `databricks.manage.tags` too; see [Access](#access).
## Contract fields
-What this pack does with each field of a table's contract. When a contract
-writes a field that no pack of the standard uses yet, Ingestron warns at that
-field with the reason below; **Elsewhere** fields are handled by another pack
-or step and never warn.
+What this pack does with each field of a table's contract. A field is judged
+for each table, against the pack that keeps it. When a contract writes a field
+this pack does not apply yet, on a table this pack keeps, Ingestron warns at
+that field with [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) and the
+reason below; **Elsewhere** fields are handled by another pack or step and
+never warn.
+
+## Intent [#intent]
+
+What this pack does with each value of [intent](/docs/reference/intent), from
+the pack itself. A value the pack does not apply, **Not yet** or **Not
+applicable**, is a [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) warning
+where it is written, with the reason below. **Elsewhere** values are applied by
+another pack or step and never warn. **Release** is the Databricks feature's
+own release level, as the linked page said on the date checked; with
+`features: ga`, a feature in preview is a
+[`PREVIEW_FEATURE`](/docs/reference/errors#PREVIEW_FEATURE) warning.
+
+
+
+Its checks, cleansing and quarantine are in
+[Quality and cleansing](#quality-and-cleansing).
+
+## Quality and cleansing [#quality-and-cleansing]
+
+What this pack does with each check, action and cleansing operation, from the
+pack itself. Each row links the Databricks page it was checked against, with
+the date; the pack's
+[sources](https://github.com/ingestron/pack-databricks/blob/main/spec/references.md#quality-and-cleansing)
+list every page it writes SQL from.
+
+
+
+- **Where checks run.** Each check is a constraint on the table's own SQL
+ file, in every layer: `EXPECT` for `warn`, `ON VIOLATION DROP ROW` for
+ `drop` and `ON VIOLATION FAIL UPDATE` for `fail`. In a layer whose `does`
+ leaves out `check`, such as `raw`, every action is `warn`, so the checks only
+ count. The pipeline's update shows each one's failing rows.
+- **Table checks** run in the `apply_contract` task after each refresh, only
+ when the platform turns on `databricks.manage.checks`; otherwise each is
+ [`CAPABILITY_OFF`](/docs/reference/errors#CAPABILITY_OFF). See
+ [Platform prerequisites](#platform-prerequisites).
+- **Quarantine** is Databricks' quarantine pattern: `
_quarantine` is
+ built from the same rows, read once, with `_failed_checks`, an array of the
+ checks each row failed.
+- **Results** are the pipeline's event log, published as a table.
+- **Cleansing** is written in the `SELECT` that builds the table's rows; a
+ transform's query is wrapped so cleansing never changes it.
+- **Deduplicate** happens only in a materialised view, with `QUALIFY`. On a
+ streaming table it is
+ [`DEDUPLICATE_MODE`](/docs/reference/errors#DEDUPLICATE_MODE): an error when
+ the table appends, and a warning when its history already keeps one row per
+ key.
+- **Lakeflow Connect** writes the table as the source gives it: its row
+ checks, their actions, quarantine, cleansing and deduplicate are not
+ applied, and the hint says to check and cleanse it in a transform in the
+ next layer. Its table checks still run in the `apply_contract` task, and the
+ event log still keeps the results.
+
+## Access [#access]
+
+What this pack does with each role, grant mode, mask and row filter, from the
+pack itself. Each row links the Databricks page it was checked against, with
+the date; the pack's
+[sources](https://github.com/ingestron/pack-databricks/blob/main/spec/references.md#access)
+list every page it writes SQL from. To set them up, see
+[Grant access, mask columns and filter rows](/docs/how-to/grant-access).
+
+
+
+- **Switches.** Grants need `databricks.manage.grants`, masks
+ `databricks.manage.masks` and row filters `databricks.manage.filters`, each
+ with what [Platform prerequisites](#platform-prerequisites) lists. Masks
+ and row filters also need `databricks.manage.tags`. While a switch is off,
+ each role, mask or filter it would apply is
+ [`CAPABILITY_OFF`](/docs/reference/errors#CAPABILITY_OFF), and the files
+ are what they were without it.
+- **Grants** are `GRANT SELECT ON TABLE`, and `MODIFY` too for a role with
+ `access: write`, made after each refresh. A materialised view takes no
+ `MODIFY`, so `write` there is
+ [not applied here](/docs/concepts/not-applied). Each principal is written
+ in backticks, with a backtick in it doubled. With
+ `databricks.manage.schemas`, each principal also gets `USE_SCHEMA` on the
+ schemas the bundle creates.
+- **Exact grants** read `SHOW GRANTS ON TABLE` and revoke what the table's
+ roles do not give. They revoke only privileges granted directly on the
+ project's own tables, and never ownership, a denial, a privilege the table
+ inherits from its schema or catalog, the `run_as` identity's or those of
+ `databricks.governance.unmasked`.
+- **Masks and row filters** are Unity Catalog policies:
+ `ingestron_mask_` for each masked classification and
+ `ingestron_filter_` for each filter. With `databricks.manage.schemas`
+ on, each is on the schema, which the bundle creates; otherwise each is on
+ the project's own tables, so a shared schema's policies are never changed.
+ Each applies to `account users` except the `run_as` identity and the
+ principals in `databricks.governance.unmasked`. Exempting `run_as` means
+ the pipelines never read masked or filtered data.
+- **Mask functions.** `null`, `hash` and `redact` are functions the task
+ creates in `databricks.governance.functions`, `ingestron_mask_null`,
+ `ingestron_mask_hash` and `ingestron_mask_redact`, only for the masks a
+ table uses. A function of your own named in one part is in that schema too;
+ one named in three parts is used as written.
+- **Stale policies.** After making the policies, the task drops each
+ `ingestron_mask_` or `ingestron_filter_` policy on the project's schemas and
+ tables that it no longer makes, for a switch that is on. Every other policy
+ stays.
+- **Order.** The task sets tags and comments, creates the mask functions,
+ makes the policies and drops the stale ones, and only then grants and
+ revokes, then runs the table checks. If a policy cannot be made, the task
+ fails before it grants anything, so no role it grants reads real values.
+- **Landed files.** Grants, masks and filters apply to the tables Databricks
+ keeps. A table that ends at the landed files, even one another pack serves,
+ gets nothing here, and the landed files themselves keep the values in
+ full: set their access on the storage account.
+- **Limits.** A schema may have 100 policies, a table 50, and a policy may
+ exempt 20 principals; more is
+ [`POLICY_LIMIT`](/docs/reference/errors#POLICY_LIMIT) or
+ [`POLICY_PRINCIPALS`](/docs/reference/errors#POLICY_PRINCIPALS). A table
+ may have one row filter
+ ([`FILTER_MULTIPLE`](/docs/reference/errors#FILTER_MULTIPLE)). Readers need
+ serverless compute or Databricks Runtime 16.4 or later.
diff --git a/content/docs/(reference)/reference/packs/duckdb.mdx b/content/docs/(reference)/reference/packs/duckdb.mdx
index be4c0cc..86729c6 100644
--- a/content/docs/(reference)/reference/packs/duckdb.mdx
+++ b/content/docs/(reference)/reference/packs/duckdb.mdx
@@ -19,9 +19,73 @@ folder with the DuckDB command line; generation itself runs nothing.
## Contract fields
-What this pack does with each field of a table's contract. When a contract
-writes a field that no pack of the standard uses yet, Ingestron warns at that
-field with the reason below; **Elsewhere** fields are handled by another pack
-or step and never warn.
+What this pack does with each field of a table's contract. A field is judged
+for each table, against the pack that keeps it. When a contract writes a field
+this pack does not apply yet, on a table this pack keeps, Ingestron warns at
+that field with [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) and the
+reason below; **Elsewhere** fields are handled by another pack or step and
+never warn.
+
+## Intent [#intent]
+
+What this pack does with each value of [intent](/docs/reference/intent), from
+the pack itself. A value the pack does not apply, **Not yet** or
+**Not applicable**, is a [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED)
+warning where it is written, with the reason below. **Elsewhere** values are
+applied by another pack or step and never warn.
+
+
+
+Its checks, cleansing and quarantine are in
+[Quality and cleansing](#quality-and-cleansing).
+
+## Quality and cleansing [#quality-and-cleansing]
+
+What this pack does with each check, action and cleansing operation, from the
+pack itself. Each row links the DuckDB page it was checked against, with the
+date; the pack's
+[sources](https://github.com/ingestron/pack-duckdb/blob/main/spec/references.md)
+list every page it writes SQL from.
+
+
+
+- **Where checks run.** `run.sql` checks a table's rows only in a layer whose
+ `does` lists `check`, and in a `raw` layer counts them, each as `warn`. In
+ any other layer it runs none, and its SQL is what it was before checks; each
+ rule is [not applied here](/docs/concepts/not-applied).
+- **When.** Each table's rows are cast and cleansed, then counted against
+ every check before the table is built, in the run's one transaction. The
+ DuckDB command line prints each check with its failing rows. A failing
+ `fail` check calls `error()`, so `-bail` stops the run and nothing is
+ published. Table checks are counted once the table is built, with nothing to
+ turn on in the platform: the pack has no `checks` capability, so
+ `manage.checks` changes nothing here.
+- **Quarantine and results** stay in the database and are not published:
+ `.
_quarantine`, rebuilt each run (or added to, for `append`),
+ and `._ingestron_checks`, one row per check and run. In Azure each
+ run starts with an empty database, so they last only for that run, and the
+ job's log shows the counts.
+- **Deduplicate** applies to a table replaced each run. An `upsert` table, or
+ one kept at `history: latest`, already keeps one row per key, so
+ `deduplicate` there is a
+ [`DEDUPLICATE_MODE`](/docs/reference/errors#DEDUPLICATE_MODE) warning, and
+ on an `append` table it is an error.
+- **Time zones** come from DuckDB's ICU extension, which its command line
+ includes. A column typed with a time zone, such as `TIMESTAMPTZ`, cannot be
+ converted. Around a clock change, a wall time in the hour skipped in spring
+ is read in the offset before the change, and one in the hour repeated in
+ autumn is read as the later of the two: in London on 25 October 2026,
+ `01:30` is `01:30` UTC.
+
+## Access [#access]
+
+DuckDB has no users, roles or permissions, so the pack applies no grant, mask
+or row filter, and its files are what they would be without them. Each role,
+grant mode, mask and row filter on a table DuckDB keeps is
+[not applied here](/docs/concepts/not-applied). When the files DuckDB
+publishes are protected another way, such as on the storage account,
+[accept the warning](/docs/how-to/grant-access#accept-a-gap).
+
+
diff --git a/content/docs/(reference)/reference/packs/synapse.mdx b/content/docs/(reference)/reference/packs/synapse.mdx
index 676f092..bc090bf 100644
--- a/content/docs/(reference)/reference/packs/synapse.mdx
+++ b/content/docs/(reference)/reference/packs/synapse.mdx
@@ -12,17 +12,132 @@ pack itself.
For a task walkthrough, use [Synapse serverless views](/docs/platforms/synapse).
The pack writes one `views.sql` for every environment, with
`environments/.sql` setting each environment's values; run that
-script with sqlcmd when the definitions change. See
+script with sqlcmd when the definitions change. Where a table's layer checks
+its rows, it also writes `checks.sql`, with `checks/.sql` for each
+environment; run that after each landing to count the checks. See
[What Ingestron writes](/docs/platforms/synapse#what-you-get). Generation itself
creates nothing in Azure.
+## Platform configuration
+
+### The `synapse:` section [#platform]
+
+What only Synapse reads goes in the platform's `synapse:` section. Its one
+switch, `synapse.manage.grants`, lets `views.sql` grant each view to the
+roles that may read it; see [Access](#access).
+
+
+
+While it is off, a role on a view is a
+[`CAPABILITY_OFF`](/docs/reference/errors#CAPABILITY_OFF) warning, with a
+quick fix that turns it on, and `views.sql` is what it was without it.
+`manage` at the top of the platform is Databricks' and turns nothing on for
+Synapse.
+
+
+
## Contract fields
-What this pack does with each field of a table's contract. When a contract
-writes a field that no pack of the standard uses yet, Ingestron warns at that
-field with the reason below; **Elsewhere** fields are handled by another pack
-or step and never warn.
+What this pack does with each field of a table's contract. A field is judged
+for each table, against the pack that keeps it. When a contract writes a field
+this pack does not apply yet, on a table this pack keeps, Ingestron warns at
+that field with [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) and the
+reason below; **Elsewhere** fields are handled by another pack or step and
+never warn.
+
+## Intent [#intent]
+
+What this pack does with each value of [intent](/docs/reference/intent), from
+the pack itself. A view keeps nothing, so most intent is not applicable. A value the pack does not apply, **Not yet** or
+**Not applicable**, is a [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED)
+warning where it is written, with the reason below. **Elsewhere** values are
+applied by another pack or step and never warn.
+
+
+
+Its checks, cleansing and quarantine are in
+[Quality and cleansing](#quality-and-cleansing).
+
+## Quality and cleansing [#quality-and-cleansing]
+
+What this pack does with each check, action and cleansing operation, from the
+pack itself. Each row links the Microsoft page it was checked against, with
+the date; the pack's
+[sources](https://github.com/ingestron/pack-synapse/blob/main/spec/references.md#quality-and-cleansing)
+list every page it writes T-SQL from. The pack cannot run T-SQL in its tests,
+so its SQL is checked against those pages.
+
+
+
+- **Where checks run.** A table runs its checks only in a layer whose `does`
+ lists `check`, and in a `raw` layer counts them, each as `warn`. In any other
+ layer its view is what it was before checks, and each rule is
+ [not applied here](/docs/concepts/not-applied).
+- **The view** leaves out the rows a `drop` check fails, each time it is
+ queried; `warn` and `fail` checks keep their rows. A view cannot stop a
+ load, so `fail` is not applied.
+- **`checks.sql`** counts every check: row checks over the files, the rows the
+ view leaves out included, and table checks over the rows the view keeps. It
+ returns one row per check, then raises an error for the first `fail` check
+ that fails. It checks nothing unless something runs it after each landing;
+ see [Run and operate](/docs/platforms/synapse#operate). Table checks need
+ nothing turned on: the pack has no `checks` capability.
+- **Text comparisons** ignore spaces at the end of a value, as T-SQL does:
+ `'GB '` equals `'GB'`. With the standard's `trim` first this makes no
+ difference.
+- **Deduplicate** keeps the last row per primary key in the view, numbered
+ with `ROW_NUMBER()`. A view keeps nothing, so it is the only way a view gives
+ one row per key.
+
+### Time zones [#time-zones]
+
+T-SQL's `AT TIME ZONE` names zones by their Windows names, so the pack maps
+each IANA name to one, with a table generated from the Unicode CLDR's
+`windowsZones.xml` and its zone aliases (release 49, read 10 November 2026).
+`Europe/London` is `GMT Standard Time`. Windows has no zone for these, so a
+conversion from or to one of them is not applied, and the column stays in the
+zone it was in:
+
+
+
+Windows keeps less of each zone's history than the IANA database: times from
+before a zone's rules changed can convert differently here than on Databricks
+and DuckDB. In the hour repeated in autumn, T-SQL gives a wall time the offset
+before the change: in London on 25 October 2026, `01:30` is `00:30` UTC,
+where DuckDB gives `01:30`. In the hour skipped in spring, it moves the time
+forward by the gap.
+
+## Access [#access]
+
+What this pack does with each role, grant mode, mask and row filter, from the
+pack itself. Each row links the Microsoft page it was checked against, with
+the date; the pack's
+[sources](https://github.com/ingestron/pack-synapse/blob/main/spec/references.md#access)
+list every page it writes T-SQL from. To set them up, see
+[Grant access, mask columns and filter rows](/docs/how-to/grant-access#on-synapse).
+
+
+
+- **Grants** are `GRANT SELECT ON OBJECT::[].[]` to each role's
+ principal, after each view in `views.sql`, in the view's own database. Each
+ principal is in brackets, with `]` doubled. A view is read-only, so a
+ role's `write` is [not applied here](/docs/concepts/not-applied).
+- **Exact grants** revoke, after a view's grants, each permission granted
+ directly on the view that its roles do not give. They keep ownership,
+ permissions on the schema or database, denials, permissions given
+ `WITH GRANT OPTION`, which only `CASCADE` could revoke, and those of the
+ database user that runs `views.sql` (`USER_NAME()`), which SQL Server
+ cannot revoke from itself. A group that user belongs to is not kept.
+- **Served views.** A view that `synapse-serve` writes over a table another
+ pack keeps is granted the same way, and judged by this pack a second time:
+ a mask or row filter on its table is not applied on the view.
+- **Masks and row filters** are not applied: serverless SQL pools have no data
+ masking or row-level security, and readers of views over files can read
+ the files directly.
+- **Readers' rights.** `ingestron request` lists each principal as a database
+ user, with the rights views over files need. Those rights also let it read
+ the files under the data source directly, not only through the views.
diff --git a/content/docs/(reference)/reference/templates.mdx b/content/docs/(reference)/reference/templates.mdx
index a855f3f..32ddde8 100644
--- a/content/docs/(reference)/reference/templates.mdx
+++ b/content/docs/(reference)/reference/templates.mdx
@@ -7,7 +7,11 @@ A template is a project folder with a `template.yaml`. `ingestron init` copies
the folder into a new project, writes its `README.md`, and asks nothing. A
starter is a template that ships with Ingestron: a complete project for one
common situation, with `REPLACE_ME` where you write your own values. See [Share templates and standards](/docs/how-to/share-a-template)
-to make a template of your own.
+to make a template of your own. A
+[published standard](/docs/how-to/choose-a-published-standard), which
+`--kit` names, is a dated standard for one platform, with a project that shows
+it; its `kit.yaml` is described in
+[Share a published standard](/docs/how-to/share-a-template#share-a-published-standard).
## Create a project from a starter [#create-a-project-from-a-starter]
diff --git a/content/docs/(reference)/reference/versions.mdx b/content/docs/(reference)/reference/versions.mdx
index 2f64233..f65460b 100644
--- a/content/docs/(reference)/reference/versions.mdx
+++ b/content/docs/(reference)/reference/versions.mdx
@@ -29,5 +29,5 @@ Connector maturity, such as Public Preview or Beta, is shown on each
The [deployment templates](/docs/how-to/deploy-through-environments) pin the
tools their stages run: `@microsoft/azure-data-factory-utilities` 1.0.3, the
Databricks CLI v1.23.0, and sqlcmd (Go) v1.10.0. Their check stage installs
-`ingestron@2.2.0`, and each GitHub action is pinned to a commit, with its
+`ingestron@2.3.0`, and each GitHub action is pinned to a commit, with its
version beside it.
diff --git a/content/docs/(reference)/tools/vscode.mdx b/content/docs/(reference)/tools/vscode.mdx
index f971a0a..a00248e 100644
--- a/content/docs/(reference)/tools/vscode.mdx
+++ b/content/docs/(reference)/tools/vscode.mdx
@@ -46,7 +46,11 @@ valid. Problems are for the environment in the status bar; click it, or run
**Ingestron: Select environment**, to choose another. Problems that generating
finds, such as a value supplied when deploying that cannot be a name
(`DEPLOY_VALUE`), show too, and a pack's notice (`PACK_NOTICE`) is a warning
-on the first line of `project.yaml` in every environment.
+on the first line of `project.yaml` in every environment. A feature the engine
+that keeps a table cannot apply is a warning at the line that asks for it
+(`NOT_APPLIED`; see [Not applied here](/docs/concepts/not-applied)). A warning
+that the platform's `accept` list accepts is not shown; see
+[Accept a warning](/docs/how-to/accept-a-warning).
A definition that uses `${envvar:NAME}` reads the variable from `.env.`
and `.env` in the project folder, and from the environment VS Code was started
@@ -112,7 +116,8 @@ CLI prints the first fix under the problem, as `Fix: …`.
When the same problem has a fix in several places, the menu also offers
**Fix all N … problems**, which applies each one's first fix in one change; an
-edit several of them share, such as turning on `manage.checks`, is made once.
+edit several of them share, such as turning on `databricks.manage.checks`, is
+made once.
A method's setting written outside the file its method says it belongs in,
such as a table's `properties` in a flow or `integration_runtime` in a
@@ -164,13 +169,6 @@ marked **fixed by the standard**, and marks the others **not allowed**. If a
definition sets a fixed value anyway, the `FIXED_SETTING` problem has a quick
fix, **Remove this setting**, so the standard's value applies.
-
-
It also fills in what the project already decides:
- **A new, empty file** offers a document to start from: the project
@@ -209,20 +207,21 @@ it resolves to:
Turn these off with VS Code's `editor.inlayHints.enabled` and
`editor.codeLens` settings.
-
-
## Why is this value what it is?
Hover a flow table or a transform to see its full name, the method for each
step, and every setting with the layer that set it: the pack's default, the
standard, the platform, a rule, the flow or transform, the table itself, or a
setting the standard fixes, with the file and line. Hover a setting under
-`with` for that one value.
+`with` for that one value. The same hover lists the table's
+[intent](/docs/reference/intent), each value with where it came from, such as
+`history`: `latest`, from the table's layer in the standard
+(`standard.yaml:18`).
+
+Hover a value that is still `REPLACE_ME` to see the field it fills, what the
+field is, where to find the value, and a suggestion of what it looks like,
+such as ``. A value in `project.yaml`'s `values` is explained
+by the field that reads it.
Hover any field or setting to see what it is, what each allowed value means,
its default, an example and where it is usually set; hover a value for that
@@ -232,13 +231,6 @@ docs, and each problem's code in the Problems panel links to its entry in
`ingestron.docs.url` setting; set it to a local build, such as
`http://localhost:3000`, to read the docs offline.
-
-
## Project view
The **Ingestron** tab in the secondary side bar, on the right, opens with a
@@ -402,29 +394,28 @@ workspace.
## Start a project
-**Ingestron: New project** starts from a connection or from a starter.
+**Ingestron: New project** starts from a connection, a starter or a published
+standard.
- **From a connection** asks for a source category, connection, network
reachability and supported route. You can search across all sources or
within a category.
- **From a starter** offers the ready-made projects, such as **DuckDB on a
laptop**, which [runs locally with DuckDB](/docs/platforms/duckdb).
+- **From a published standard** offers the dated standards installed with the
+ extension, such as **Local medallion on DuckDB**, and makes the same project
+ as `ingestron init --kit`; see
+ [Start from a published standard](/docs/how-to/start-from-a-published-standard).
Nothing else is asked. The destination folder comes last, and its name is the
-project's id; the environments are `dev` and `prod`, or the starter's, and the
-standard is the route's or the starter's own. The new `README.md` opens in a
+project's id; the environments are `dev` and `prod`, or the starter's or the
+published standard's, and the standard is the route's, the starter's or the
+published one. The new `README.md` opens in a
preview, and a message says how many values are still `REPLACE_ME`: the
README lists each with where to find it. When the folder is outside the
workspace, the message offers to open it or add it to the workspace. See
[Create a project](/docs/how-to/start-from-a-template).
-
-
## Check files with the YAML extension [#check-files-with-the-yaml-extension]
Without the Ingestron extension, the
diff --git a/content/docs/(reference)/troubleshooting.mdx b/content/docs/(reference)/troubleshooting.mdx
index 7c94195..ec3510d 100644
--- a/content/docs/(reference)/troubleshooting.mdx
+++ b/content/docs/(reference)/troubleshooting.mdx
@@ -192,6 +192,33 @@ setting nobody gave.
**Fix:** correct the name at the reported line, or add the missing setting,
usually in the standard or the platform.
+### Plan lists warnings under `Not applied on this platform` [#not-applied]
+
+**Symptom:** every `plan`, `apply` and `check` prints warnings under their own
+heading, such as the [first project's](/docs/start/first-project):
+
+```text
+Not applied on this platform
+dev:
+contracts/orders.yaml:11:59 warning NOT_APPLIED
+ The notNull rule isn't applied on duckdb: it is applied only in a layer whose does lists check.
+ Add `check` to the layer's `does` to apply it here.
+```
+
+**Cause:** the engine that keeps a table cannot apply something the
+definitions ask for, here a contract's required column: DuckDB and Synapse
+check rows only in a layer whose `does` lists `check`. A warning never stops a
+run; the files are written without that feature. Projects written before
+layers had duties show these after an upgrade, with the same files as before.
+
+**Check:** the message's reason, and the pack's
+[Quality and cleansing](/docs/reference/packs/duckdb#quality-and-cleansing).
+
+**Fix:** follow the hint, such as adding `check` to the layer's `does`; move
+the feature where it is applied, or remove it; or, once you have decided to
+live with it, [accept it](/docs/how-to/accept-a-warning). See
+[Not applied here](/docs/concepts/not-applied#what-to-do).
+
### The method cannot read this source
**Symptom:**
@@ -544,8 +571,9 @@ flows/erp.yaml:6:13 error METHOD_CHECK
under the same name every load would never be read again. For the same reason,
a file you re-export under its old name is skipped.
-**Fix:** use `replace` or `upsert`, or give each load its own file, such as
-with `increments`. To reload everything, delete `ingestron.duckdb` and run
+**Fix:** keep the table with `history: none` (the method's `replace`) or
+`history: latest` (`upsert`) instead of `mode: append`, or give each load its
+own file, such as with `increments`. To reload everything, delete `ingestron.duckdb` and run
again. See [Keep a table current](/docs/how-to/keep-a-table-current#on-duckdb).
### Upsert needs a primary key
@@ -557,8 +585,9 @@ flows/erp.yaml:6:13 error METHOD_CHECK
"duckdb-table" for table "invoices": upsert needs the contract's primaryKey.
```
-**Cause:** `upsert` replaces rows with the same key, and the contract names no
-key. `sequence_by` has to be a column of the contract too.
+**Cause:** `history: latest`, the method's `upsert`, replaces rows with the
+same key, and the contract names no key. `sequence` has to be a column of the
+contract too.
**Fix:** mark the key column `primaryKey: true` in the contract. See
[Keep a table current](/docs/how-to/keep-a-table-current).
@@ -578,6 +607,30 @@ building publishes nothing.
duckdb -bail ingestron.duckdb -c ".read build/duckdb/dev/run.sql"
```
+### A check stops the run [#a-check-stops-the-run]
+
+**Symptom:** the run prints its checks, each with its failing rows, then stops
+with:
+
+```text
+Invalid Input Error: The check "" failed for rows of
, so nothing is published.
+```
+
+or, for a check over the whole table,
+`The table check "" failed on
, so nothing is published.`
+
+**Cause:** a check whose action is `fail` found rows that fail it. The run
+stops before it inserts them: with `-bail`, nothing is published and no table
+changes.
+
+**Check:** the table the run printed just before the error lists each check,
+its action and its `failing_rows`. Query the files the table reads for the
+rows the check describes.
+
+**Fix:** correct the data and run again. If such rows should be kept and
+counted, or left out, change the check's action to `warn` or `drop`; see
+[Check data quality](/docs/how-to/add-quality-checks).
+
### Publishing fails after the tables are built
**Symptom:** the run reaches the step that publishes and fails there, or no
@@ -649,12 +702,13 @@ flows/crm.yaml:9:5 error METHOD_CHECK
or `mode upsert needs primary key columns in the contract`.
-**Cause:** every `mode` but `append` keeps a table current by key, in the order
+**Cause:** `history: latest` or `full`, which the method writes as
+`mode: upsert` or `mode: history`, keeps a table current by key, in the order
of a column.
-**Fix:** set `sequence_by` to the column that orders a key's changes, and mark
-the key columns `primaryKey: true` in the contract. See
-[Keep a table current](/docs/how-to/keep-a-table-current).
+**Fix:** set `sequence` on the flow or the table to the column that orders a
+key's changes, and mark the key columns `primaryKey: true` in the contract.
+See [Keep a table current](/docs/how-to/keep-a-table-current).
### Databricks rejects the bundle
@@ -667,7 +721,7 @@ them, or see
A Lakeflow Connect pipeline refers to a Unity Catalog connection, which
Ingestron does not create.
-**Fix:** when the platform sets `manage.schemas`, an administrator runs the
+**Fix:** when the platform sets `databricks.manage.schemas`, an administrator runs the
generated `prerequisites/.sql` once before the first deploy: it grants
`USE CATALOG` and `CREATE SCHEMA` on each catalog to the identity the pipelines
run as, and the same must be given to the identity that deploys when it
diff --git a/content/docs/concepts/environments-and-platform.mdx b/content/docs/concepts/environments-and-platform.mdx
index 100a715..30c870a 100644
--- a/content/docs/concepts/environments-and-platform.mdx
+++ b/content/docs/concepts/environments-and-platform.mdx
@@ -38,11 +38,21 @@ an ADF method. A project still has one Platform document when it uses several
packs. The method reference says which fields it consumes and what must already
exist before deployment.
-Workspace, compute, identity, scheduling and `manage` fields mean something
-specific to each platform, and not every platform uses each one. See
-[Databricks platform configuration](/docs/reference/packs/databricks#platform-configuration)
+`schedule`, `tags` and `alerts` sit at the top of the platform, for every
+pack that runs jobs: when they run, how they are tagged, and who is told when
+a run fails. What only one platform reads goes in that pack's own section,
+under its id: the Databricks workspace, identity, compute, permissions and
+`manage` switches are under `databricks:`, and Synapse's grants switch under
+`synapse:`. A section for a pack the standard
+does not name is [`UNKNOWN_PACK`](/docs/reference/errors#UNKNOWN_PACK). See
+[the `databricks:` section](/docs/reference/packs/databricks#platform)
and [ADF runtime and Key Vault](/docs/platforms/adf#runtime-and-key-vault) for
-their effects.
+their effects. The platform's `accept` list holds the warnings the platform
+team has decided to live with, each with its reason; see
+[Accept a warning](/docs/how-to/accept-a-warning). Its `principals` say which
+user, group or service principal each contract role is, often per environment
+through a value; see
+[Grant access, mask columns and filter rows](/docs/how-to/grant-access#roles).
## One set of files for every environment [#one-set-of-files]
diff --git a/content/docs/concepts/layers.mdx b/content/docs/concepts/layers.mdx
index 17ae5be..4d30324 100644
--- a/content/docs/concepts/layers.mdx
+++ b/content/docs/concepts/layers.mdx
@@ -1,15 +1,16 @@
---
title: Layers
-description: The stages a project's tables are grouped in, where each table lands, and what its layer changes.
+description: The stages a project's tables are grouped in, where each table lands, what its layer changes, what each layer does with its rows, and who reads them.
---
A layer is a named stage that a table belongs to. A project has one ordered
list of layers. Without `layers` in the standard, the list is ,
the default of every pack.
-A layer changes no data by itself. Ingestron uses a table's layer for its
-names, for the standard's rules, and to decide where a table lands when its
-definition does not say. A [transform](/docs/how-to/write-a-transform) is
+Ingestron uses a table's layer for its names, for the standard's rules, and to
+decide where a table lands when its definition does not say. A layer can also
+say what it does with its rows: keep them as they arrived, or cleanse,
+deduplicate and check them. A [transform](/docs/how-to/write-a-transform) is
what reads tables and writes a new one; it can read tables in any layer and
write its table to any layer.
@@ -52,12 +53,56 @@ use `{{ layer }}` anywhere; see
**Rules.** A rule in the standard can match `layer`, to choose a method or
settings for the tables in that layer. This standard makes every `gold`
-table a materialised view, with no clustering and a longer log retention:
+table a materialised view, with a longer log retention:
-`region_sales` is the only `gold` table, so only its view has clustering
-off and the longer retention. `store_sales` keeps the pack's settings.
+`region_sales` is the only `gold` table, so only its view has the longer
+retention. `store_sales` keeps the pack's settings.
+
+## What a layer does [#what-a-layer-does]
+
+Write a layer as an object to give it `does`, what the layer does with its
+rows, and defaults for the tables in it, such as `history` and `quality`; see
+[Intent](/docs/reference/intent#where). This is the standard of the
+[local medallion](/docs/how-to/kits/duckdb-local-medallion) published standard:
+
+
+
+`does` is `raw`, or a list of duties:
+
+| `does` | The layer's tables |
+| ------------- | ------------------------------------------------------------------------------------------------------------------ |
+| `raw` | Keep every row as it arrived. Nothing is cleansed, and checks only count: every action is `warn`. |
+| `cleanse` | Get the standard's `cleanse` defaults, and may cleanse their own columns. |
+| `deduplicate` | May keep one row per key with `cleanse.deduplicate`. |
+| `check` | Run their checks, each with its action: `warn`, `drop` or `fail`. |
+| `quarantine` | May keep the rows a `drop` check leaves out, with `quality.quarantine`, and get the standard's `quarantine: true`. |
+
+A table asks only for what its layer does. Writing a duty in a table whose
+layer's `does` leaves it out, such as `cleanse.columns` on a table in a `raw`
+layer, or an expectation's `drop` in a layer that does not `check`, is
+[`LAYER_RESPONSIBILITY`](/docs/reference/errors#LAYER_RESPONSIBILITY), an
+error that names the layers that do it. In a layer whose `does` leaves out
+`check`, every action is `warn`.
+
+A layer without `does` asks nothing of its tables, so a project written before
+layers had duties generates the same files. Its flow tables still get the
+standard's `cleanse` defaults; a transform's table does not. Databricks checks
+rows in every layer. DuckDB and Synapse check rows only where `does` lists
+`check` or is `raw`; elsewhere each rule is
+[not applied here](/docs/concepts/not-applied).
+[Cleanse data in the right layer](/docs/how-to/cleanse-data) puts each duty in
+its layer, and [Check data quality](/docs/how-to/add-quality-checks#layers)
+says what each engine does.
+
+## Who reads a layer [#access]
+
+A layer written as an object can also say who may use its tables, with
+`access`: `contract: false` grants none of the contracts' roles there, such
+as on raw rows, and `roles` are granted on every table in the layer, beside
+the contracts'. See
+[Grant access, mask columns and filter rows](/docs/how-to/grant-access#roles).
## Name your own layers
diff --git a/content/docs/concepts/meta.json b/content/docs/concepts/meta.json
index 6bae42d..8ee50b7 100644
--- a/content/docs/concepts/meta.json
+++ b/content/docs/concepts/meta.json
@@ -7,6 +7,7 @@
"standards-and-packs",
"settings",
"layers",
+ "not-applied",
"environments-and-platform",
"yaml-references",
"generated-files"
diff --git a/content/docs/concepts/not-applied.mdx b/content/docs/concepts/not-applied.mdx
new file mode 100644
index 0000000..9c6bd6e
--- /dev/null
+++ b/content/docs/concepts/not-applied.mdx
@@ -0,0 +1,133 @@
+---
+title: Not applied here
+description: What it means when the engine that keeps a table cannot apply something your definitions ask for, where Ingestron shows it, and what to do about it.
+---
+
+Your definitions say what each table should have: a check and its action, a
+cleansing operation, a quarantine, a way of keeping history, who may read it.
+The engine that
+keeps the table does the work, in its own SQL. When that engine cannot do one
+of these things, Ingestron says so at the line that asks for it, with the
+engine's reason. That state is **not applied here**, the warning
+[`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED).
+
+It is always a warning. The files are still written and run; the table goes
+without that feature on that engine. The same definitions can be applied
+in full on one engine and not on another.
+
+## What it covers [#what-it-covers]
+
+Each table is judged against the pack that keeps it: its store step, or its
+ingest step when that step writes the table or the table ends at the landed
+files. Another pack in the same standard does not count. Not applied can be:
+
+- a contract rule's kind, such as `pattern` on Synapse, which has no regular
+ expressions; an action, such as `fail` on a Synapse view; or a table check;
+- a cleansing operation, `deduplicate`, or a time zone conversion, including
+ the [zones Synapse cannot convert](/docs/reference/packs/synapse#quality-and-cleansing);
+- quarantine, or the results of the checks;
+- a value of [intent](/docs/reference/intent#support), such as `load.mode` on a
+ method that reads the source itself;
+- a contract field the pack does not apply yet;
+- a role's grant, its `write`, exact grants, a mask or a row filter, such as
+ any of them on DuckDB, which has no access control, or a mask on a Synapse
+ view; a table one pack keeps and another serves is judged by both;
+- anything that would check or cleanse a table that ends at the landed files,
+ since nothing reads those files before your reports do.
+
+What each pack applies, and why it does not, is on its page under
+**Quality and cleansing**, **Access** and **Intent**:
+[Databricks](/docs/reference/packs/databricks#quality-and-cleansing) (its
+[access](/docs/reference/packs/databricks#access)),
+[DuckDB](/docs/reference/packs/duckdb#quality-and-cleansing)
+([access](/docs/reference/packs/duckdb#access)),
+[Synapse](/docs/reference/packs/synapse#quality-and-cleansing)
+([access](/docs/reference/packs/synapse#access)) and
+[Azure Data Factory](/docs/reference/packs/adf#quality-and-cleansing)
+([access](/docs/reference/packs/adf#access)).
+
+## Where it shows [#where-it-shows]
+
+**In `check`, `plan` and `apply`,** under its own heading,
+`Not applied on this platform`, after the other problems and warnings,
+grouped by the environments that have it. This is what the
+[first project](/docs/start/first-project) prints:
+
+```text
+Not applied on this platform
+dev:
+contracts/orders.yaml:11:59 warning NOT_APPLIED
+ The notNull rule isn't applied on duckdb: it is applied only in a layer whose does lists check.
+ Add `check` to the layer's `does` to apply it here.
+```
+
+Each message says what is not applied, on which pack, and why. A warning never
+changes the exit code, so `check` in CI still passes.
+
+**In VS Code,** as a warning at the same line, with the same message. A
+warning the platform [accepts](/docs/how-to/accept-a-warning) is not shown.
+
+**In a data product's passport,** in its Quality section: each check the
+engine does not apply is listed with its rule in words, then `Not enforced:`
+and the reason, and each cleansing operation it does not apply says
+`is not applied:` and the reason. Its Who may use it section says
+`Not enforced on :` and the reason for each role, mask or filter the
+engine does not apply; see
+[Read the passport](/docs/how-to/define-a-data-product#read-the-passport).
+The passport says this whether or not the warning is accepted: accepting a
+warning quiets it, and does not make the check run or the role granted.
+
+## A layer that does not check [#layers]
+
+On DuckDB and Synapse the most common case is the one above: a table in a
+layer that does not say it checks. These engines run checks only in a layer
+whose `does` lists `check`, or is `raw`, where checks only count. The hint
+says what to add. Write each layer as an object with what it does:
+
+```yaml
+layers:
+ - { name: bronze, does: raw } # every row as it arrived; checks only count
+ - { name: silver, does: [check] }
+```
+
+With these layers the first project's plan lists no warning, and `run.sql`
+counts the orders that have no `order_id`. Databricks checks rows in every
+layer, so it never gives this hint; see
+[Say which layers check](/docs/how-to/add-quality-checks#layers) and
+[What a layer does](/docs/concepts/layers#what-a-layer-does).
+
+## How it differs from other warnings [#other-warnings]
+
+- **Most warnings** say something in your definitions could be better: a
+ setting has moved ([`SETTING_MOVED`](/docs/reference/errors#SETTING_MOVED)),
+ `deduplicate` is redundant ([`DEDUPLICATE_MODE`](/docs/reference/errors#DEDUPLICATE_MODE)),
+ or quarantine has nothing to keep
+ ([`QUARANTINE_UNUSED`](/docs/reference/errors#QUARANTINE_UNUSED)). Not
+ applied says your definitions are fine and this engine cannot do what they
+ ask, so it is listed apart.
+- **[`CAPABILITY_OFF`](/docs/reference/errors#CAPABILITY_OFF)** says the
+ engine can do it, but the platform has not turned it on, such as Databricks
+ table checks without `databricks.manage.checks`, tags without
+ `databricks.manage.tags`, or a contract's roles without
+ `databricks.manage.grants` or `synapse.manage.grants`. Its fix turns it on
+ in `platform.yaml`, and the hint says what it needs. Not applied has no
+ switch to turn.
+- **[`QUALITY_RULE_NOT_CHECKED`](/docs/reference/errors#QUALITY_RULE_NOT_CHECKED)**
+ says no engine could check a rule as it is written, such as a `custom` rule
+ or one with no comparison.
+- **[`PREVIEW_FEATURE`](/docs/reference/errors#PREVIEW_FEATURE)** says the
+ feature is applied, but it is in preview on the platform, and the standard
+ asks for generally available features only.
+
+## What to do [#what-to-do]
+
+Read the reason, then choose one:
+
+- **Move it where it is applied.** Add the duty to the layer's `does`, as the
+ hint says; check a table that Lakeflow Connect writes in a transform in the
+ next layer; give a table that ends at the landed files a store step.
+- **Choose another method or pack** that applies it, if your platform has one.
+- **Remove it,** when the table can do without it.
+- **Accept it,** when you have looked at it and decided to live with it, with
+ the reason in `platform.yaml`; see
+ [Accept a warning](/docs/how-to/accept-a-warning).
diff --git a/content/docs/concepts/settings.mdx b/content/docs/concepts/settings.mdx
index 0174ca9..c6c7381 100644
--- a/content/docs/concepts/settings.mdx
+++ b/content/docs/concepts/settings.mdx
@@ -37,6 +37,22 @@ missing setting is reported with the file and line where it was written
([`BAD_SETTING`](/docs/reference/errors#BAD_SETTING)). Each method's
settings and defaults are in [Reference: packs and methods](/docs/reference/packs).
+## Intent
+
+[Intent](/docs/reference/intent), such as `history` or `audit`, has an order
+of its own: the standard, then the table's layer, a `load.when` rule, the flow
+or transform, and the table, with the standard's `fixed` intent last. What is
+still unset comes from the chosen method's own default. See
+[Which value wins](/docs/reference/intent#precedence).
+
+Some settings are other names for intent, such as `streaming-table`'s
+`mode: upsert`, which is `history: latest`. Written as a setting, one still
+works, in the place it is written, and `plan` warns with
+[`SETTING_MOVED`](/docs/reference/errors#SETTING_MOVED). In a flow's or a flow
+table's `with`, its fix writes the intent there instead; anywhere else, such
+as the standard's settings, write the intent by hand; see
+[Settings that are intent](/docs/reference/intent#settings-that-are-intent).
+
## Names, layers, types and quality
These are not method settings, and `fixed` does not apply to them. For each,
@@ -48,8 +64,10 @@ a later place wins:
transform picks one of them with `layer`.
- **Types:** the pack's type for each logical type, the standard's `types`,
then a column's `physicalType`.
-- **Quality:** the pack's default actions, the standard's `quality`, then an
- expectation's `action`.
+- **Quality:** the pack's default actions, the standard's `quality`, the
+ table's layer's `quality`, then an expectation's `action`. In a layer whose
+ `does` leaves out `check`, every action is then `warn`; see
+ [Check data quality](/docs/how-to/add-quality-checks#layers).
The pack is the standard's first pack. A name pattern in the platform
document replaces the standard's pattern for that name, and the standard's
diff --git a/content/docs/concepts/standards-and-packs.mdx b/content/docs/concepts/standards-and-packs.mdx
index 854f3b4..3fe67dc 100644
--- a/content/docs/concepts/standards-and-packs.mdx
+++ b/content/docs/concepts/standards-and-packs.mdx
@@ -31,9 +31,31 @@ This example uses a pack for copying and another for downstream storage:
-[Choose where your data goes](/docs/start/choose-a-route) lists the routes and
-the packs each one uses; [Packs and methods](/docs/reference/packs) lists each
-pack's methods and capabilities.
+Its `layers` and `load` are intent, below. [Choose where your data
+goes](/docs/start/choose-a-route) lists the routes and the packs each one
+uses; [Packs and methods](/docs/reference/packs) lists each pack's methods and
+capabilities.
+
+## Intent: what to keep, not how [#intent]
+
+Besides choosing methods, a standard can say how data is loaded, kept and run
+in words that belong to no platform: load only new files, keep the latest row
+per key, add a column saying when a row was loaded, try a failed run twice
+more. That is **intent**. Each pack turns it into its own platform's terms:
+`history: latest` is AUTO CDC on Databricks and `INSERT OR REPLACE` on DuckDB.
+
+Methods are chosen first, as below; intent changes what a chosen method
+writes, never which method is chosen, unless a `when` rule matches on it. A
+value the chosen pack does not apply is
+[not applied here](/docs/concepts/not-applied), a warning with the pack's
+reason, so a standard never promises what a project does not build. So is a
+check or cleansing operation the engine that keeps a table cannot apply. Each pack's
+reference page lists what it does with every value, and the
+[intent reference](/docs/reference/intent) lists the words, where each can be
+written and which value wins.
+
+A [published standard](/docs/how-to/choose-a-published-standard) is a
+standard of this kind for one platform, with a small project that builds.
## How a method is chosen
diff --git a/content/docs/how-to/(build)/add-a-source.mdx b/content/docs/how-to/(build)/add-a-source.mdx
index 610ab7d..fd074e4 100644
--- a/content/docs/how-to/(build)/add-a-source.mdx
+++ b/content/docs/how-to/(build)/add-a-source.mdx
@@ -67,6 +67,14 @@ and apply it again first.
```
```text
+ Not applied on this platform
+ dev:
+ contracts/orders.yaml:11:59 warning NOT_APPLIED
+ The notNull rule isn't applied on duckdb: it is applied only in a layer whose does lists check.
+ Add `check` to the layer's `does` to apply it here.
+ contracts/shop.customers.yaml:12:80 warning NOT_APPLIED
+ The notNull rule isn't applied on duckdb: it is applied only in a layer whose does lists check.
+ Add `check` to the layer's `does` to apply it here.
duckdb → build/duckdb (dev)
update dev/run.sql
create dev/tables/bronze.customers.sql
@@ -75,8 +83,10 @@ and apply it again first.
```
The new table lands in `bronze`, the [first layer](/docs/concepts/layers),
- like `orders`. Apply,
- then run `run.sql` again to build it.
+ like `orders`. Its required `customer_id` is not checked there, as
+ `order_id` is not: the starter's layers do not say they check rows; see
+ [Not applied here](/docs/concepts/not-applied). Apply, then run `run.sql`
+ again to build it.
Every table in a flow reads the flow's connection with the same ingest method.
A table can change that method's settings with its own `ingest.with`.
@@ -111,6 +121,11 @@ Each source is a connection, in `connections/`, and the flows that read it, in
"duckdb-files" cannot read the "postgresql" connector.
No pack in the standard reads postgresql.
Fix: Add the "adf" pack and use "adf-table" for this flow.
+ Not applied on this platform
+ dev:
+ contracts/orders.yaml:11:59 warning NOT_APPLIED
+ The notNull rule isn't applied on duckdb: it is applied only in a layer whose does lists check.
+ Add `check` to the layer's `does` to apply it here.
```
VS Code offers the same fix, and each other one, from the light bulb.
diff --git a/content/docs/how-to/(build)/add-an-environment.mdx b/content/docs/how-to/(build)/add-an-environment.mdx
index 21a4a7a..8b9f245 100644
--- a/content/docs/how-to/(build)/add-an-environment.mdx
+++ b/content/docs/how-to/(build)/add-an-environment.mdx
@@ -42,8 +42,8 @@ Other platforms add their own file or folder for it; see
Only an environment named `dev` is in development mode; every other one,
`test` included, is in production mode, and schedules run only in `prod` or
-`production`, so `test`'s refresh job is paused. A platform can set `mode` and
-`schedule.paused` itself; see the
+`production`, so `test`'s refresh job is paused. A platform can set
+`databricks.mode` and `schedule.paused` itself; see the
[platform defaults](/docs/reference/packs/databricks#platform-configuration).
Environments may differ only in values, such as names, paths and hosts: a table
built in one environment only is refused as
@@ -72,6 +72,7 @@ For the example above, it prints:
- `mode`: `development` → `production`
- `workspace.root_path`: `/Workspace/Shared/.bundle/finance/dev` → `/Workspace/Shared/.bundle/finance/test`
- `tags.environment`: `dev` → `test`
+- `sections.databricks.workspace.root_path`: `/Workspace/Shared/.bundle/finance/dev` → `/Workspace/Shared/.bundle/finance/test`
## Table journals
@@ -83,7 +84,9 @@ For the example above, it prints:
place, and lists only what differs:
- **Platform:** the platform document's values, and the defaults that follow
- from the environment, such as `mode`. Compared with `prod`, it also shows
+ from the environment, such as `mode`. The values of a pack's own section,
+ such as `databricks:`, show twice: as the pack reads them, such as
+ `workspace.root_path`, and as written, under `sections.databricks`. Compared with `prod`, it also shows
`schedule.paused`, because only `prod` runs its schedule.
- **Names:** the pipeline and job names, which are not per table.
- **Table:** one section for each table that differs: its full name, each
diff --git a/content/docs/how-to/(build)/add-quality-checks.mdx b/content/docs/how-to/(build)/add-quality-checks.mdx
index 626e834..d92a482 100644
--- a/content/docs/how-to/(build)/add-quality-checks.mdx
+++ b/content/docs/how-to/(build)/add-quality-checks.mdx
@@ -1,57 +1,229 @@
---
title: Check data quality
-description: Count, drop or stop on rows that break a rule.
+description: Count, leave out or stop on rows that break a rule, in the layers that check, on Databricks, DuckDB and Synapse.
---
-Checks run where the platform builds the table, and not every platform checks
-rows yet:
+A table's checks come from its contract and from its flow or transform. Each
+check has an action: count the rows that fail it, leave them out, or stop. The
+engine that keeps the table runs the checks, in the layers whose `does` says
+they check.
-| Where the table is built | What happens to the checks |
-| --------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- |
-| [Databricks](/docs/platforms/databricks) | Runs them, as this guide describes. |
-| [DuckDB](/docs/platforms/duckdb), on a laptop or in Azure | Checks no rows yet. An `expectations` entry is an error (`METHOD_CHECK`): remove it. |
-| [Synapse serverless views](/docs/platforms/synapse) | A view checks no rows. `expectations` are not run, and required columns are reported as not used (`CONTRACT_FIELD_UNUSED`). |
-| [Land files only](/docs/platforms/landing-only) | Nothing reads the files, so each expectation is reported as not checked (`QUALITY_RULE_NOT_CHECKED`). |
+This guide uses the `quality` example, one contract's checks generated for
+three engines. Each of its environments builds the readings table on a
+different engine, so the same rules can be compared:
-The rest of this guide is about Databricks.
+
-Every column the contract marks `required: true` gets a check that it is not
-null. You add your own checks on the table with `expectations`:
+## Write the rules
+
+The contract says what every row must hold. Each required column, each limit
+in a column's `logicalTypeOptions`, and each `quality` rule that allows no
+failing row becomes a check of each row:
+
+
+
+That gives four checks: `sensor_id_not_null` and `read_at_not_null` from
+`required: true`, `sensor_id_pattern` from the `pattern` option, and
+`known_unit`. A check is named by the rule's `name`, else after the column and
+the option or metric. The table's own rule, `has_readings`, compares a count
+over all rows, so it is a **table check**, counted once the table is built;
+see [Rules about the data](/docs/how-to/write-a-contract#rules-about-the-data).
+
+A flow table or a transform adds checks of its own, `expectations`, each a SQL
+condition the rows must meet:
-Each check is a SQL condition rows must meet. `action` says what happens to
-rows that fail:
+An expectation is written in the SQL of the engine that keeps the table. This
+one reads the same in Databricks, DuckDB and T-SQL; the contract's rules are
+written for each engine by its pack.
+
+## Choose what a failing row does
-| `action` | Effect |
-| -------- | ---------------------------------- |
-| `warn` | keep the row and count the failure |
-| `drop` | leave the row out and count it |
-| `fail` | stop the update |
+`action` says what happens to a row that fails a check:
-Without an `action`, the standard decides. Its `quality` sets the action for
-required columns and the default for your own checks:
+| `action` | Effect |
+| -------- | ----------------------------------------------- |
+| `warn` | keep the row and count it |
+| `drop` | leave the row out and count it |
+| `fail` | stop: the update or the run fails, and says why |
+
+Without an `action`, the standard decides: `quality.required` for a required
+column, `quality.default` for every other check, `warn` unless they say
+otherwise. A layer written as an object can give its own `quality`, which wins
+for its tables. A table check fails with the default action when it is
+`fail`, and otherwise warns, since no row can be left out once the table is
+built.
+## Say which layers check [#layers]
+
+A layer's `does` says what it does with its rows; see
+[What a layer does](/docs/concepts/layers#what-a-layer-does):
+
+- A layer whose `does` lists `check` runs its tables' checks, each with its
+ action.
+- `does: raw` keeps every row as it arrived. Its checks only count: every
+ action there is `warn`.
+- A list without `check`, such as `[cleanse]`, makes every action `warn` too.
+ Databricks counts the checks there; DuckDB and Synapse do not run them. An
+ expectation that asks for `drop` or `fail` there is
+ [`LAYER_RESPONSIBILITY`](/docs/reference/errors#LAYER_RESPONSIBILITY), an
+ error.
+- A layer without `does` asks nothing of its tables, and Databricks checks
+ them as it always has.
+
+DuckDB and Synapse run checks only in a layer whose `does` lists `check` or is
+`raw`. Anywhere else each contract rule is
+[`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED), with the hint to add
+`check` to the layer's `does`, and on DuckDB a flow's own `expectations` are an
+error, [`METHOD_CHECK`](/docs/reference/errors#METHOD_CHECK).
+
+In the example, `bronze` does `check` and `quarantine`, so every engine checks
+the readings there.
+
+## What each engine does [#engines]
+
+Every engine writes the checks into the files it generates, as plain SQL. What
+it can apply differs:
+
+
+
+What an engine cannot apply is a
+[not applied here](/docs/concepts/not-applied) warning at the line that asks
+for it. Plan the example:
+
```sh
-ingestron plan
+ingestron plan --no-diff
```
-Run **Ingestron: Plan** from the Command Palette (⇧⌘P on macOS, Ctrl+Shift+P elsewhere). The output is in the Ingestron output panel. Problems show in the Problems panel as you type, before you plan.
+Run **Ingestron: Plan** from the Command Palette (⇧⌘P on macOS, Ctrl+Shift+P
+elsewhere). The output is in the Ingestron output panel. Each warning is also
+marked at its line as you type, for the environment selected in the status
+bar: select `synapse` to see these.
-The checks become constraints on the table:
-
-
+The plan lists them first, for the `synapse` environment:
+
+```text
+Not applied on this platform
+synapse:
+contracts/readings.yaml:14:40 warning NOT_APPLIED
+ The pattern rule isn't applied on synapse: serverless T-SQL has no regular expressions.
+flows/readings.yaml:11:87 warning NOT_APPLIED
+ The fail action isn't applied on synapse: a view can't stop a load; checks.sql checks this rule and raises an error when it runs.
+standard.yaml:12:15 warning NOT_APPLIED
+ Quarantine isn't applied on synapse: a view keeps no rows to quarantine; checks.sql returns how many rows each drop check leaves out instead.
+standard.yaml:13:12 warning NOT_APPLIED
+ quality.results: true isn't applied on synapse: a view keeps no results; checks.sql returns each check's counts instead when it runs.
+```
-Tables written by Lakeflow Connect cannot run your own checks; put them on a
-transform that reads the table.
+Each is a warning: the files are still written. Remove what an engine cannot
+apply, or keep it and, once you have decided to live with it,
+[accept the warning](/docs/how-to/accept-a-warning).
+
+### On Databricks [#databricks]
+
+Each check is a constraint on the table: `EXPECT` for `warn`,
+`ON VIOLATION DROP ROW` for `drop` and `ON VIOLATION FAIL UPDATE` for `fail`.
+The pipeline's update shows how many rows failed each one:
+
+
+
+Table checks run in a task after each refresh, `tasks/iot/apply_contract.py`,
+only when the platform turns on `databricks.manage.checks`, as the example's
+platform does:
+
+
+
+Without it, each table check is
+[`CAPABILITY_OFF`](/docs/reference/errors#CAPABILITY_OFF), a warning whose fix
+turns it on. A table that Lakeflow Connect writes is written as the source
+gives it: its row checks, their actions and quarantine are `NOT_APPLIED`
+there, and the hint says to check the table in a transform in the next layer.
+Its table checks still run in the task after each refresh, and the event log
+still keeps the results.
+
+### On DuckDB [#duckdb]
+
+`run.sql` counts every check before it builds the table, in the run's one
+transaction. The DuckDB command line prints each check's failing rows, a
+`drop` check's rows are not inserted, and a failing `fail` check stops the run
+before anything is published. Table checks are counted once the table is
+built, with nothing to turn on in the platform:
+
+
+
+### On Synapse serverless [#synapse]
+
+A view keeps no rows, so the checks are split in two. The view leaves out the
+rows a `drop` check fails each time it is queried:
+
+
+
+`build/synapse/checks.sql` counts every check, the rows the view leaves out
+included, returns one row per check, and raises an error when a `fail` check
+fails. It checks nothing unless something runs it after each load: run it with
+sqlcmd, from a pipeline's Script activity, or in CI; see
+[Synapse serverless views](/docs/platforms/synapse#operate).
+
+
+
+## Keep the rows a check leaves out [#quarantine]
+
+`quality.quarantine: true` keeps the rows a `drop` check leaves out in a table
+of their own, `
_quarantine`, with a column `_failed_checks` naming the
+checks each row failed. Written in the standard, it reaches the tables of the
+layers whose `does` lists `quarantine`. Written on a layer, a flow table or a
+transform, that layer's `does` must list `quarantine`
+([`LAYER_RESPONSIBILITY`](/docs/reference/errors#LAYER_RESPONSIBILITY)), and a
+table with no `drop` check has nothing to keep
+([`QUARANTINE_UNUSED`](/docs/reference/errors#QUARANTINE_UNUSED)).
+
+Databricks builds `readings_quarantine` beside the table, from the same rows,
+read once. DuckDB rebuilds `bronze.readings_quarantine` each run, or adds to it
+for an `append` table, and does not publish it. A Synapse view has no rows to
+keep.
+
+## Keep the results [#results]
+
+`quality.results: true` keeps each check's outcome where it can be queried:
+
+- **Databricks:** the pipeline's event log, published as a table.
+- **DuckDB:** `._ingestron_checks` in the run's database, a row per
+ check and run, with the run's id, the time, the action, the failing rows and
+ whether it passed. In Azure the database lasts only for that run, so read
+ the job's log instead.
+- **Synapse:** nothing keeps them. Each run of `checks.sql` returns them.
+
+A [data product's passport](/docs/how-to/define-a-data-product#read-the-passport)
+says for each table what checks it, when, and where its results are.
diff --git a/content/docs/how-to/(build)/check-in-ci.mdx b/content/docs/how-to/(build)/check-in-ci.mdx
index 8b449e5..3b3a371 100644
--- a/content/docs/how-to/(build)/check-in-ci.mdx
+++ b/content/docs/how-to/(build)/check-in-ci.mdx
@@ -8,7 +8,7 @@ from the definitions project folder on every pull request: it fails when
someone changed the definitions without generating, so the files you deploy
are always what the definitions say. It needs only Node.js 22.13 or later:
each template installs an exact version of the command from npm,
-`npm install -g ingestron@2.2.0`, then runs it. Change the version when you
+`npm install -g ingestron@2.3.0`, then runs it. Change the version when you
upgrade, as
[Upgrade Ingestron and its packs](/docs/how-to/upgrade#choose-the-version-you-run)
shows, so a new release never fails a pull request that changed nothing.
diff --git a/content/docs/how-to/(build)/cleanse-data.mdx b/content/docs/how-to/(build)/cleanse-data.mdx
new file mode 100644
index 0000000..853627c
--- /dev/null
+++ b/content/docs/how-to/(build)/cleanse-data.mdx
@@ -0,0 +1,258 @@
+---
+title: Cleanse data in the right layer
+description: Keep raw rows in bronze, and trim, map and default text and bring timestamps to UTC in silver, with a transform that needs no code.
+---
+
+The standard says what each layer does with its rows: bronze keeps them as they
+arrived, silver cleanses and checks them. You then write the cleansing once,
+and it happens in the layer that is responsible for it. This guide starts from
+the [local medallion](/docs/how-to/kits/duckdb-local-medallion) published
+standard, which runs on your laptop with DuckDB, and adds a silver table of
+cleansed orders without writing SQL.
+
+You need Ingestron and the
+[DuckDB command line](https://duckdb.org/docs/installation/), as for
+[your first project](/docs/start/first-project#before-you-start).
+
+## Create the project
+
+```sh
+ingestron init shop --kit duckdb/local-medallion@2026-11-10
+cd shop
+```
+
+The remaining steps run in the `shop` folder.
+
+## What each layer does
+
+The standard's `layers` give each layer its duties, in `does`:
+
+
+
+- `bronze` does `raw`. Its tables keep every row as the file had it, so you can
+ always see what arrived. Nothing there is cleansed, and its checks only
+ count.
+- `silver` does `cleanse`, `deduplicate`, `check` and `quarantine`. Its tables
+ are cleansed, then checked, and the rows a check leaves out are kept in a
+ quarantine table.
+
+A flow's table lands in the first layer and a transform's in the second, so
+the orders file becomes `bronze.orders`, and each transform that reads it
+builds a `silver` table. [What a layer does](/docs/concepts/layers#what-a-layer-does)
+lists each duty.
+
+## What every silver table gets
+
+The standard's `cleanse` gives defaults to every table in a layer that
+cleanses:
+
+
+
+- `text`: each text column, one whose contract `logicalType` is `string`, has
+ the spaces at either end removed, and an empty text becomes null.
+- `timestamps`: each timestamp column is converted to UTC, from the time zone
+ its source writes in. A connection without `timezone` writes UTC.
+
+Only `silver` gets them; `bronze` keeps its rows as they arrived.
+
+## Add the columns to cleanse
+
+The shop's till also writes the time of each order, in London time, and the
+channel it came through, typed by hand. Replace `samples/orders.csv` with:
+
+```csv
+order_id,customer,amount,ordered_at,channel
+1,Ana,25.50,2026-10-01 09:15:00,web
+2, Ben,12.00,2026-10-01 18:40:00,Shop
+3,Ana,40.25,2026-10-02 12:05:00,WEB
+4,Chen,8.75,2026-10-03 08:30:00,
+5,Ben,30.00,2026-10-03 21:10:00,store
+```
+
+In `contracts/orders.yaml`, make `ordered_at` a timestamp and add `channel`,
+so its columns end:
+
+{/* prettier-ignore */}
+```yaml
+ - { name: ordered_at, logicalType: timestamp }
+ - { name: channel, logicalType: string }
+```
+
+Then say which zone the till writes in. Add this line to
+`connections/shop.yaml`, under `connectivity`:
+
+```yaml
+timezone: Europe/London # the till writes local time
+```
+
+A zone is `UTC` or a name from the IANA time zone database, written exactly;
+any other name is [`TIMEZONE_NAME`](/docs/reference/errors#TIMEZONE_NAME).
+
+## Add a silver table without code
+
+A transform with one input can leave out `code`: it reads every column of its
+input, and keeps the columns its contract names. Save this as
+`transforms/orders_clean.yaml`:
+
+```yaml
+apiVersion: ingestron.io/v1
+kind: Transform
+id: orders_clean
+description: Each order, cleansed.
+inputs: [orders]
+contract: shop.orders
+cleanse:
+ columns:
+ channel: { map: { Shop: store }, case: lower, default: unknown }
+```
+
+`orders_clean` lands in `silver`, so it gets the standard's defaults, and
+`cleanse.columns` adds its own for `channel`. A column's operations run in a
+fixed order, after the column is cast to its contract's type: `timezone`,
+`trim`, `empty`, `map`, `case`, then `default`. So `channel` is trimmed, an
+empty one becomes null, `Shop` becomes `store`, every value is put in lower
+case, and a channel with no value becomes `unknown`. A column the contract
+does not have is [`CLEANSE_COLUMN`](/docs/reference/errors#CLEANSE_COLUMN),
+and an operation that does not fit the column's type, such as `case` on a
+number, is [`CLEANSE_TYPE`](/docs/reference/errors#CLEANSE_TYPE).
+
+A transform with more than one input needs its code:
+[`TRANSFORM_CODE`](/docs/reference/errors#TRANSFORM_CODE).
+
+## Plan, apply and run
+
+```sh
+ingestron plan --no-diff
+```
+
+```text
+duckdb → build/duckdb (dev)
+create dev/README.md
+create dev/run.sql
+create dev/tables/bronze.orders.sql
+create dev/tables/silver.customer_totals.sql
+create dev/tables/silver.orders_clean.sql
+
+5 files: 5 create. Ready to apply.
+```
+
+No warning: DuckDB applies every operation in `silver`. Apply, and build the
+tables:
+
+```sh
+ingestron apply
+duckdb -bail ingestron.duckdb -c ".read build/duckdb/dev/run.sql"
+```
+
+The run prints each table's checks, with the rows that fail each one; none
+fails here. Compare the orders as they arrived with the cleansed ones:
+
+```sh
+duckdb ingestron.duckdb -c "SELECT order_id, customer, ordered_at, channel FROM bronze.orders ORDER BY order_id"
+duckdb ingestron.duckdb -c "SELECT order_id, customer, ordered_at, channel FROM silver.orders_clean ORDER BY order_id"
+```
+
+```text
+┌──────────┬──────────┬─────────────────────┬─────────┐
+│ order_id │ customer │ ordered_at │ channel │
+│ int64 │ varchar │ timestamp │ varchar │
+├──────────┼──────────┼─────────────────────┼─────────┤
+│ 1 │ Ana │ 2026-10-01 09:15:00 │ web │
+│ 2 │ Ben │ 2026-10-01 18:40:00 │ Shop │
+│ 3 │ Ana │ 2026-10-02 12:05:00 │ WEB │
+│ 4 │ Chen │ 2026-10-03 08:30:00 │ NULL │
+│ 5 │ Ben │ 2026-10-03 21:10:00 │ store │
+└──────────┴──────────┴─────────────────────┴─────────┘
+┌──────────┬──────────┬─────────────────────┬─────────┐
+│ order_id │ customer │ ordered_at │ channel │
+│ int64 │ varchar │ timestamp │ varchar │
+├──────────┼──────────┼─────────────────────┼─────────┤
+│ 1 │ Ana │ 2026-10-01 08:15:00 │ web │
+│ 2 │ Ben │ 2026-10-01 17:40:00 │ store │
+│ 3 │ Ana │ 2026-10-02 11:05:00 │ web │
+│ 4 │ Chen │ 2026-10-03 07:30:00 │ unknown │
+│ 5 │ Ben │ 2026-10-03 20:10:00 │ store │
+└──────────┴──────────┴─────────────────────┴─────────┘
+```
+
+Bronze keeps `" Ben"`, `Shop` and the London times as the till wrote them.
+Silver has trimmed the name, mapped and lowered each channel, given order 4 a
+channel, and moved each time an hour back: London is an hour ahead of UTC in
+October. Around a clock change, the hour that is skipped or repeated converts
+as each engine documents; see the pack's
+[Quality and cleansing](/docs/reference/packs/duckdb#quality-and-cleansing).
+
+`ordered_at` in `silver.orders_clean` is now in UTC, and Ingestron knows it: a
+table that reads it with the same rule converts nothing again.
+
+## Ask a layer for a duty it does not do
+
+Bronze does not cleanse, so a table there that asks for cleansing is refused.
+Replace `flows/orders.yaml` with:
+
+```yaml
+apiVersion: ingestron.io/v1
+kind: Flow
+id: orders
+source: shop
+tables:
+ orders:
+ object: orders.csv
+ contract: shop.orders
+ cleanse:
+ columns:
+ channel: { case: lower }
+```
+
+```sh
+ingestron plan --no-diff
+```
+
+```text
+dev: the definitions have problems
+flows/orders.yaml:11:9 error LAYER_RESPONSIBILITY
+ The layer "bronze" does not list cleanse in its does, so cleansing cannot be set here.
+ Layers that do cleanse: silver. Or add `cleanse` to this layer's `does`.
+```
+
+Nothing is written. Cleanse in the transform that builds the silver table, as
+above, and put `flows/orders.yaml` back:
+
+```yaml
+apiVersion: ingestron.io/v1
+kind: Flow
+id: orders
+source: shop
+tables:
+ orders: { object: orders.csv, contract: shop.orders }
+```
+
+## Keep one row per key
+
+`silver` also does `deduplicate`. A table there can keep one row per primary
+key, the last by the columns you name:
+
+```yaml
+cleanse:
+ deduplicate: { order_by: [ordered_at] }
+```
+
+`orders_clean` does not need it: the standard keeps `silver` tables at
+`history: latest`, which already keeps one row per `order_id`, and plan would
+warn that deduplicate is redundant
+([`DEDUPLICATE_MODE`](/docs/reference/errors#DEDUPLICATE_MODE)). Use it for a
+table that is replaced each run, such as one with `history: none`. A table
+whose contract has no primary key cannot deduplicate:
+[`DEDUPLICATE_KEY`](/docs/reference/errors#DEDUPLICATE_KEY).
+
+## On other engines
+
+The same YAML cleanses on Databricks and Synapse: each pack writes the
+operations in its own SQL, in the table's `SELECT` or the view's. What each
+applies, and the time zones Synapse cannot convert, are in each pack's
+**Quality and cleansing**:
+[Databricks](/docs/reference/packs/databricks#quality-and-cleansing),
+[DuckDB](/docs/reference/packs/duckdb#quality-and-cleansing) and
+[Synapse](/docs/reference/packs/synapse#quality-and-cleansing). A table that
+ends at the landed files is not cleansed: each operation there is
+[not applied](/docs/concepts/not-applied).
diff --git a/content/docs/how-to/(build)/define-a-data-product.mdx b/content/docs/how-to/(build)/define-a-data-product.mdx
index b8c1c37..2dbf407 100644
--- a/content/docs/how-to/(build)/define-a-data-product.mdx
+++ b/content/docs/how-to/(build)/define-a-data-product.mdx
@@ -26,7 +26,9 @@ one owner. Other ODPS fields, such as `inputPorts`, are kept as documentation.
What a product promises about its columns stays in the ODCS contract:
`classification` on each column, `roles` for who may use the data, and
-`slaProperties` for freshness.
+`slaProperties` for freshness. Ingestron can grant those roles, and mask
+columns by their classification; see
+[Grant access, mask columns and filter rows](/docs/how-to/grant-access).
@@ -43,6 +45,11 @@ The standard's `governance` section says what every product must give:
- `access.sensitive.require`: for a contract with a sensitive column, the
fields each of its `roles` must give.
+The same section can also say how access is generated, for every table, not
+only those a product publishes: `access.grants`, `masks` and `filters`; see
+[Grant access, mask columns and filter rows](/docs/how-to/grant-access).
+They check nothing here.
+
Only contracts a product publishes are governed. A broken rule is an error,
so `ingestron check` fails in CI and the pull request is stopped. Remove
`classification: confidential` from `email` in the example to see it:
@@ -61,14 +68,35 @@ a `README.md` listing every product, and a page per product. It says who owns
the product and what it is for, which table it is in this environment and
where it lives, its columns, with the contract's type for each (such as
`integer` or `timestamp`, also for columns the landed files add) and sensitive ones
-marked, its quality checks,
+marked, its quality checks and cleansing,
how fresh it is, who may use it, its lineage as a diagram, and the governance
rules it meets:
+Its Quality section says, for each table, which engine checks it, when, and
+where the results are, then each check in words with what a failing row does.
+A check the engine does not apply says `Not enforced` and why, even when the
+platform [accepts the warning](/docs/how-to/accept-a-warning); see
+[Not applied here](/docs/concepts/not-applied). Here `customers` is in
+`bronze`, which keeps raw rows, so its check only counts.
+
Passports are Markdown with Mermaid diagrams, so GitHub and Azure DevOps show
them as pages. They are generated with the pipelines, so they are always up to
date, and `ingestron check` fails when one is stale.
-Access is shown as the contract declares it. Ingestron does not grant it yet.
+Its **Who may use it** section lists, for each table, each role granted on
+it, the role's principal in this environment, and whether each engine grants
+it. A role the engine cannot grant, or the platform has not turned on, says
+`Not enforced` and why. Here DuckDB has no access control, so the role of
+`customers` is not enforced; the example's `platform.yaml` accepts that
+warning, and the passport still says so. The section also gives the grant
+mode, and each masked column and row filter. See
+[Grant access, mask columns and filter rows](/docs/how-to/grant-access).
+
+## Next
+
+- Have Ingestron grant the roles and mask the sensitive columns:
+ [Grant access, mask columns and filter rows](/docs/how-to/grant-access).
+- Live with a warning the engine cannot fix:
+ [Accept a warning](/docs/how-to/accept-a-warning).
diff --git a/content/docs/how-to/(build)/deploy-through-environments.mdx b/content/docs/how-to/(build)/deploy-through-environments.mdx
index 0952e03..4f34c5f 100644
--- a/content/docs/how-to/(build)/deploy-through-environments.mdx
+++ b/content/docs/how-to/(build)/deploy-through-environments.mdx
@@ -43,7 +43,7 @@ The tools are pinned: Microsoft's `@microsoft/azure-data-factory-utilities`
1.0.3, the Databricks CLI v1.23.0 and sqlcmd (Go) v1.10.0, each download
checked by its SHA-256 where the template downloads it. Each GitHub action is
pinned to a full commit, with its version beside it, and the check installs an
-exact Ingestron, `ingestron@2.2.0`: change them when you upgrade. The ADF
+exact Ingestron, `ingestron@2.3.0`: change them when you upgrade. The ADF
build fetches its build tool from `adf.azure.com`, so the build stage needs
network access there.
diff --git a/content/docs/how-to/(build)/grant-access.mdx b/content/docs/how-to/(build)/grant-access.mdx
new file mode 100644
index 0000000..2bbd964
--- /dev/null
+++ b/content/docs/how-to/(build)/grant-access.mdx
@@ -0,0 +1,339 @@
+---
+title: Grant access, mask columns and filter rows
+description: Say who may read each table, hide sensitive columns and filter rows, and have Ingestron grant, mask and filter on Databricks, grant on Synapse, and say what an engine cannot do.
+---
+
+Who may read a table belongs with its contract, and how sensitive columns
+and rows are hidden belongs in the standard. Ingestron turns both into the
+`GRANT`, `REVOKE` and policy statements the generated files run, so access
+changes in a pull request, as the tables do. Each part is off until the
+platform turns it on, and an engine that cannot apply one says so.
+
+This page uses the `governed-databricks` example: ADF copies a web shop's
+customers and orders, Databricks keeps the tables, and each table says who
+may read it.
+
+
+
+## Say who may read each table [#roles]
+
+A contract's `roles` name who may read the tables built from it. Each role
+reads (`access: read`, the default) or reads and changes the table
+(`access: write`):
+
+
+
+A layer can change who reads its tables. In the standard, a layer written as
+an object gives `access`: `contract: false` grants none of the contracts'
+roles on the layer's tables, and `roles` are granted on every table in the
+layer, beside the contracts':
+
+
+
+Here `bronze` holds the raw rows, so only `data-engineers` read them, and
+`marketing-analysts`, the role of the customers contract, is not granted on
+`bronze.customers`.
+
+A role is a name, not a group. `principals` in `platform.yaml` says which
+user, group or service principal each role is, usually per environment from
+`project.yaml`'s values:
+
+
+
+A role that `principals` does not list is granted under its own name. A
+role's `access` other than `read` or `write` is
+[`ROLE_ACCESS`](/docs/reference/errors#ROLE_ACCESS), an error where the role
+would be granted.
+
+## Turn grants on [#turn-on]
+
+Grants change who can read shared data, so the platform turns them on in its
+pack's section: `databricks.manage.grants` on Databricks and
+`synapse.manage.grants` on Synapse:
+
+
+
+While the switch is off, each role is a
+[`CAPABILITY_OFF`](/docs/reference/errors#CAPABILITY_OFF) warning that names
+its tables, with a fix that turns the switch on. Without `grants: true`, the
+example's plan warns, for one role:
+
+```text
+contracts/regional_sales.yaml:16:5 warning CAPABILITY_OFF
+ Access for the role "finance-analysts" in table "regional_sales" is not applied: the platform has Grants off.
+ Turn it on with databricks.manage.grants in platform.yaml. It needs: The run_as identity owns the tables the pipelines create, so it may grant and revoke on them; Each principal needs USE CATALOG, and USE SCHEMA unless the bundle manages the schemas; ingestron request lists them.
+ Fix: Turn on manage.grants.
+```
+
+With the switch off, the files grant nothing, as they did before the
+contracts named roles.
+
+With the switch on, the `apply_contract` task grants each table after each
+refresh, as the `run_as` identity, which owns the tables its pipelines
+create. Each role's principal gets `SELECT`, and `MODIFY` too for `write`;
+a principal that differs between environments is a value the job passes in:
+
+
+
+The platform's `databricks.grants` is a different thing: the privileges on
+the schemas the bundle creates, by layer. With `databricks.manage.schemas`
+on, the bundle also gives each principal granted a table `USE_SCHEMA` on its
+schema:
+
+
+
+## Choose a grant mode [#grant-mode]
+
+The standard's `governance.access.grants` says what happens to access given
+some other way:
+
+- **`additive`**, the default, grants what the contracts and layers name and
+ takes nothing away. Removing a role from a contract does not remove its
+ grant: revoke it by hand, once, on each table, such as
+ `` REVOKE SELECT ON TABLE `dev`.`silver`.`regional_sales` FROM `shop-finance-analysts-dev` ``
+ on Databricks.
+- **`exact`** also takes away every privilege granted directly on the
+ project's own tables that their roles do not give, each time the grants
+ run. A table whose layer and contract grant no role loses every direct
+ grant.
+
+
+
+Exact grants never take away:
+
+- ownership, or a denial;
+- a privilege granted on the schema or catalog, which the table inherits;
+- the `run_as` identity's privileges, or those of the principals in
+ `databricks.governance.unmasked`.
+
+Before you turn exact grants on, move any access given by hand into the
+contracts or the layers: the next refresh takes the rest away. The request
+says so, and lists the tables:
+
+
+
+## Mask sensitive columns [#masks]
+
+A column's `classification` in its contract says how sensitive it is. The
+standard's `governance.masks` gives a mask for each classification to hide:
+
+
+
+
+
+A mask is one of:
+
+- `null`: readers see no value. It works on a column of any type. Quote it,
+ `"null"`; written unquoted, YAML's null means the same.
+- `hash`: readers see a hash of the text, so equal values still match.
+- `redact`: readers see `****`.
+- the name of a function your team owns, in one part, which is in
+ `databricks.governance.functions`, or in three, `catalog.schema.function`.
+
+`hash` and `redact` work on text. On a column whose `logicalType` is not
+`string`, they are [`MASK_TYPE`](/docs/reference/errors#MASK_TYPE), an error:
+
+```text
+contracts/customers.yaml:11:77 error MASK_TYPE
+ The mask "hash" for "confidential" works on text, and "id" is integer in contract "shop.customers".
+ Mask "confidential" with "null" or a function of your own, or classify "id" differently.
+```
+
+On Databricks, masks need `databricks.manage.masks` and
+`databricks.manage.tags`: Ingestron sets each classification as a tag on its
+column, and the mask matches the tag. The platform's `databricks:` section
+also says where the mask functions are and who sees real values:
+
+
+
+- `governance.functions` is the schema, `catalog.schema`, where the task
+ creates the built-in masks, `ingestron_mask_null`, `ingestron_mask_hash`
+ and `ingestron_mask_redact`, and where a function named in one part is.
+ Without it, a mask that needs it is
+ [`MASK_FUNCTIONS`](/docs/reference/errors#MASK_FUNCTIONS), an error.
+- `governance.unmasked` lists the principals that see real values.
+- `run_as` always sees real values. The pipelines run as it, so they never
+ read masked data and write it into the next table. Without `run_as`, masks
+ are [`MASK_NEEDS_RUN_AS`](/docs/reference/errors#MASK_NEEDS_RUN_AS), an
+ error. Together with `unmasked`, at most 20 principals may see real values
+ ([`POLICY_PRINCIPALS`](/docs/reference/errors#POLICY_PRINCIPALS)); put
+ them in groups.
+
+Each masked classification is one Unity Catalog column mask policy,
+`ingestron_mask_`, for every account user except `run_as` and
+`unmasked`. The task makes it after the tags and before it grants anything,
+so if a policy cannot be made, such as when a function it uses does not
+exist yet, the task fails before any role it grants can read the real values:
+
+
+
+The tables Ingestron builds beside a table, such as `
_quarantine`,
+get the same tags and policies, so their copies hide the same values.
+
+Masks hide values in the tables, not in the files the tables are loaded
+from. In this example ADF lands the customers as files first, and anyone who
+can read the landing container reads `email` and `phone` in full. Ingestron
+does not set access to the landed files: give storage access only to those
+who may see real values, as
+[the ADF pack](/docs/reference/packs/adf#access) says.
+
+## Filter rows [#filters]
+
+A row filter decides which rows each reader sees, with a function your team
+owns that returns `BOOLEAN` for a column's value. The standard's
+`governance.filters` names each filter, its function and the column it is
+given:
+
+
+
+`match` finds the column on each table:
+
+- `{ tag: region }`: the column whose contract `tags` has the key `region`,
+ as every contract here does;
+- `{ tag: region, value: emea }`: the column with that tag key and value;
+- `{ classification: restricted }`: the column classified `restricted`.
+
+A filter applies to every table with a matching column, and to none without
+one. It may match at most one column of each table
+([`FILTER_MATCH`](/docs/reference/errors#FILTER_MATCH)), and a table may
+have one row filter
+([`FILTER_MULTIPLE`](/docs/reference/errors#FILTER_MULTIPLE)): on Databricks
+a query fails when two row filters apply to a table. Write one function that
+combines their conditions.
+
+On Databricks, filters need `databricks.manage.filters` and
+`databricks.manage.tags`, and use the same `run_as`, `governance.functions`
+and `unmasked` as masks. Each filter is one row filter policy,
+`ingestron_filter_`. Write the function itself as Databricks describes
+in
+[Create and manage row filter and column mask policies](https://docs.databricks.com/aws/en/data-governance/unity-catalog/abac/policies)
+(checked 11 November 2026), and create it before the first refresh: the
+request lists it.
+
+## Where the policies go [#policies]
+
+- **On each schema** when `databricks.manage.schemas` is on: the bundle
+ creates the schemas, so they are the project's own.
+- **On each table** of the project otherwise, so a schema other projects
+ share keeps its own policies.
+
+A schema may have 100 policies and a table 50; more is
+[`POLICY_LIMIT`](/docs/reference/errors#POLICY_LIMIT), an error. Readers need
+serverless compute or Databricks Runtime 16.4 or later: older runtimes cannot
+read a table with a policy.
+
+After making the policies, the task drops each `ingestron_mask_` and
+`ingestron_filter_` policy on the project's schemas and tables that it no
+longer makes, so removing a mask or a filter from the standard removes its
+policy. It drops only the policies of a switch that is on, and never another
+team's. So:
+
+- To remove every mask, remove `governance.masks` and refresh with
+ `databricks.manage.masks` still on. Turning the switch off leaves the
+ policies in place.
+- Turning `databricks.manage.schemas` off moves the policies to the tables,
+ and leaves those on the schemas. Drop them by hand, such as
+ `` DROP POLICY `ingestron_mask_confidential` ON SCHEMA `dev`.`bronze` ``.
+
+## On Synapse [#on-synapse]
+
+Synapse serverless grants each view, and cannot mask or filter. The
+[`synapse-views` example](/docs/platforms/synapse) turns grants on in its
+`synapse:` section, and grants `data-engineers` on `bronze` and
+`finance-analysts` on `customer_totals`:
+
+
+
+`views.sql` grants each view right after it, in the view's own database:
+
+
+
+A principal that differs between environments is a sqlcmd variable, which
+each environment's script sets:
+
+
+
+- A view is read-only, so a role's `write` grants `SELECT` only, with a
+ [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) warning.
+- Exact grants revoke, each time `views.sql` runs, each permission granted
+ directly on a view that its roles do not give. They keep ownership,
+ permissions on the schema or database, denials, permissions given
+ `WITH GRANT OPTION`, and those of the database user that runs `views.sql`.
+ A group that user belongs to is not kept, so give the user its rights
+ through ownership, `db_owner` or the schema or database, not through a
+ group granted on a view. In additive mode, revoke a removed role by hand, such as
+ `REVOKE SELECT ON OBJECT::[silver].[customer_totals] FROM [sales-finance-analysts-dev]`.
+- Each principal must be a database user, with the rights views over files
+ need. Those rights also let it read the files directly, not only through
+ the views; the request says so.
+- A mask or a row filter is not applied:
+
+ ```text
+ contracts/customers.yaml:12:60 warning NOT_APPLIED
+ The "hash" mask on "name" in table "customers" isn't applied on synapse: Synapse serverless has no data masking, and readers of views over files can read the files directly.
+ ```
+
+ The same holds for a view `synapse-serve` writes over a table Databricks
+ keeps: Databricks masks the table, but the view does not.
+
+## Accept a gap [#accept-a-gap]
+
+DuckDB has no users or permissions, so a role, mask or filter on a table
+DuckDB keeps is [not applied here](/docs/concepts/not-applied), as is a mask
+or filter on a Synapse view. When the data is protected another way, such as
+on the storage account, accept the warning with the reason. The [`data-products` example](/docs/how-to/define-a-data-product)
+does, for its role on DuckDB:
+
+
+
+`about: { feature: … }` narrows an entry to `grants`, `write`, `exact`,
+`masks` or `filters`. The same features narrow a
+[`CAPABILITY_OFF`](/docs/reference/errors#CAPABILITY_OFF) entry, when the
+platform team grants or masks some other way. See
+[Accept a warning](/docs/how-to/accept-a-warning).
+
+Accepting a warning does not grant anything. A data product's passport still
+says, under **Who may use it**, which roles are granted on each table, by
+which engine, and which are not enforced and why, with its masked columns
+and row filters; see
+[Read the passport](/docs/how-to/define-a-data-product#read-the-passport).
+
+## Ask the platform team [#request]
+
+Grants, masks and filters need things only an administrator can create.
+`ingestron request --env dev` lists them:
+
+
+
+The governed tags come first: an account administrator creates
+`classification`, with the standard's values, and each tag a filter matches.
+Run `prerequisites/.sql`, which the request ends with, once as an
+administrator: it gives `run_as` what it needs on the function schema and the
+team's functions, and `MANAGE` on the schemas, after the first deployment.
+See [Ask the platform team](/docs/how-to/ask-the-platform-team).
+
+## Next
+
+- Publish the tables as a data product, with who may use each:
+ [Define a data product](/docs/how-to/define-a-data-product).
+- Every switch, and what each pack applies:
+ [Databricks](/docs/reference/packs/databricks#access),
+ [Synapse](/docs/reference/packs/synapse#access).
+- The fields: [`governance`](/docs/reference/documents#standard-governance),
+ [`principals`](/docs/reference/documents#platform-principals) and a
+ [layer's `access`](/docs/reference/documents#standard-layers-item-access).
diff --git a/content/docs/how-to/(build)/keep-a-table-current.mdx b/content/docs/how-to/(build)/keep-a-table-current.mdx
index c417eb4..34a5bd2 100644
--- a/content/docs/how-to/(build)/keep-a-table-current.mdx
+++ b/content/docs/how-to/(build)/keep-a-table-current.mdx
@@ -4,25 +4,33 @@ description: Keep the current row for each key, or every version, from loads of
---
Use this when each load brings rows that change rows the table already has:
-a key, a column that orders the changes, and sometimes a column that says
-which rows are deletions. The store method's `mode` decides what the table
-keeps. Set it in the flow's `store.with` for every table of the flow, or in a
-table's own `store.with` for that table alone. These modes need the contract to
-mark the key with `primaryKey: true`; see
+a key, a column that orders the changes, and sometimes a condition that says
+which rows are deletions. Three words of [intent](/docs/reference/intent) say
+what the table keeps:
+
+- `history: latest` keeps the current row per key, and `history: full`
+ every version;
+- `sequence` names the column that orders a key's changes;
+- `load: { deletes: { when } }` is a SQL condition that is true for a row
+ deleted at the source.
+
+Write them on a flow for every table of the flow, on a table for that table
+alone, or `history` on a layer of the standard for every table in it. They
+need the contract to mark the key with `primaryKey: true`; see
[Describe a table with a contract](/docs/how-to/write-a-contract#keys-and-required-columns).
+What each engine does with them:
-| Engine | Current row per key | Every version | Deleted keys |
-| ---------- | ------------------- | ------------- | ------------- |
-| DuckDB | `upsert` | No | No |
-| Databricks | `upsert` | `history` | `delete_when` |
-| Synapse | No | No | No |
+
## On DuckDB
`duckdb-table` replaces the table with the files' rows each run, unless the
-table sets another `mode`. To keep one row per order, the latest by
-`ordered_at`, give the orders table in the
-[DuckDB starter](/docs/start/first-project)'s `flows/orders.yaml` this `store`:
+table keeps history. To keep one row per order, the latest by `ordered_at`,
+give the orders table in the [DuckDB starter](/docs/start/first-project)'s
+`flows/orders.yaml` this intent:
```yaml
# flows/orders.yaml
@@ -30,19 +38,19 @@ tables:
orders:
object: orders.csv
contract: shop.orders
- store: { with: { mode: upsert, sequence_by: ordered_at } }
+ history: latest
+ sequence: ordered_at
```
-
-
-Each run then inserts each key's latest row, by `sequence_by` and then by file
-name, and replaces the row the table had for that key. Set `sequence_by` when
-one file can hold a key twice. Running again is safe in every mode. A changed
-primary key or column type needs the table built again: delete
-`ingestron.duckdb` and run again, which reloads everything from the files.
-Every setting is listed under
+Each run then inserts each key's latest row, by `sequence` and then by file
+name, and replaces the row the table had for that key: the method's
+`mode: upsert`. Set `sequence` when one file can hold a key twice. Running
+again is safe in every mode. A changed primary key or column type needs the
+table built again: delete `ingestron.duckdb` and run again, which reloads
+everything from the files. Every setting is listed under
[`duckdb-table`](/docs/reference/packs/duckdb#duckdb-table).
+The method also has a `mode` that no intent stands for, `append`.
`append` adds each file's rows once: it reads each file name once, so a file
re-exported under the same name is skipped and its new rows are not loaded.
Give a re-export a new name, or delete `ingestron.duckdb` and run again to
@@ -54,19 +62,22 @@ cannot be appended; see
## On Databricks
`streaming-table` keeps the current row per key, or every version, with
-`AUTO CDC`. Set its `mode` on the flow or on each table:
+`AUTO CDC`. This flow orders both tables' changes by `changed_at`, keeps the
+current account and every version of each contact, and removes an account
+whose change says `DELETE`:
-
+`history: latest` keeps the current row per key (SCD type 1), and
+`history: full` every version (SCD type 2), with `__START_AT` and `__END_AT`
+columns. `except_columns` is a setting of the method, not intent: the columns
+left out of the kept table, such as the one that says what the change was.
+
+
-`upsert` keeps the current row per key (SCD type 1). `history` keeps every
-version (SCD type 2), with `__START_AT` and `__END_AT` columns. Every setting is
-listed under [`streaming-table`](/docs/reference/packs/databricks#streaming-table).
+Every setting is listed under
+[`streaming-table`](/docs/reference/packs/databricks#streaming-table), with
+the settings that mean the same as this intent.
diff --git a/content/docs/how-to/(build)/meta.json b/content/docs/how-to/(build)/meta.json
index b4787e4..91f8ddb 100644
--- a/content/docs/how-to/(build)/meta.json
+++ b/content/docs/how-to/(build)/meta.json
@@ -9,7 +9,9 @@
"write-a-transform",
"keep-a-table-current",
"add-quality-checks",
+ "cleanse-data",
"define-a-data-product",
+ "grant-access",
"add-an-environment",
"plan-and-generate",
"check-in-ci",
diff --git a/content/docs/how-to/(build)/start-from-a-template.mdx b/content/docs/how-to/(build)/start-from-a-template.mdx
index c9912fc..702b863 100644
--- a/content/docs/how-to/(build)/start-from-a-template.mdx
+++ b/content/docs/how-to/(build)/start-from-a-template.mdx
@@ -4,7 +4,7 @@ description: Start from a connection and a supported route, or from a starter or
---
A project is a folder of YAML definitions with a `project.yaml` at its root.
-Create one in either of two ways:
+Create one in any of three ways:
- **From a connection:** choose the source, how it can be reached, and the
route its data takes. The project holds only that connection, its example
@@ -12,8 +12,12 @@ Create one in either of two ways:
- **From a starter:** copy a complete project for a common situation, such as
[the DuckDB starter](/docs/start/first-project), or one your organisation
shares as a template.
+- **From a published standard:** copy a dated standard for your platform,
+ which says how every table is loaded, kept and run, with a small project
+ that builds; see
+ [Start from a published standard](/docs/how-to/start-from-a-published-standard).
-Either way, the result is ordinary definitions you edit, then plan and apply.
+Each way, the result is ordinary definitions you edit, then plan and apply.
@@ -76,13 +81,6 @@ panel.
-
-
The project holds the connection, its example flows and contracts, and the
platform settings the route needs. What Ingestron cannot know, such as the
source's server, the tables to copy, the storage account or the integration
diff --git a/content/docs/how-to/(build)/write-a-contract.mdx b/content/docs/how-to/(build)/write-a-contract.mdx
index c430213..ec3c6fa 100644
--- a/content/docs/how-to/(build)/write-a-contract.mdx
+++ b/content/docs/how-to/(build)/write-a-contract.mdx
@@ -34,7 +34,9 @@ Ingestron reads:
- for the table: the schema entry's `description`, for the table's comment,
and its `quality` rules;
- the table's and each column's `tags`, and each column's `classification`,
- which it keeps for packs that tag tables.
+ which it keeps for packs that tag tables, and which choose masks and row
+ filters;
+- the contract's `roles`, who may use its tables, which a pack can grant.
[Data contract](/docs/reference/documents#data-contract) lists every ODCS
field.
@@ -43,9 +45,10 @@ What each pack does with a field, or why it does not yet, is on its page:
[Databricks](/docs/reference/packs/databricks#contract-fields),
[Azure Data Factory](/docs/reference/packs/adf#contract-fields),
[Synapse](/docs/reference/packs/synapse#contract-fields) and
-[DuckDB](/docs/reference/packs/duckdb#contract-fields). A contract field that
-no pack of your standard uses yet is a
-[warning](/docs/reference/errors#CONTRACT_FIELD_UNUSED) at that field.
+[DuckDB](/docs/reference/packs/duckdb#contract-fields). A field is judged for
+each table that uses the contract, against the pack that keeps that table. A
+field that pack does not apply yet is
+[not applied here](/docs/concepts/not-applied), a warning at that field.
## Give each column a type
@@ -82,15 +85,15 @@ When a column has a different name at the source, write that name as its
- `primaryKey: true` marks the table's key. With several key columns,
`primaryKeyPosition` orders them. The key is what `upsert` and `history`
keep the current row by; see [Keep a table current](/docs/how-to/keep-a-table-current).
-- `required: true` says the column always has a value. Where the pack checks
- rows, a required column gets a check that it is not null, with the
- standard's `quality.required` action; see
+- `required: true` says the column always has a value. A required column
+ gets a check that it is not null, with the standard's `quality.required`
+ action, run where the table's layer checks rows; see
[Check data quality](/docs/how-to/add-quality-checks).
## Rules about the data
-Where the pack checks rows, these become checks on each row, with the
-standard's `quality.default` action:
+These become checks on each row, with the standard's `quality.default`
+action, run where the table's layer checks rows:
- A column's `logicalTypeOptions`: `minLength` and `maxLength`, `pattern`,
`minimum`, `maximum`, `exclusiveMinimum` and `exclusiveMaximum`,
@@ -104,18 +107,37 @@ standard's `quality.default` action:
Other rules compare a count over all rows, such as a row count or a share of
nulls, which a check of one row cannot. So does `unique: true`, which says no
-value repeats. Where the pack checks rows, these run after each refresh when
-the platform turns on `manage.checks`. Otherwise, and always for a `custom`
-rule, each `quality` rule is a
-[warning](/docs/reference/errors#QUALITY_RULE_NOT_CHECKED) and stays as
-documentation.
+value repeats. These are **table checks**, counted once the table is built.
+DuckDB and Synapse count them in their own run, wherever the table's layer
+checks rows. Databricks counts them after each refresh only when the platform
+turns on `databricks.manage.checks`; otherwise each is
+[`CAPABILITY_OFF`](/docs/reference/errors#CAPABILITY_OFF), a warning with a
+fix that turns it on.
+
+A rule no engine could check as written, such as a `custom` rule, which
+only another engine could run, or one with no comparison, is
+[`QUALITY_RULE_NOT_CHECKED`](/docs/reference/errors#QUALITY_RULE_NOT_CHECKED),
+a warning, and stays as documentation. A `text` rule is documentation and is
+never reported. What each engine applies is in
+[Check data quality](/docs/how-to/add-quality-checks#engines).
+
+A contract says what the data must hold, not how to tidy it: trimming text or
+converting time zones belongs to the table that cleanses it; see
+[Cleanse data in the right layer](/docs/how-to/cleanse-data).
## Tags and classification
A table's `tags`, a column's `tags` and its `classification` are kept for
packs that tag tables. Databricks sets them as Unity Catalog tags after each
-refresh when the platform turns on `manage.tags`; see
+refresh when the platform turns on `databricks.manage.tags`; see
[Databricks contract fields](/docs/reference/packs/databricks#contract-fields).
+The classification is set under the key the standard's
+`governance.classification.tag` gives, `classification` by default.
+
+A column's classification also chooses its mask, from the standard's
+`governance.masks`, and a row filter can match a column by its tag or its
+classification. The contract's `roles` say who may use its tables. See
+[Grant access, mask columns and filter rows](/docs/how-to/grant-access).
## Draft from a sample [#draft-from-a-sample]
diff --git a/content/docs/how-to/(build)/write-a-transform.mdx b/content/docs/how-to/(build)/write-a-transform.mdx
index ee8192c..1b8fa35 100644
--- a/content/docs/how-to/(build)/write-a-transform.mdx
+++ b/content/docs/how-to/(build)/write-a-transform.mdx
@@ -33,6 +33,30 @@ which reads the file next to the definition:
+## Leave out the code [#without-code]
+
+A transform with one input can leave out `code`. It then reads every column of
+its input, `SELECT * FROM {{ tables. }}`, and keeps the columns its
+contract names. Use it for a table that only cleanses, checks or keeps
+another, such as a silver copy of the first project's orders, saved as
+`transforms/orders_clean.yaml`:
+
+```yaml
+apiVersion: ingestron.io/v1
+kind: Transform
+id: orders_clean
+inputs: [orders]
+contract: shop.orders
+cleanse:
+ columns:
+ customer: { trim: true, empty: null }
+```
+
+A transform with any other number of inputs needs its code
+([`TRANSFORM_CODE`](/docs/reference/errors#TRANSFORM_CODE)).
+[Cleanse data in the right layer](/docs/how-to/cleanse-data) builds this table
+and runs it.
+
## What each engine makes of it
On DuckDB, the transform is a table, built again each run from its query,
diff --git a/content/docs/how-to/(standards)/accept-a-warning.mdx b/content/docs/how-to/(standards)/accept-a-warning.mdx
new file mode 100644
index 0000000..58f10bb
--- /dev/null
+++ b/content/docs/how-to/(standards)/accept-a-warning.mdx
@@ -0,0 +1,188 @@
+---
+title: Accept a warning
+description: Take a warning you have looked at and decided to live with out of the problem lists, with the reason, for every environment or only some.
+---
+
+Some warnings stay true however you write the definitions: the engine cannot
+do what a table asks, or the platform team does it another way. Once you have
+looked at one and decided to live with it, accept it in `platform.yaml`, with
+the reason. It then leaves the problem lists, and the run says how many were
+accepted. Nothing generated changes.
+
+This guide uses the [`quality` example](/docs/how-to/add-quality-checks), whose
+`synapse` environment builds the readings table as a Synapse serverless view.
+A view cannot stop a load, so its plan warns:
+
+```text
+flows/readings.yaml:11:87 warning NOT_APPLIED
+ The fail action isn't applied on synapse: a view can't stop a load; checks.sql checks this rule and raises an error when it runs.
+```
+
+## Accept or fix [#when]
+
+Accept a warning only when the definitions are right and the warning will stay:
+
+- **Accept** when the engine cannot do it and you have something else in
+ place, such as `checks.sql` run from CI after each landing; when the
+ platform team does the work its own way, such as tags applied with
+ Terraform while `databricks.manage.tags` is off
+ ([`CAPABILITY_OFF`](/docs/reference/errors#CAPABILITY_OFF)); or when a table
+ can go without the feature.
+- **Fix** it when the definitions can change: a duty the layer's `does` should
+ list, a setting that has moved, a value that is still `REPLACE_ME`. The
+ warning's hint says how; see [Not applied here](/docs/concepts/not-applied#what-to-do).
+
+Only a warning can be accepted. An error stops the run until it is fixed.
+
+## Accept one warning [#accept]
+
+Add an `accept` list at the top level of `platform.yaml`. Each entry gives the
+warning's `code`, narrows it with `about`, and says why in `reason`:
+
+```yaml
+accept:
+ - code: NOT_APPLIED
+ about: { feature: fail, table: readings }
+ reason: checks.sql runs from CI after each landing and pages the on-call team.
+```
+
+Plan again:
+
+```sh
+ingestron plan --no-diff
+```
+
+Before the files, it prints:
+
+```text
+databricks, duckdb: warnings
+platform.yaml:16:5 warning ACCEPT_UNUSED
+ The accept entry for NOT_APPLIED (table "readings", feature "fail") matches no warning in this run.
+ Remove the entry, or correct its code or about. To accept different warnings in each environment, write the list as a value: accept: ${values.accept}.
+ Fix: Remove the entry.
+Not applied on this platform
+synapse:
+contracts/readings.yaml:14:40 warning NOT_APPLIED
+ The pattern rule isn't applied on synapse: serverless T-SQL has no regular expressions.
+standard.yaml:12:15 warning NOT_APPLIED
+ Quarantine isn't applied on synapse: a view keeps no rows to quarantine; checks.sql returns how many rows each drop check leaves out instead.
+standard.yaml:13:12 warning NOT_APPLIED
+ quality.results: true isn't applied on synapse: a view keeps no results; checks.sql returns each check's counts instead when it runs.
+1 accepted warning (platform.yaml)
+```
+
+The `fail` warning has gone, and the last line counts it. The other
+environments build on engines that apply `fail`, so there the entry matches
+nothing, and
+[`ACCEPT_UNUSED`](/docs/reference/errors#ACCEPT_UNUSED) says so: an entry
+that no longer matches anything is reported, so the list never outlives the
+warnings it was written for.
+
+## Accept it in some environments only [#per-environment]
+
+To accept a warning in one environment, write the list as a value that each
+environment gives. In `platform.yaml`, replace the list with:
+
+```yaml
+accept: ${values.accept}
+```
+
+and give the list in `project.yaml`, empty unless an environment says
+otherwise:
+
+```yaml
+# One set of checks on three engines: each environment builds the readings
+# table on a different one, so you can compare what each does with the same
+# rules. A real project builds its tables on one engine.
+id: iot
+environments:
+ databricks:
+ values: { ingest: files, store: streaming-table, source: sensors }
+ duckdb:
+ values: { ingest: duckdb-files, store: duckdb-table, source: sensors }
+ synapse:
+ values:
+ ingest: adf-files
+ store: synapse-view
+ source: drop
+ accept:
+ - code: NOT_APPLIED
+ about: { feature: fail, table: readings }
+ reason: checks.sql runs from CI after each landing and pages the on-call team.
+values:
+ accept: [] # no environment accepts a warning unless it says so
+imports:
+ - standard.yaml
+ - platform.yaml
+ - connections/*.yaml
+ - flows/*.yaml
+ - contracts/*.yaml
+```
+
+An environment's list replaces the one in `values`, whole. Plan again, and
+before the files only the warnings still to look at remain:
+
+```text
+Not applied on this platform
+synapse:
+contracts/readings.yaml:14:40 warning NOT_APPLIED
+ The pattern rule isn't applied on synapse: serverless T-SQL has no regular expressions.
+standard.yaml:12:15 warning NOT_APPLIED
+ Quarantine isn't applied on synapse: a view keeps no rows to quarantine; checks.sql returns how many rows each drop check leaves out instead.
+standard.yaml:13:12 warning NOT_APPLIED
+ quality.results: true isn't applied on synapse: a view keeps no results; checks.sql returns each check's counts instead when it runs.
+1 accepted warning (platform.yaml)
+```
+
+## Say which warnings [#about]
+
+An entry accepts every warning with its `code` that matches each key of its
+`about`:
+
+- `file`: a pattern for the file the warning is in, relative to the project
+ folder, as `imports` writes them: `*` stays within a folder and `**` crosses
+ folders, such as `flows/legacy/**`.
+- `table`: the table the warning is about, by its name.
+- `feature`: what is not applied or is off. For `NOT_APPLIED`, a rule kind
+ (`notNull`, `pattern`, …), an action (`warn`, `drop`, `fail`),
+ `table-checks`, `expectations`, `quarantine`, a cleansing operation such as
+ `cleanse.trim` or `deduplicate`, an intent value such as `history=full`, or
+ a contract field's path; for `CAPABILITY_OFF`, the capability, such as
+ `tags` or `checks`. For a role, a grant mode, a mask or a row filter, both
+ codes use `grants`, `write`, `exact`, `masks` or `filters`, such as
+ `about: { feature: grants }` for roles DuckDB cannot grant; see
+ [Accept a gap](/docs/how-to/grant-access#accept-a-gap).
+
+Without `about`, an entry accepts every warning with that code, so narrow it to
+what you have looked at. A warning that several entries match is accepted by
+the first. [Documents](/docs/reference/documents#platform-accept) lists the
+fields.
+
+An entry whose `code` cannot be accepted is
+[`ACCEPT_CODE`](/docs/reference/errors#ACCEPT_CODE), an error: a code that is
+only ever an error, one that is no longer reported, such as
+`CONTRACT_FIELD_UNUSED`, or `ACCEPT_CODE` and `ACCEPT_UNUSED` themselves. Each
+code on the [errors page](/docs/reference/errors) says how it is reported.
+
+A few warnings come only from a pack building the files, such as
+[`PACK_NOTICE`](/docs/reference/errors#PACK_NOTICE) and
+[`SECRET_NAME_DIFFERS`](/docs/reference/errors#SECRET_NAME_DIFFERS). An entry
+for one of these is matched, and judged unused, only once the packs have built
+the files, by `plan`, `apply` and `check`; the editor, which builds nothing,
+never reports it unused.
+
+## What accepting does not change [#what-still-shows]
+
+- **The files.** Accepting changes nothing generated, and a warning never
+ changed an exit code: `check` passes in CI with or without the list.
+- **The count.** `check`, `plan` and `apply` end their problems with
+ ` accepted warnings (platform.yaml)`, so an accepted warning is never
+ hidden entirely.
+- **The editor.** VS Code leaves accepted warnings out of the Problems panel.
+- **The passport.** A [data product's passport](/docs/how-to/define-a-data-product#read-the-passport)
+ still says `Not enforced` for an accepted `NOT_APPLIED` or
+ `CAPABILITY_OFF`: accepting a warning does not make the check run, or grant
+ the role.
+
+Review the list as you would the definitions: in a pull request, with its
+reasons.
diff --git a/content/docs/how-to/(standards)/ask-the-platform-team.mdx b/content/docs/how-to/(standards)/ask-the-platform-team.mdx
index 1f37692..7c80550 100644
--- a/content/docs/how-to/(standards)/ask-the-platform-team.mdx
+++ b/content/docs/how-to/(standards)/ask-the-platform-team.mdx
@@ -27,12 +27,16 @@ Each section appears only when it has something to say, in this order:
where it is used, whether it is set, and the name the deploying pipeline
supplies it as; never its value. See [Values to provide](#values-to-provide).
- The identity pipelines and jobs run as, and the workspace.
-- **Switched on:** each `manage` switch the platform turns on, what it does
+- **Switched on:** each `databricks.manage` switch the platform turns on, what it does
and what it needs on the platform, from the pack's
[platform prerequisites](/docs/reference/packs/databricks#platform-prerequisites).
+ The Synapse switch `synapse.manage.grants` is listed the same way, when a
+ table of the environment runs a Synapse step; see
+ [Grant access](/docs/how-to/grant-access).
- **Used by the definitions, but switched off:** features the definitions use
while their switch is off, such as a contract's `unique: true` without
- `manage.checks`.
+ `databricks.manage.checks`, or a role on a Synapse view without
+ `synapse.manage.grants`.
- **Shared infrastructure:** the platform's settings for the methods the
environment uses, with this environment's values. A secret reads "a
secret"; its value is never shown.
@@ -67,7 +71,8 @@ Its request for `dev` is:
-A platform document that names `run_as` and turns on `manage` switches adds
+A platform document whose `databricks:` section names `run_as` and turns on
+`manage` switches adds
what each switch needs and the SQL to run once; see
[Before you deploy](/docs/platforms/databricks-deploy#prerequisites) in Deploy the Databricks bundle.
@@ -83,8 +88,8 @@ When another pack reads the landed files, its sections follow these.
### Synapse serverless views
For the project in [Synapse serverless views](/docs/platforms/synapse), the
-request for `dev` adds the database, the external data source and the report
-users' permissions:
+request for `dev` adds the database, the external data source, and the
+database user each role is, with the permissions views over files need:
+
+## What one decides
+
+Each published standard is a folder of ordinary definitions:
+
+- `standard.yaml` says how every table is loaded, kept and run, in
+ [intent](/docs/reference/intent): which layers there are and what each
+ keeps, how files are read, the columns that say when and where each row came
+ from, and how a failed run is tried again and who is told.
+- `platform.yaml` and `project.yaml` hold what belongs to your workspace,
+ such as its address and who the jobs run as. What only you can know is
+ `REPLACE_ME`, for you to write.
+- `STANDARD.md`, once it is in your project, explains each choice and the
+ platform page it was checked against.
+
+Each published standard has a page that shows these files, what each
+`REPLACE_ME` stands for, and how each table of its project resolves, such as
+[Streaming lakehouse on Databricks](/docs/how-to/kits/databricks-lakehouse-streaming).
+
+## Versions
+
+A published standard is named `/`, and each version by the
+date it was checked: `duckdb/local-medallion@2026-11-10`. A version never
+changes once it is published; a newer one has a later date and says in its
+notes what changed. A version whose date ends `-preview`, such as
+`2026-12-01-preview`, uses platform features still in preview, and is used
+only when you name it. The others use only generally available features, and
+their standard says `features: ga`.
+
+Each version names the `ingestron` it needs, such as `>=2.3.0`. `init`
+refuses a version that needs a newer one than you have, and names both,
+before it writes anything.
+
+## Choose one
+
+- **Your platform.** A published standard is for the packs it names, such as
+ `databricks@2`. To keep your source's route, its packs must be among them;
+ see [With a connection](/docs/how-to/start-from-a-published-standard#with-a-connection).
+- **Try it first.** [Local medallion on DuckDB](/docs/how-to/kits/duckdb-local-medallion)
+ runs on a laptop with nothing to replace, so you can see how intent becomes
+ SQL before you choose one for a workspace.
+- **What it keeps.** Read its page: each table's intent, and what the pack
+ does with each value on its [reference page](/docs/reference/packs). A value
+ that does not fit your tables can be changed once the project is yours.
+
+## List them
+
+```sh
+ingestron standard list
+```
+
+```text
+databricks/lakehouse-streaming@2026-11-10 Streaming lakehouse on Databricks (ga)
+duckdb/local-medallion@2026-11-10 Local medallion on DuckDB (ga)
+```
+
+Each line is a version, with its title and `(ga)` or `(preview)`. The versions
+installed with `ingestron` come first, and work offline. Then come the
+versions only the public catalogue has, the
+[`ingestron/standards`](https://github.com/ingestron/standards) repository,
+each marked `(online)`: `ingestron` reads its tags with your own `git`. When it
+cannot reach the catalogue, the list starts with
+`Could not reach the catalogue; showing the kits installed with ingestron.`
+`--platform duckdb` keeps one platform's.
+
+Set `INGESTRON_STANDARDS_REPOSITORY` to read another catalogue, such as a
+mirror inside your network. A team's own published standard is read from its
+Git repository instead; see
+[Share a published standard](/docs/how-to/share-a-template#share-a-published-standard).
+
+## Next
+
+- [Start from a published standard](/docs/how-to/start-from-a-published-standard):
+ create the project and write what it leaves to you.
+- [Move to a newer published standard](/docs/how-to/upgrade-a-standard):
+ replace a project's standard with a later version.
diff --git a/content/docs/how-to/(standards)/configure-a-standard.mdx b/content/docs/how-to/(standards)/configure-a-standard.mdx
index 0130898..8fa6952 100644
--- a/content/docs/how-to/(standards)/configure-a-standard.mdx
+++ b/content/docs/how-to/(standards)/configure-a-standard.mdx
@@ -19,6 +19,10 @@ To make every project follow your rules:
4. Answer each project's [platform request](/docs/how-to/ask-the-platform-team),
and [upgrade](/docs/how-to/upgrade) on purpose.
+To start from a standard that is already written and checked for your
+platform, [choose a published standard](/docs/how-to/choose-a-published-standard):
+once a project has it, it is your copy, and this page applies to it.
+
## What a standard holds [#what-a-standard-holds]
This is Acme's standard. Its naming, types and storage choices belong to
@@ -51,6 +55,16 @@ set what happens to rows that fail a check (`quality`), and require
[Which setting wins](/docs/concepts/settings) explains what happens when a project
sets one too.
+It can also say how tables are loaded, kept and run, in
+[intent](/docs/reference/intent) that every pack reads: what each layer keeps
+(`history`) and does with its rows (`does`), columns that record when and
+where each row came from (`audit`), how text and timestamps are cleansed
+(`cleanse`), how files and rows are loaded (`load`), how runs are retried and
+who is told (`operations`), and whether platform features in preview may be
+used (`features`). See [What a layer does](/docs/concepts/layers#what-a-layer-does)
+and [Cleanse data in the right layer](/docs/how-to/cleanse-data). The [published standards](/docs/how-to/choose-a-published-standard)
+are complete examples.
+
## One standard for several packs
A project has exactly one standard, even when it uses several packs. Each
@@ -161,8 +175,8 @@ To take it from a Git repository instead, see
-**Ingestron: New project** keeps the starter's or route's own standard. To
-start from yours, run `ingestron init` with `--standard` in a terminal, or
+**Ingestron: New project** keeps the starter's or route's own standard, or
+takes a published one with **From a published standard**. To start from yours, run `ingestron init` with `--standard` in a terminal, or
replace the new project's `standard.yaml` with your standard and plan.
@@ -191,6 +205,11 @@ classify each column it publishes, and say who may use sensitive data. A
product that breaks a rule stops generation, so the pull request fails. See
[Define a data product](/docs/how-to/define-a-data-product).
+The same section says how access is generated for every table: whether
+grants are exact, how each classification is masked, and which rows each
+reader sees. A layer can also say who reads its tables. See
+[Grant access, mask columns and filter rows](/docs/how-to/grant-access).
+
## Next
- What each project needs from you: [Ask the platform team](/docs/how-to/ask-the-platform-team).
diff --git a/content/docs/how-to/(standards)/kits/databricks-lakehouse-streaming.mdx b/content/docs/how-to/(standards)/kits/databricks-lakehouse-streaming.mdx
new file mode 100644
index 0000000..bfa9cda
--- /dev/null
+++ b/content/docs/how-to/(standards)/kits/databricks-lakehouse-streaming.mdx
@@ -0,0 +1,112 @@
+---
+title: Streaming lakehouse on Databricks
+description: A published standard for Databricks. Streaming tables that read each new file once, silver cleansed, checked and kept to the latest row per key, gold materialised views, and a refresh job retried and alerted on failure. Contract roles are granted, and confidential and restricted columns masked.
+---
+
+
+
+A web shop's files become three layers in Databricks: `bronze.customers`, a row
+each time a customer is exported, with the values that do not fit kept aside;
+`silver.orders`, the latest row per order, from change records, cleansed and
+checked; and `gold.daily_sales`, totals per customer and day. One Lakeflow
+pipeline builds them, and one job refreshes it each day, tries again when it
+fails, and mails your team when it still fails. The contracts name who may
+read `silver.orders` and `gold.daily_sales`, and the customers' email and phone
+number are masked for everyone but the people who run the pipeline.
+
+## Start from it
+
+From the folder that will hold the project:
+
+```sh
+ingestron init shop --kit databricks/lakehouse-streaming@2026-11-10
+```
+
+Then write each value that is still `REPLACE_ME`, below, and plan; see
+[Start from a published standard](/docs/how-to/start-from-a-published-standard)
+and [Deploy the Databricks bundle](/docs/platforms/databricks-deploy).
+
+## What to write [#replace-me]
+
+What belongs to your workspace is `REPLACE_ME`. Point at each one (or tap it
+on a touch screen) to see the field it fills, where to find the value, and a
+suggestion; in VS Code, hover it. The platform holds who is told of failures,
+who the contracts' role `shop-analysts` is (`principals`), and the
+`databricks:` section, which also turns on tags, grants and masks and names
+the schema for the mask functions:
+
+
+
+`project.yaml` holds the two groups the `databricks:` section's permissions
+read, one that manages the pipeline and job and one that may view them, and
+what differs between `dev` and `prod`: each workspace's address, and the
+volume the shop's files land in:
+
+
+
+## The standard
+
+
+
+Each table starts from its layer's intent. `customers` lands in `bronze`, the
+first layer; the orders flow says `layer: silver`, and gives the column that
+orders its changes and the condition for a delete. Each value, and where it
+came from:
+
+
+
+Each layer also says what it does with its rows, in
+[`does`](/docs/concepts/layers#what-a-layer-does). `bronze` does
+`raw`: it keeps every row as it arrived, and its checks only count. `silver`
+does `cleanse`, `deduplicate`, `check` and `quarantine`: the standard's
+`cleanse` trims its text and turns an empty text into null, `required: drop`
+leaves out a change missing a required value, and `quarantine: true` keeps it
+in `silver.orders_quarantine`. `silver.orders` does not deduplicate: AUTO CDC
+already keeps one row per order. `gold` does `check`, so `gold.daily_sales`
+leaves out rows that fail its checks; its SQL trims the emails it reads from
+`bronze.customers` and leaves out customers with no id or email itself.
+
+What the Databricks pack does with each value, with each feature's release
+level and the date it was checked, is in
+[the Databricks pack's intent](/docs/reference/packs/databricks#intent), and
+its checks and cleansing in
+[Quality and cleansing](/docs/reference/packs/databricks#quality-and-cleansing).
+With `features: ga`, Ingestron warns about any value that needs a feature in
+preview ([`PREVIEW_FEATURE`](/docs/reference/errors#PREVIEW_FEATURE)). The
+standard writes none: the job alerts on failure but not on slow runs, because
+Databricks' run duration alert is in Public Preview.
+
+## Who reads what [#governance]
+
+The standard's `governance` section says how data is classified and protected,
+and the Platform turns each part on:
+
+- `classification` lists the classes a column may have, and `sensitive` names
+ the two that need care, `confidential` and `restricted`.
+ `contracts/customers.yaml` classifies `email` as `confidential` and `phone`
+ as `restricted`; `contracts/daily_sales.yaml` classifies its `email` too.
+ Ingestron sets each class as a tag on its column.
+- `access: grants: additive` grants each table the roles its contract names, as
+ `GRANT SELECT`, and never takes a privilege away. The orders and sales
+ contracts name the role `shop-analysts`; `principals` in `platform.yaml` says
+ which group that is. `exact` would also revoke the direct grants a contract
+ does not give.
+- `masks` hides a class: `confidential: hash` shows a hash of the text, and
+ `restricted: "null"` shows null. Each is a Unity Catalog column mask policy
+ that matches the class tag, and exempts the `run_as` identity and the groups
+ in `unmasked`, here `engineers`, so the pipeline never reads masked data.
+- `bronze` has `access: { contract: false }`: no contract role is granted on a
+ raw table, which is for the engineers who load it.
+
+Masks need governed tags, which an account administrator creates;
+`ingestron request` lists them, and the schema of the mask functions, with
+what `run_as` needs on each. Readers need serverless compute or Databricks
+Runtime 16.4 or later. See
+[Grant access, mask columns and filter rows](/docs/how-to/grant-access) for
+exact grants, row filters and where the policies go.
+
+## Why each choice
+
+In your project, `STANDARD.md` explains each section of the standard on
+Databricks, and links the Databricks pages each was checked against, with the
+date.
diff --git a/content/docs/how-to/(standards)/kits/duckdb-local-medallion.mdx b/content/docs/how-to/(standards)/kits/duckdb-local-medallion.mdx
new file mode 100644
index 0000000..117ea92
--- /dev/null
+++ b/content/docs/how-to/(standards)/kits/duckdb-local-medallion.mdx
@@ -0,0 +1,64 @@
+---
+title: Local medallion on DuckDB
+description: A published standard for DuckDB on a laptop. Bronze loaded in full from CSV files, silver cleansed, checked and kept to the latest row per key, and nothing to replace.
+---
+
+
+
+Sample orders become two layers in one DuckDB file: `bronze.orders`, the rows
+as they arrive in the file, and `silver.customer_totals`, one row per customer,
+with its text trimmed and its check run. Both are published as Parquet. Nothing
+is `REPLACE_ME`: the project runs as written, so it is the published standard
+to try first.
+
+## Start from it
+
+From the folder that will hold the project:
+
+```sh
+ingestron init shop --kit duckdb/local-medallion@2026-11-10
+```
+
+Then run it as the new `README.md` says; see
+[Start from a published standard](/docs/how-to/start-from-a-published-standard)
+and [DuckDB](/docs/platforms/duckdb).
+[Cleanse data in the right layer](/docs/how-to/cleanse-data) then adds a
+cleansed silver table to it.
+
+## The standard
+
+
+
+Each table starts from its layer's intent. The flow's table lands in `bronze`,
+the first layer, and the transform's in `silver`. Neither flow nor transform
+writes intent of its own, so each value comes from the standard, or from the
+DuckDB pack's default:
+
+
+
+Each layer also says what it does with its rows, in
+[`does`](/docs/concepts/layers#what-a-layer-does). `bronze` does
+`raw`: it keeps every row as it arrived, nothing there is cleansed, and its
+checks only count: the run prints how many orders have no `order_id`. `silver` does `cleanse`, `deduplicate`, `check` and `quarantine`: the
+standard's `cleanse` trims its text and turns an empty text into null, the
+contract's required `customer` is checked, a row without one is left out, and
+`quarantine: true` keeps such a row in `silver.customer_totals_quarantine`.
+`quality: results: true` keeps each check's result in the `_ingestron_checks`
+table of each schema. The plan reports no warning.
+
+What the DuckDB pack does with each value, and with every other, is in
+[the DuckDB pack's intent](/docs/reference/packs/duckdb#intent), and its
+checks and cleansing in
+[Quality and cleansing](/docs/reference/packs/duckdb#quality-and-cleansing).
+
+## The platform
+
+Nothing in it is yours to replace. `published` is the folder each table is
+written to as Parquet:
+
+
+
+## Why each choice
+
+In your project, `STANDARD.md` explains each section of the standard, and
+links the DuckDB pages each was checked against, with the date.
diff --git a/content/docs/how-to/(standards)/kits/meta.json b/content/docs/how-to/(standards)/kits/meta.json
new file mode 100644
index 0000000..cfdca50
--- /dev/null
+++ b/content/docs/how-to/(standards)/kits/meta.json
@@ -0,0 +1,4 @@
+{
+ "title": "Published standards",
+ "pages": ["duckdb-local-medallion", "databricks-lakehouse-streaming"]
+}
diff --git a/content/docs/how-to/(standards)/meta.json b/content/docs/how-to/(standards)/meta.json
index d141e65..b3d43de 100644
--- a/content/docs/how-to/(standards)/meta.json
+++ b/content/docs/how-to/(standards)/meta.json
@@ -1,11 +1,16 @@
{
"title": "Standards and teams",
- "description": "One standard for every project, shared from Git.",
+ "description": "One standard for every project: your own, or a published one, shared from Git.",
"root": true,
"pages": [
"configure-a-standard",
+ "choose-a-published-standard",
+ "start-from-a-published-standard",
+ "upgrade-a-standard",
+ "kits",
"name-things-your-way",
"ask-the-platform-team",
+ "accept-a-warning",
"share-a-template",
"upgrade"
]
diff --git a/content/docs/how-to/(standards)/share-a-template.mdx b/content/docs/how-to/(standards)/share-a-template.mdx
index 12dffb8..d9d488d 100644
--- a/content/docs/how-to/(standards)/share-a-template.mdx
+++ b/content/docs/how-to/(standards)/share-a-template.mdx
@@ -1,6 +1,6 @@
---
title: Share templates and standards
-description: Keep your team's project templates and your organisation's standard in Git, and start projects from them.
+description: Keep your team's project templates, your organisation's standard and your own published standards in Git, and start projects from them.
---
A template is a project folder your team starts from, and a standard is the
@@ -61,16 +61,61 @@ says how to give a project a newer version, and how to keep projects to it.
**Ingestron: New project** in VS Code keeps the template's own standard; use
the command for `--standard`.
+## Share a published standard [#share-a-published-standard]
+
+To give teams a dated standard with an example that builds, as Ingestron's own
+[published standards](/docs/how-to/choose-a-published-standard) are, make a
+folder `///` that holds a project, and a `kit.yaml`
+beside its files:
+
+- `standard.yaml`, with `origin: { standard: /, version: }`
+ and the same `packs` as `kit.yaml`;
+- `platform.yaml` and `project.yaml`, with `REPLACE_ME` and a comment for
+ each value that is the team's;
+- `README.md`, which explains each choice, and becomes the project's
+ `STANDARD.md`;
+- the connections, flows, contracts and transforms of a small example.
+
+This is the `kit.yaml` of the local medallion:
+
+
+
+`id` is `/`, its platform the first pack's id; `version` is
+the date it was checked, `YYYY-MM-DD`, or `YYYY-MM-DD-preview` with
+`features: preview`. `requires` names the `ingestron` versions it needs, as
+`>=`, `^` or `~` and a version; `replaces` names the version before it, and
+`notes` says what changed, which `standard install` prints. Any other key is
+refused, so a mistake shows at once.
+
+Commit the folder, and tag the commit `/@`. Teams
+start from it with the repository's URL, the folder after `#`, and the tag
+after `@`:
+
+```sh
+ingestron init orders \
+ --kit https://github.com/acme/data-platform.git#kits/databricks/acme-lakehouse/2026-11-10@databricks/acme-lakehouse@2026-11-10
+```
+
+`ingestron standard install` takes the same form. A version never changes once
+tagged: publish a change as a new version, in a new folder. A published
+standard from a Git URL is never listed by `standard list`.
+
## What Ingestron does with a Git URL
-Fetching a template or standard is the only time Ingestron uses the network.
+Fetching a template, a standard or a published standard is the only time
+Ingestron uses the network, besides reading the catalogue's tags for
+`ingestron standard list` and for a published standard named without its
+version.
- Cloning uses your own `git`, so your Git credentials and settings apply. It
- takes only the latest commit of the default branch, and runs none of the repository's hooks or
+ takes only the latest commit of the default branch, or of the tag for a
+ published standard, and runs none of the repository's hooks or
submodules; filters from your own git settings, such as Git LFS, still
- apply.
+ apply. For a published standard, neither Git nor SSH asks for a password:
+ they use the credentials Git and your SSH agent already have. Reading the
+ catalogue's tags stops after 15 seconds, and a clone after 60 seconds.
- The part after `#` must stay inside the repository, and must be a folder
- for a template or a regular file for a standard.
-- It never runs anything from a template or standard.
+ for a template or a published standard, or a regular file for a standard.
+- It never runs anything from a template, a standard or a published standard.
- A user name or token in a URL is never printed or written into the
project.
diff --git a/content/docs/how-to/(standards)/start-from-a-published-standard.mdx b/content/docs/how-to/(standards)/start-from-a-published-standard.mdx
new file mode 100644
index 0000000..2dacac3
--- /dev/null
+++ b/content/docs/how-to/(standards)/start-from-a-published-standard.mdx
@@ -0,0 +1,143 @@
+---
+title: Start from a published standard
+description: Create a project from a published standard, write the values it leaves to you, and plan.
+---
+
+A [published standard](/docs/how-to/choose-a-published-standard) gives a new
+project its standard, its platform skeleton and a small example that builds.
+This page creates one on your laptop with
+[Local medallion on DuckDB](/docs/how-to/kits/duckdb-local-medallion), then
+shows what changes for a workspace, with
+[Streaming lakehouse on Databricks](/docs/how-to/kits/databricks-lakehouse-streaming).
+You need the `ingestron` command or the VS Code extension; see
+[Install](/docs/start/install).
+
+## Create the project
+
+
+
+
+From the folder that will hold the project, name the published standard and
+its version with `--kit`:
+
+```sh
+ingestron init shop --kit duckdb/local-medallion@2026-11-10
+```
+
+Without `@`, `--kit duckdb/local-medallion` takes the newest version
+that is not a preview, among the versions installed with `ingestron` and those
+in the public catalogue. When `ingestron` cannot reach the catalogue, it takes
+the newest installed version, and its first line says so:
+`Could not reach the catalogue; using the newest installed version, duckdb/local-medallion@2026-11-10.`
+`--kit` replaces `--template` and `--standard`: give one of the three.
+
+
+
+
+Run **Ingestron: New project** from the Command Palette (⇧⌘P on macOS,
+Ctrl+Shift+P elsewhere), choose **From a published standard**, then
+**Local medallion on DuckDB**, then a folder named `shop`. The new
+`README.md` opens in a preview, and the plan is in the Ingestron output panel.
+
+
+
+
+Ingestron copies the published standard's files into `shop`, checks every
+environment, and plans:
+
+```text
+Created shop from the kit duckdb/local-medallion@2026-11-10 (13 files).
+
+duckdb → shop/build/duckdb (dev)
+create dev/README.md
+create dev/run.sql
+create dev/tables/bronze.orders.sql
+create dev/tables/silver.customer_totals.sql
+
+4 files: 4 create. Ready to apply.
+Nothing to replace: the project runs as written.
+```
+
+The project holds the standard's files as they are, except two:
+
+- `STANDARD.md` is the published standard's own README: what each section of
+ `standard.yaml` does on DuckDB, and the DuckDB pages it was checked
+ against.
+- `README.md` is the project's, as for every new project: how to run it, and
+ the values still to write.
+
+`standard.yaml` names its `origin`, the published standard and version it
+came from, so a later version can be compared with it; see
+[Move to a newer published standard](/docs/how-to/upgrade-a-standard).
+
+Run it as its `README.md` says: `ingestron apply`, then the DuckDB command
+line; see [DuckDB](/docs/platforms/duckdb).
+
+## Write the values it leaves to you [#write-the-values]
+
+A published standard for a workspace cannot know your workspace. Each value
+that is yours is `REPLACE_ME`, with a comment above it. These are the
+platform's, in the streaming lakehouse; point at a `REPLACE_ME` (or tap it on
+a touch screen) to see what to write:
+
+
+
+In VS Code, hover a `REPLACE_ME` to see the same: the field it fills, where to
+find the value, and a suggestion. The project's `README.md` lists every value
+still to write, with its file and line. `ingestron plan` warns about each one,
+and `ingestron check` fails until each is written; see
+[`REPLACE_ME`](/docs/reference/errors#REPLACE_ME).
+
+To write some as you create the project, give each with `--set =`,
+or `--set .=` for one environment:
+
+```sh
+ingestron init shop --kit databricks/lakehouse-streaming@2026-11-10 \
+ --set dev.host=https://adb-1111111111111111.1.azuredatabricks.net \
+ --set service_principal_name=7f3c2a10-0000-4000-8000-000000000001
+```
+
+Each published standard's page lists all of its values, in every file.
+
+## With a connection [#with-a-connection]
+
+To start from your own source with a published standard, add the connection
+options, as [Create a project](/docs/how-to/start-from-a-template#start-from-a-connection)
+describes:
+
+```sh
+ingestron init sales --kit databricks/lakehouse-streaming@2026-11-10 \
+ --connector sqlserver --connectivity public --via databricks --yes
+```
+
+The published standard gives `standard.yaml`, `project.yaml`, `platform.yaml`
+and `STANDARD.md`; the route gives the connection, its flows, contracts and
+transforms. Where both write a key of `project.yaml` or `platform.yaml`, the
+published standard's value stays. Every pack the route needs must be one the
+published standard names: `--via adf-databricks` needs `adf@2` too, so this
+one refuses it.
+
+Then read the plan. A value of the standard that the route's methods do not
+apply is a [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) warning with
+the reason, listed under `Not applied on this platform`. The streaming
+lakehouse reads files, and Lakeflow Connect reads SQL Server itself, so that
+list begins:
+
+```text
+Not applied on this platform
+dev, prod:
+standard.yaml:28:12 warning NOT_APPLIED
+ load.mode: files isn't applied on databricks: Managed ingestion reads the source, not files.
+```
+
+Keep a warning you can live with, and
+[accept it](/docs/how-to/accept-a-warning) in the platform with the reason, or
+change the standard where it does not fit your source: the project is yours.
+
+## Make it yours
+
+Once created, the project is ordinary definitions: change the contracts and
+flows to your source's, and the standard where your team decides otherwise.
+Run `ingestron plan` after each change. To take a later version of the
+published standard, see
+[Move to a newer published standard](/docs/how-to/upgrade-a-standard).
diff --git a/content/docs/how-to/(standards)/upgrade-a-standard.mdx b/content/docs/how-to/(standards)/upgrade-a-standard.mdx
new file mode 100644
index 0000000..500681f
--- /dev/null
+++ b/content/docs/how-to/(standards)/upgrade-a-standard.mdx
@@ -0,0 +1,114 @@
+---
+title: Move to a newer published standard
+description: Replace a project's standard with a later version of its published standard, see what changes with plan and git diff, and keep your own edits.
+---
+
+A published standard never changes once it is published: a newer version has
+a later date. To move a project to it, replace the project's standard in a
+pull request, read what the plan and `git diff` show, and put back the edits
+your team made. Only `standard.yaml` is replaced; the rest of the project is
+yours and stays as it is.
+
+Upgrading `ingestron` itself is a separate step; see
+[Upgrade Ingestron and its packs](/docs/how-to/upgrade).
+
+## Find the version
+
+`ingestron standard list` lists the versions there are; see
+[List them](/docs/how-to/choose-a-published-standard#list-them). The
+project's `standard.yaml` names the one it has, in its `origin`:
+
+
+
+`ingestron check` never says a newer version exists, and makes no network
+call: moving is your choice.
+
+## Install it
+
+1. On a new branch, commit any change to the standard first. `install` refuses
+ to replace a file with changes that are not committed, so that `git diff`
+ shows only the new version:
+
+ ```text
+ standard.yaml has changes that are not committed; commit them first so git diff shows only the new standard.
+ ```
+
+2. From the project's folder, name the published standard and the version to
+ move to, such as `duckdb/local-medallion@2026-11-10`:
+
+ ```sh
+ ingestron standard install duckdb/local-medallion@2026-11-10
+ ```
+
+ The name alone takes the newest version that is not a preview, among the
+ versions installed with `ingestron` and those in the public catalogue. When
+ `ingestron` cannot reach the catalogue, it takes the newest installed
+ version, and its output starts by saying so.
+
+ It writes the new version's `standard.yaml` in place of the project's,
+ with the project's own `id` and at the same path, even when the standard
+ is in another file, such as `standards/acme.yaml`. Its comments come with
+ it. In a project created from that same version, it prints:
+
+ ```text
+ Replaced standard.yaml with duckdb/local-medallion@2026-11-10 (was 2026-11-10).
+
+ First version.
+
+ Not applied: no other kit file changed since 2026-11-10.
+
+ Next: run ingestron plan, then review git diff.
+ ```
+
+ After `Replaced`, the new version's notes say what changed. **Not
+ applied** lists the published standard's other files that changed since
+ the project's version, such as `platform.yaml` or a contract: `install`
+ does not copy them, because they are yours now. Compare each with yours.
+ When the project's `origin` names no version it can find, it says the
+ earlier version is not available, and you compare by hand. If the project
+ no longer builds with the new standard, the problems follow, as `plan`
+ prints them; the new file stays written.
+
+## Review the change
+
+3. Plan, to see every generated file the new standard changes:
+
+ ```sh
+ ingestron plan
+ ```
+
+ Review each `update`, `create` and `delete` as you would a change to the
+ definitions. A value the new standard sets that a pack does not apply is a
+ [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) warning, with the
+ reason.
+
+4. Read the standard's own change:
+
+ ```sh
+ git diff standard.yaml
+ ```
+
+ Lines the new version adds, and lines your team had changed, both show
+ here. The new file has the published version's text, so an edit your team
+ made to `standard.yaml`, such as another catalog name, shows as a removed
+ line.
+
+5. Put back each edit you want to keep, in `standard.yaml`, and plan again.
+ For each file **Not applied** listed, take what you want from the new
+ version into yours.
+
+6. Apply, and commit the standard with the generated files:
+
+ ```sh
+ ingestron apply
+ ```
+
+ `ingestron check` in CI then passes on the pull request, because the
+ committed files are what the new standard generates.
+
+## Next
+
+- [Set your organisation's standard](/docs/how-to/configure-a-standard): keep
+ the project to the standard in CI and in review.
+- [Intent](/docs/reference/intent): what each value of a standard means, and
+ what each pack does with it.
diff --git a/content/docs/how-to/(standards)/upgrade.mdx b/content/docs/how-to/(standards)/upgrade.mdx
index 4a14349..8ad8ff0 100644
--- a/content/docs/how-to/(standards)/upgrade.mdx
+++ b/content/docs/how-to/(standards)/upgrade.mdx
@@ -1,6 +1,6 @@
---
title: Upgrade Ingestron and its packs
-description: Choose the version every project runs, see what an upgrade changes with plan, move CI and your team together, and upgrade from 1.x to 2.0.
+description: Choose the version every project runs, see what an upgrade changes with plan, move CI and your team together, and what changed from 1.x to 2.3.
---
The packs come inside the `ingestron` command: each release of the command
@@ -46,7 +46,7 @@ choose to upgrade. The [CI templates](/docs/how-to/check-in-ci) and the
[deployment templates](/docs/how-to/deploy-through-environments) do:
```sh
-npm install -g ingestron@2.2.0
+npm install -g ingestron@2.3.0
```
`npm ls -g ingestron` shows the version a computer has installed.
@@ -95,7 +95,7 @@ next; see [Deploy through environments](/docs/how-to/deploy-through-environments
### The standard and the folders
-1. Install `ingestron@2.2.0` and plan. Every pack is version 2, so a standard that
+1. Install `ingestron@2.3.0` and plan. Every pack is version 2, so a standard that
names `databricks@1`, `adf@1`, `synapse@1` or `duckdb@1` stops with
`PACK_VERSION`, once for all the environments that have it:
@@ -135,7 +135,7 @@ next; see [Deploy through environments](/docs/how-to/deploy-through-environments
[`DEPLOY_VALUE`](/docs/reference/errors#DEPLOY_VALUE): write it in
`project.yaml` for each environment. See
[Supplied when deploying](/docs/concepts/yaml-references#supplied-when-deploying).
-5. Change CI: the check templates install `ingestron@2.2.0`, and the deploy
+5. Change CI: the check templates install `ingestron@2.3.0`, and the deploy
pipelines deploy `build//` by target, parameter file or script, as
[Deploy through environments](/docs/how-to/deploy-through-environments)
shows.
@@ -151,28 +151,29 @@ given `--env`, they exit with `2`. `compare` and `request` still take `--env`.
resources in place, if the same identity deploys it. Run
`databricks bundle validate` and `databricks bundle summary` first.
- **Deploy as the identity that deployed 1.x.** Without
- `workspace.root_path` in the platform, a target's deployment state is in the
+ `databricks.workspace.root_path` in the platform, a target's deployment state is in the
deploying identity's own folder, `~/.bundle/…` in the workspace.
Another identity, such as the service principal of the new CI templates,
finds no state there and creates a second copy of every pipeline and job; in
`prod`, both copies run on schedule. `databricks bundle summary` shows the
folder a target uses. So run the first 2.0 deploy, and the schema steps
below, as the identity that deployed 1.x. Before deploys move to CI, set a
- shared `workspace.root_path` in `platform.yaml`, such as
+ shared `databricks.workspace.root_path` in `platform.yaml`, such as
`/Workspace/Shared/.bundle//${environment.name}`, and bind each
existing resource to it as the CI identity, as
[Deploy as one identity](/docs/platforms/databricks-deploy#one-identity)
shows.
- Only an environment named `dev` is in development mode; `test`, `uat`,
`staging` and every other environment are in production mode, unless the
- platform sets `mode`. On such an environment's first 2.0 deploy, jobs and
+ platform sets `databricks.mode`. On such an environment's first 2.0 deploy, jobs and
pipelines lose the `[dev ]` prefix from their names, pipelines
leave development, the deployment lock comes on, and production mode checks
the deploying identity's `run_as`, `permissions` and `root_path`. To keep
- development mode there, write `mode: ${values.mode}` in the platform,
+ development mode there, write `mode: ${values.mode}` in the platform's
+ `databricks:` section,
`mode: production` in `project.yaml`'s `values`, and `mode: development` in
that environment's `values`.
-- With `manage.schemas`, managed schemas are keyed by layer, such as `bronze`,
+- With `databricks.manage.schemas`, managed schemas are keyed by layer, such as `bronze`,
or by schema name when a layer has several, not `_`. Each
environment's deployment state still holds the old key, so before the first
2.0 deploy, for each environment and managed schema, as the identity that
@@ -351,6 +352,171 @@ extension works as before. Only code that calls the packages changes:
[`@ingestron/authoring` README](https://github.com/ingestron/ingestron/tree/main/packages/authoring)
shows an example.
+## Upgrade to 2.3 [#upgrade-to-2-3]
+
+2.3 generates the same files as 2.2 for a project that changes nothing,
+apart from a data product's passport, whose Quality and Who may use it
+sections are rewritten, and one fix on Databricks (below). It adds
+[intent](/docs/reference/intent),
+[published standards](/docs/how-to/choose-a-published-standard), layers that
+say what they do, checks on every engine, cleansing, accepted warnings, and
+grants, masks and row filters. It asks for some settings to be written in
+their new places, and some warnings have new codes:
+
+- **Settings that are intent.** `streaming-table`'s `mode`, `sequence_by`,
+ `delete_when` and `cluster_by`, `duckdb-table`'s `mode` and `sequence_by`,
+ and `adf-copy`'s `mode` and `watermark_column` mean the same as
+ [intent](/docs/reference/intent#settings-that-are-intent), such as
+ `history: latest` for `mode: upsert`. They still work, and `plan` reports
+ each as [`SETTING_MOVED`](/docs/reference/errors#SETTING_MOVED), a warning.
+ In a flow's or a flow table's `with`, its fix writes the intent and removes
+ the setting; anywhere else, such as the standard, write the intent by hand.
+- **The `databricks:` section.** The platform's `workspace`, `mode`, `run_as`,
+ `compute`, `permissions`, `manage` and `grants` now go under `databricks:`,
+ and `notifications.on_failure` becomes an email alert in `alerts`. At the
+ top they still work, with a
+ [`PLATFORM_FIELD_MOVED`](/docs/reference/errors#PLATFORM_FIELD_MOVED)
+ warning whose fix moves each; one written in both places with different
+ values is [`BAD_SETTING`](/docs/reference/errors#BAD_SETTING). See
+ [the `databricks:` section](/docs/reference/packs/databricks#platform).
+- **What each pack applies.** A value of intent the table's pack does not
+ apply is [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED), and with the
+ standard's `features: ga`, one that needs a feature in preview is
+ [`PREVIEW_FEATURE`](/docs/reference/errors#PREVIEW_FEATURE). Projects
+ written before 2.3 declare no intent, so they get neither.
+- **Published standards.** `ingestron init --kit` starts a project from one,
+ `ingestron standard install` moves a project to another version, and
+ `ingestron standard list` lists them; in VS Code, **New project** offers
+ **From a published standard**. Hover a `REPLACE_ME` to see where to find its
+ value.
+- **Quality and cleansing.** A layer can say what it does with its rows,
+ `does`; a table can cleanse its columns, keep one row per key, convert time
+ zones, and keep the rows a `drop` check leaves out; and a transform with one
+ input can leave out `code`. See
+ [What a layer does](/docs/concepts/layers#what-a-layer-does),
+ [Check data quality](/docs/how-to/add-quality-checks) and
+ [Cleanse data in the right layer](/docs/how-to/cleanse-data). All of it is
+ optional.
+- **DuckDB and Synapse check rows where a layer says so.** They run a table's
+ checks only in a layer whose `does` lists `check`, and count them, each as
+ `warn`, in a `raw` one. In a layer without `does` their files are what they
+ were, and each contract rule they do not run is a new warning,
+ [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED), with the hint to add
+ `check` to the layer's `does`. Expect these on the first plan with 2.3; see
+ [Not applied here](/docs/concepts/not-applied).
+- **Table checks on DuckDB and Synapse need nothing turned on.** Where a layer
+ checks, they count table checks and `unique` in their own run, whatever
+ `manage.checks` says. Only Databricks needs `databricks.manage.checks`.
+- **New codes for old warnings.**
+ - `CONTRACT_FIELD_UNUSED` is no longer reported. A contract field the pack
+ that keeps a table does not apply yet is `NOT_APPLIED` instead, judged for
+ each table against that pack, not against every pack of the standard.
+ - `QUALITY_RULE_NOT_CHECKED` for a check an engine cannot run, such as one
+ on a table that ends at the landed files, is `NOT_APPLIED`. It stays for a
+ rule no engine can check as written, such as a `custom` rule.
+ - A rule that counts over all rows while `databricks.manage.checks` is off
+ was `QUALITY_RULE_NOT_CHECKED`; it is
+ [`CAPABILITY_OFF`](/docs/reference/errors#CAPABILITY_OFF), with the fix
+ that turns checks on.
+ - A flow's `expectations` on a table that Lakeflow Connect writes were an
+ error; each is `NOT_APPLIED`, and so is each required column there.
+
+ A script that looks for an old code should look for the new one.
+
+- **Accepted warnings.** A warning you have decided to live with can be
+ accepted in `platform.yaml`, with the reason; see
+ [Accept a warning](/docs/how-to/accept-a-warning). An entry for
+ `CONTRACT_FIELD_UNUSED` is [`ACCEPT_CODE`](/docs/reference/errors#ACCEPT_CODE),
+ an error: accept `NOT_APPLIED` instead.
+- **Grants, masks and row filters.** A contract's `roles`, and a layer's
+ `access`, can be granted on each table; the standard's `governance` can
+ make grants exact, mask columns by their classification, and filter rows.
+ Databricks applies them after each refresh, with `databricks.manage.grants`,
+ `masks` and `filters`; Synapse grants each view, with
+ `synapse.manage.grants`. `principals` in `platform.yaml` says who each role
+ is. Each switch is off by default, so the files do not change until you
+ turn one on. See
+ [Grant access, mask columns and filter rows](/docs/how-to/grant-access).
+- **New warnings for roles.** A project whose contracts name `roles` gets a
+ warning for each role on the first plan with 2.3, and the same files:
+ - on Databricks or Synapse,
+ [`CAPABILITY_OFF`](/docs/reference/errors#CAPABILITY_OFF), whose fix
+ turns on `databricks.manage.grants` or `synapse.manage.grants`;
+ - on DuckDB, [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED): DuckDB
+ has no access control.
+
+ Masks and row filters, which only a standard written for 2.3 can give,
+ warn for each classified column they mask and each filter:
+ `CAPABILITY_OFF` on Databricks, and `NOT_APPLIED` on Synapse and DuckDB,
+ which cannot mask or filter.
+ Turn the capability on, or accept the warning with
+ `about: { feature: grants }` (or `write`, `exact`, `masks` or `filters`)
+ and the reason; see [Accept a gap](/docs/how-to/grant-access#accept-a-gap).
+ Either way, the passport's Who may use it says which roles are granted
+ and which are not enforced, and why.
+
+- **A role's `access` is checked where it is granted.** A role whose
+ `access` is neither `read` nor `write` is
+ [`ROLE_ACCESS`](/docs/reference/errors#ROLE_ACCESS), an error, only on a
+ table where its grant would be applied. A project that does not turn grants
+ on gets no new error.
+- **Tag keys Unity Catalog refuses.** With `databricks.manage.tags` on, a
+ contract tag key, or the classification's tag key, holding
+ `* < > % & ? \`, a control character, or a space at either end is now
+ [`PACK_CHECK`](/docs/reference/errors#PACK_CHECK), an error, as `. , - = / :`
+ already were. Unity Catalog refused such a key when the task ran; now
+ `plan` says so first. Rename the tag.
+- **A Databricks `run_as` user name with a dot.** `prerequisites/.sql`
+ wrote a `run_as` `user_name` such as `first.last@example.com` as two names,
+ `` `first`.`last@example.com` ``, which is not the user. It is now one
+ name, `` `first.last@example.com` ``, so that file changes for such a
+ project. A service principal's application ID has no dot, and is written as
+ before.
+- **A classified column that already has the tag key.** On Databricks, a
+ column whose contract `tags` already hold the classification's tag key, and
+ that has a `classification` too, was tagged with the key twice. The apply
+ task now sets it once, from the `classification`, so that task changes for
+ such a project.
+- **`ingestron request` lists each pack's switches.** A Synapse switch is read
+ from the platform's `synapse:` section and listed as
+ `synapse.manage.grants`; Databricks' list is unchanged. A name holding a
+ backtick is written as code with two backticks around it.
+
+To move a project to the new places, plan with 2.3.0, apply each fix (in
+VS Code, **Fix all** applies one kind at once), and plan again: the plan should
+list no change to the generated files, other than the passports. Then read
+what it lists under `Not applied on this platform`, and decide for each
+whether to give the layer a duty, change the definitions or accept it.
+
+For code that calls the packages:
+
+- `@ingestron/definitions` 1.13 adds intent: `intentPaths`, `intentUseOf` and
+ `intentCoverageGaps`; a pack's `intent`, and a method's `intent`,
+ `intentDefaults` and `aliases`; each table's `intent` and `intentOrigins`;
+ a pack's `platformSection` and `platformSectionReference`; and `find` and
+ `suggest` on each `FieldInfo`.
+- It also adds quality and cleansing: an expectation's `rule` (`RowRule`); a
+ table's `does`, `cleanse` (`TableCleanse` and `ColumnCleanse`), `timezone`,
+ `quality`, `qualityOrigins`, `quarantine` and `notApplied`; a coverage
+ entry's `onlyWhere` and `zonesNotApplied`; a diagnostic's `subject`; a
+ result's `accepted` and `acceptAtBuild`, and `acceptsWarning`; and
+ `severities` and `origin` in each code's help (`CodeHelp`).
+ `CONTRACT_FIELD_UNUSED` stays in `DefinitionCode`, no longer reported.
+- It also adds access: each table's `access` (`TableAccess`, with
+ `AccessGrant`, `ColumnMask`, `RowFilter` and `FilterMatch`), a serve step's
+ `notApplied`, the platform's `principals`, the `grants`, `masks` and
+ `filters` capabilities, `classificationTag` and `defaultClassificationTag`,
+ and the codes `ROLE_ACCESS`, `MASK_TYPE`, `FILTER_MATCH`, `FILTER_MULTIPLE`,
+ `MASK_NEEDS_RUN_AS`, `MASK_FUNCTIONS`, `POLICY_LIMIT` and
+ `POLICY_PRINCIPALS`.
+- `@ingestron/config` adds `matchesImport`, the one rule `imports` and an
+ accepted warning's `file` follow, and `@ingestron/core`'s `Report` gains
+ `accepted`.
+- `@ingestron/core` 1.1 adds published standards: `installedKits`,
+ `readKit`, `parseKitSpec` and `kitSchema`.
+- `@ingestron/authoring` 2.3 adds `ProjectSnapshot.replaceMeHelp`, what a
+ `REPLACE_ME` stands for, and lists a table's intent in its hover.
+
## What changed in a release
Recent releases are commits whose message starts with the version, such as
diff --git a/content/docs/index.mdx b/content/docs/index.mdx
index d76814e..69e9546 100644
--- a/content/docs/index.mdx
+++ b/content/docs/index.mdx
@@ -84,7 +84,7 @@ with each column cast to the type its contract gives.
The Databricks pack writes the `quality` example's readings table as a
streaming table in a Databricks pipeline, typed from its contract and checked
-by the rules its flow and the standard set:
+by the rules its contract, its flow and the standard set:
@@ -118,7 +118,9 @@ The platform team writes one standard: the packs, names, layers, types,
default settings, settings no project may change, and what every data product
must declare. VS Code marks a break as you type, `ingestron plan` reports
every one, and `ingestron check` fails the CI build. See
-[Set your organisation's standard](/docs/how-to/configure-a-standard).
+[Set your organisation's standard](/docs/how-to/configure-a-standard). Or
+start from a [published standard](/docs/how-to/choose-a-published-standard):
+one written for your platform, checked against its documentation and dated.
### Reuse your team's work
@@ -141,8 +143,10 @@ each one and what runs after each load.
## What Ingestron does not do
-- It doesn't connect to your sources or your cloud. Its only network call is
- to fetch a template or standard from a Git URL you give it.
+- It doesn't connect to your sources or your cloud. Its only network calls
+ use your own `git`: to fetch a template, a standard or a published standard
+ from a Git URL you give it, or a published standard that is not installed,
+ and to list the published standards' versions.
- It doesn't deploy anything. Your platform's tools do, from your CI/CD
pipeline.
- It doesn't read secret values. A `${secret:…}` reference carries only the
diff --git a/content/docs/platforms/adf-pipelines.mdx b/content/docs/platforms/adf-pipelines.mdx
index efc92f9..cd75202 100644
--- a/content/docs/platforms/adf-pipelines.mdx
+++ b/content/docs/platforms/adf-pipelines.mdx
@@ -85,6 +85,11 @@ platform's `schedule.paused`; see the
+The standard's [`operations`](/docs/reference/intent#operations) gives every
+ADF table that sets neither its `retries`, from `retries.count`, and its Copy
+activity's timeout, from `timeout`; see what the
+[ADF pack does with each intent value](/docs/reference/packs/adf#intent).
+
To stop only the copies, for example while the source database is down for
maintenance, set `paused: true` under `settings.adf-copy` in the platform,
beside the method's other settings, or under the method your flows use.
@@ -171,6 +176,11 @@ run values ADF fills in when it runs, or text:
+The `audit` [intent](/docs/reference/intent) adds the same kind of column in
+words every pack reads, such as `audit: [loaded_at, run_id]` on a flow or a
+layer: ADF adds those its copy can fill, each named `_`, and leaves the
+file values, such as `source_file`, to the pack that reads the files.
+
-The platform names the shared infrastructure the copies use, and the flow
-names the tables to copy from the `erp` connection, which reaches the ERP
-privately with its password in Key Vault:
+The platform names the shared infrastructure the copies use. The flow names
+the tables to copy from the `erp` connection, which reaches the ERP privately
+with its password in Key Vault, and the column each copy reads changed rows
+by, its `load: { cursor }`. The standard copies changed rows for every table,
+`load: { default: incremental }`, and its `bronze` layer keeps the latest row
+of each key, `history: latest`, in the order of the flow's `sequence`:
@@ -227,9 +230,10 @@ source its own landing folder.
Native table copies write a full selection each window. On the
`adf-databricks` route, repeated snapshots append repeated rows with the
-default store; choose [upsert or history](/docs/how-to/keep-a-table-current)
+default store; choose [`history: latest` or `full`](/docs/how-to/keep-a-table-current)
for current rows or change history. SQL-family copies, as in this starter,
-can copy only the rows changed in each window (`mode: incremental`). An
+can copy only the rows changed in each window: `load: incremental`, by the
+flow's `load: { cursor }`. An
unknown setting stops generation with a diagnostic, and the standard's fixed
settings still win.
diff --git a/content/docs/platforms/databricks-deploy.mdx b/content/docs/platforms/databricks-deploy.mdx
index 240bc8e..31fabe9 100644
--- a/content/docs/platforms/databricks-deploy.mdx
+++ b/content/docs/platforms/databricks-deploy.mdx
@@ -16,8 +16,12 @@ v0.264.0 or later, which applies preset tags to pipelines.
-With `manage` switches on, the request also says what each needs and gives the
-SQL to run once. See [Ask the platform team](/docs/how-to/ask-the-platform-team)
+With `databricks.manage` switches on, the request also says what each needs and gives the
+SQL to run once, `prerequisites/.sql`. Run it before the first
+deploy, except the lines it marks to run after: with masks or row filters on
+schemas the bundle creates, `MANAGE` on those schemas can be granted only once
+the first deploy has created them. Those lines are not needed when `run_as`
+deploys and so owns the schemas. See [Ask the platform team](/docs/how-to/ask-the-platform-team)
for the whole request.
## Deploy [#deploy]
@@ -39,8 +43,8 @@ not start the ADF copies.
### Deploy as one identity [#one-identity]
A target's deployment state, which records the pipelines and jobs it created,
-is kept in the target's `root_path`. Without `workspace.root_path` in the
-platform, that is the deploying identity's own folder,
+is kept in the target's `root_path`. Without `databricks.workspace.root_path`
+in the platform, that is the deploying identity's own folder,
`~/.bundle//` in the workspace. Another identity
deploying the same target finds no state there and creates a second copy of
every pipeline and job; in `prod`, both copies then run on their schedule.
@@ -49,7 +53,7 @@ So deploy each environment as one identity, such as the service principal your
pipeline uses, and give every target a shared `root_path` in `platform.yaml`
before a second identity deploys it, as the `environments` example does:
-
+
A target that is already deployed under another `root_path` has its state
there. Before the first deployment with the new path, bind each of its
diff --git a/content/docs/platforms/databricks.mdx b/content/docs/platforms/databricks.mdx
index 125f4c9..fabe722 100644
--- a/content/docs/platforms/databricks.mdx
+++ b/content/docs/platforms/databricks.mdx
@@ -9,7 +9,7 @@ tables. The Databricks pack writes one
every environment, with a target per environment: Lakeflow pipelines with
their SQL and Python files, Lakeflow Connect ingestion for databases, and a
refresh job when the platform has a schedule or a contract task (tags, column
-comments or table checks) runs after the refresh. The same bundle reads the
+comments, table checks, grants, masks or row filters) runs after the refresh. The same bundle reads the
files ADF lands on the `adf-databricks` route;
[Azure Data Factory](/docs/platforms/adf) covers the copies.
@@ -177,9 +177,9 @@ instead, each one names its file, line and fix; see
## Settings that shape it [#settings]
-The platform document decides the targets: `workspace`, `run_as`,
-`permissions` and `mode` go into each target, and `tags` that differ become
-the target's preset tags. The defaults:
+The platform document decides the targets: the `workspace`, `run_as`,
+`permissions` and `mode` of its `databricks:` section go into each target, and
+`tags` that differ become the target's preset tags. The defaults:
@@ -206,9 +206,19 @@ before the first deploy. They are not deployed:
-Only set `manage.schemas` when no other project uses the schemas: deploying
+Only set `databricks.manage.schemas` when no other project uses the schemas: deploying
the bundle can change them, and `databricks bundle destroy` deletes them.
+### Grant, mask and filter [#access]
+
+With `databricks.manage.grants`, the task after each refresh grants each
+table to the roles its contract and its layer name, and with exact grants
+takes away what they do not give. With `databricks.manage.masks` and
+`databricks.manage.filters`, it masks classified columns and filters rows
+with Unity Catalog policies, for everyone but `run_as` and the principals
+you list. See
+[Grant access, mask columns and filter rows](/docs/how-to/grant-access).
+
### Refresh on a schedule [#schedule]
A `schedule` in the platform, with a
@@ -232,6 +242,13 @@ platform's `schedule` stops both the job and the Data Factory triggers; to
stop only the copies, see
[Retry and pause](/docs/platforms/adf-pipelines#retry-and-pause).
+The standard's [`operations`](/docs/reference/intent#operations) says how the
+job runs: how often a failed refresh is tried again, how long a run may take,
+and whether a run that starts while another is running waits. Who is told when
+a run fails is the platform's `alerts`, as in `platform-setup` above. The
+[streaming lakehouse](/docs/how-to/kits/databricks-lakehouse-streaming)
+published standard sets each of them.
+
The tables themselves are shaped by the flows and the standard:
[Keep a table current](/docs/how-to/keep-a-table-current),
[Write a transform](/docs/how-to/write-a-transform) and
@@ -262,7 +279,9 @@ as one identity, and runs the refresh.
- **What has been tried.** The screenshots show the demo deployed with
ingestron 1.8, which wrote a bundle per environment. The 2.0 bundle, one for
every environment, validates against the Databricks CLI's bundle schema
- (v1.23.0), but has not yet been deployed to a live workspace.
+ (v1.23.0), but has not yet been deployed to a live workspace. Nor have the
+ grants, masks and row filters: their SQL follows the Databricks pages the
+ [pack's reference](/docs/reference/packs/databricks#access) links.
## Next
@@ -271,4 +290,5 @@ as one identity, and runs the refresh.
- [Add a source and its tables](/docs/how-to/add-a-source): another table, or
another source, in the same project.
- [Databricks pack](/docs/reference/packs/databricks): every setting, the
- platform defaults and the `manage` switches.
+ platform's `databricks:` section and its defaults, the `manage` switches,
+ and what the pack does with each intent value and each part of access.
diff --git a/content/docs/platforms/duckdb-in-azure.mdx b/content/docs/platforms/duckdb-in-azure.mdx
index 02fb560..d8f88a9 100644
--- a/content/docs/platforms/duckdb-in-azure.mdx
+++ b/content/docs/platforms/duckdb-in-azure.mdx
@@ -50,7 +50,9 @@ factory's managed identity:
The job's `run.sql` reads and writes `abfss://` paths only, so it runs in
Azure. It reads each table's landed file, `abfss://landing@…`, at the end of
its first line, and publishes each table to `abfss://published@…` at the end
-of the last ones:
+of the last ones. `bronze` keeps its rows as they arrived, so the run only
+counts its check of the required `invoice_id`, and prints the count to the
+job's log:
-- **No rows are checked yet.** DuckDB tables check no rows yet. A flow's
- `expectations` are an error here, so leave them out; see
- [Check data quality](/docs/how-to/add-quality-checks).
+- **Rows are checked only where a layer says so.** DuckDB checks rows only in
+ a layer whose `does` lists `check`, and counts them in a `raw` one. In any
+ other layer a flow's `expectations` are an error, and each contract rule is
+ [not applied here](/docs/concepts/not-applied); see
+ [Check data quality](/docs/how-to/add-quality-checks#duckdb). Table checks
+ need nothing turned on in the platform.
- **A local run starts after the files land.** It finds mistakes in contracts,
types, modes and transforms, but not failures inside ADF: connectors,
credentials, networking, the self-hosted integration runtime, or the source
diff --git a/content/docs/platforms/landing-only.mdx b/content/docs/platforms/landing-only.mdx
index b37d12d..ec79b6f 100644
--- a/content/docs/platforms/landing-only.mdx
+++ b/content/docs/platforms/landing-only.mdx
@@ -147,9 +147,11 @@ Troubleshooting.
-- **No rows are checked.** Nothing reads the files before your reports do, so
- each contract quality rule and flow expectation on these tables is reported
- as not checked (`QUALITY_RULE_NOT_CHECKED`); see
+- **No rows are checked or cleansed.** Nothing reads the files before your
+ reports do, so each contract quality rule and flow expectation on these
+ tables is [not applied here](/docs/concepts/not-applied)
+ ([`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED)), with the hint to add a
+ store step, and so is any cleansing; see
[Check data quality](/docs/how-to/add-quality-checks).
- **No transforms.** A transform cannot read a table that ends at the landed
files: give its flow a store step first.
diff --git a/content/docs/platforms/synapse.mdx b/content/docs/platforms/synapse.mdx
index 098ac12..675abd1 100644
--- a/content/docs/platforms/synapse.mdx
+++ b/content/docs/platforms/synapse.mdx
@@ -15,7 +15,10 @@ queries a view.
The ADF pack writes the factory, as on
[Azure Data Factory](/docs/platforms/adf#what-you-get), and the Synapse pack
writes one script of views for every environment, with a short script for each
-environment that sets its values:
+environment that sets its values. Where a table's layer checks its rows, it
+also writes `checks.sql`, with `checks/.sql` for each
+environment, which counts the checks when it runs; see
+[Run and operate](#operate):
@@ -104,11 +107,13 @@ finished project, which you can download:
The standard lists Synapse first, so it names and types the views (see
[Combine packs](/docs/concepts/standards-and-packs#combine-packs-when-their-actions-fit)),
-and keeps every table as a view:
+keeps every table as a view, and says what each layer does with its rows
+([What a layer does](/docs/concepts/layers#what-a-layer-does)):
-The platform names the external data source the views read through:
+The platform names the external data source the views read through, turns
+on grants, and says which database user each role is:
@@ -144,6 +149,23 @@ schedule, runtime and landing settings are on
SQL can do is in
[What each engine makes of it](/docs/how-to/write-a-transform#what-each-engine-makes-of-it).
+### Who may read the views [#grants]
+
+With `synapse.manage.grants`, `views.sql` grants each view to the roles its
+contract and its layer name, right after the view, in its own database:
+
+
+
+Serverless SQL has no data masking or row-level security, so a masked column
+or a row filter is [not applied here](/docs/concepts/not-applied). See
+[On Synapse](/docs/how-to/grant-access#on-synapse) for exact grants and what
+each reader needs.
+
### Views or tables
A view cannot merge changes or keep history. For that, keep the table with a
@@ -162,8 +184,8 @@ store pack and serve it with `synapse-serve`; see
Ingestron creates nothing in Synapse. `ingestron request --env dev` lists what
the team that owns the workspace creates first, with this project's names: the
serverless database, the external data source with a managed-identity
-credential, the workspace's read access to the storage, and report users'
-permissions.
+credential, the workspace's read access to the storage, and the database user
+each role is, with the permissions views over files need.
.sql` for each environment. It counts
+every check's failing rows, returns one row per check, and raises an error
+when a `fail` check fails. A view reads the files each time it is queried, so
+nothing runs the checks by itself: run the script after each landing, from
+`build/synapse`, here for `dev`:
+
+```sh
+sqlcmd -S -ondemand.sql.azuresynapse.net --authentication-method ActiveDirectoryDefault -i checks/dev.sql
+```
+
+or from a pipeline's Script activity, with each `$(…)` replaced by the
+environment's value, or as a step in CI. See
+[Check data quality](/docs/how-to/add-quality-checks#synapse).
+
When a view fails to create or to read its files, see
[Synapse serverless views](/docs/troubleshooting#synapse-serverless-views) in
Troubleshooting.
## Limits [#limits]
-- **No rows are checked.** A view checks no rows; see
- [Check data quality](/docs/how-to/add-quality-checks).
+- **Checks only where a layer checks, and no `fail`.** In a layer whose `does`
+ lists `check`, the view leaves out the rows a `drop` check fails, and
+ `checks.sql` counts every check, but only when something runs it. A view
+ cannot stop a load, keep the rows it leaves out, or keep results, and
+ serverless T-SQL has no regular expressions: each is
+ [not applied here](/docs/concepts/not-applied). See
+ [Check data quality](/docs/how-to/add-quality-checks#synapse).
- **No merge or history.** A view shows the files as they are; see
[Views or tables](#views-or-tables).
+- **Grants, but no masks or row filters.** A reader who may query a view over
+ files can also read the files under the data source directly.
- **Environments may differ only in values.** A table built in some
environments only, or a transform whose code differs in more than a value,
is [`ENVIRONMENTS_DIFFER`](/docs/reference/errors#ENVIRONMENTS_DIFFER). A
diff --git a/content/docs/start/first-project.mdx b/content/docs/start/first-project.mdx
index c5a711d..c96e980 100644
--- a/content/docs/start/first-project.mdx
+++ b/content/docs/start/first-project.mdx
@@ -84,6 +84,11 @@ Run **Ingestron: Plan**. The output appears in the **Ingestron** output panel.
You should see:
```text
+Not applied on this platform
+dev:
+contracts/orders.yaml:11:59 warning NOT_APPLIED
+ The notNull rule isn't applied on duckdb: it is applied only in a layer whose does lists check.
+ Add `check` to the layer's `does` to apply it here.
duckdb → build/duckdb (dev)
create dev/README.md
create dev/run.sql
@@ -97,6 +102,15 @@ Each platform's files go to one folder, here `build/duckdb/`, with a folder
for each environment that DuckDB runs: `dev` is the project's one environment.
Plan writes nothing: it shows what `apply` would write.
+The warning above is about the contract you meet in step 5. It says every
+order has an `order_id` (`required: true`), and DuckDB checks rows only in a
+layer whose `does` says it checks them. This starter's layers say nothing
+about what they do, so DuckDB builds the tables without that check, and every
+command says so. A warning never stops a run.
+[Not applied here](/docs/concepts/not-applied) explains these warnings, and
+[Cleanse data in the right layer](/docs/how-to/cleanse-data) starts from a
+standard whose layers check their rows.
+
## 3. Apply
@@ -124,6 +138,11 @@ file generated for it and its full name:
```text
+Not applied on this platform
+dev:
+contracts/orders.yaml:11:59 warning NOT_APPLIED
+ The notNull rule isn't applied on duckdb: it is applied only in a layer whose does lists check.
+ Add `check` to the layer's `does` to apply it here.
duckdb → build/duckdb (dev)
create dev/README.md
create dev/run.sql
@@ -251,6 +270,11 @@ Plan shows each changed line: the `channel` column is read, cast and
published. A reviewer sees the same change in a pull request.
```text
+Not applied on this platform
+dev:
+contracts/orders.yaml:11:59 warning NOT_APPLIED
+ The notNull rule isn't applied on duckdb: it is applied only in a layer whose does lists check.
+ Add `check` to the layer's `does` to apply it here.
duckdb → build/duckdb (dev)
update dev/run.sql
@@ -5,5 +5,5 @@
@@ -313,6 +337,11 @@ ingestron plan
```
```text
+Not applied on this platform
+dev:
+contracts/orders.yaml:11:59 warning NOT_APPLIED
+ The notNull rule isn't applied on duckdb: it is applied only in a layer whose does lists check.
+ Add `check` to the layer's `does` to apply it here.
duckdb → build/duckdb (dev)
conflict dev/run.sql
Changed by hand since it was generated. Move the change into the definitions, or delete the file to generate it again.
@@ -330,6 +359,11 @@ ingestron apply
```
```text
+Not applied on this platform
+dev:
+contracts/orders.yaml:11:59 warning NOT_APPLIED
+ The notNull rule isn't applied on duckdb: it is applied only in a layer whose does lists check.
+ Add `check` to the layer's `does` to apply it here.
duckdb → build/duckdb (dev)
create dev/run.sql
diff --git a/content/docs/start/how-it-works.mdx b/content/docs/start/how-it-works.mdx
index c6f9780..4df28f0 100644
--- a/content/docs/start/how-it-works.mdx
+++ b/content/docs/start/how-it-works.mdx
@@ -57,7 +57,7 @@ environment's files in it, laid out as that platform's CI/CD tools expect:
- **[DuckDB](/docs/platforms/duckdb#what-you-get):** for each environment,
`run.sql`, and each table's SQL on its own.
- **Data products:** for each environment, a passport for each product, in
- Markdown: who owns it, its columns and its lineage.
+ Markdown: who owns it, its columns, who may use it and its lineage.
Only what differs between environments varies, so a reviewer sees the
difference between dev and prod in one place, and a deployment pipeline
@@ -108,7 +108,15 @@ from then on; see
- **What it holds:** the packs, and naming patterns, layers and column types.
`settings` gives defaults, `allowed` limits the methods a project may use,
`fixed` holds settings no project may change, and `governance` says what
- every data product must declare.
+ every data product must declare, and how columns are masked and rows
+ filtered; see
+ [Grant access, mask columns and filter rows](/docs/how-to/grant-access).
+ [Intent](/docs/reference/intent) says how
+ tables are loaded, kept and run, in words every pack reads.
+- **Or a published one.** `ingestron init --kit @`
+ starts a project from a standard written for one platform, checked and
+ dated, with a small project that builds; see
+ [Choose a published standard](/docs/how-to/choose-a-published-standard).
- **The project overrides only what the standard leaves open.** A flow that
changes a fixed setting reports `FIXED_SETTING`, and nothing is generated.
[Which setting wins](/docs/concepts/settings) explains the order.
@@ -125,8 +133,10 @@ from then on; see
## What it never does
-- It doesn't connect to your sources or your cloud. Its only network call is
- to fetch a template or standard from a Git URL you give it.
+- It doesn't connect to your sources or your cloud. Its only network calls
+ use your own `git`: to fetch a template, a standard or a published standard
+ from a Git URL you give it, or a published standard that is not installed,
+ and to list the published standards' versions.
- It doesn't deploy anything. Your platform's tools do.
- It doesn't read secret values. A `${secret:…}` reference carries only the
secret's name; see [YAML references](/docs/concepts/yaml-references#reference-a-secret).
diff --git a/examples/change-data/README.md b/examples/change-data/README.md
index df37345..b3cb86f 100644
--- a/examples/change-data/README.md
+++ b/examples/change-data/README.md
@@ -1,15 +1,17 @@
# Change data
The source files hold one row per change: a key, a time and an operation.
+The flow orders each key's changes by `changed_at`, with `sequence`.
-- `accounts` uses `mode: upsert`: [the table](build/databricks/src/crm/bronze/accounts.sql)
- keeps only the current row per key (`STORED AS SCD TYPE 1`); rows whose
- `operation` is `DELETE` remove the key.
-- `contacts` uses `mode: history`: [the table](build/databricks/src/crm/bronze/contacts.sql)
+- `accounts` keeps `history: latest`: [the table](build/databricks/src/crm/bronze/accounts.sql)
+ keeps only the current row per key (`STORED AS SCD TYPE 1`); a change whose
+ `operation` is `DELETE` removes the key, by `load.deletes.when`.
+- `contacts` keeps `history: full`: [the table](build/databricks/src/crm/bronze/contacts.sql)
keeps every version (`STORED AS SCD TYPE 2`), keyed by two columns in
`primaryKeyPosition` order.
Each also gets a `
_changes` streaming table with the records as
-received. Keys come from the contract's `primaryKey` columns; `sequence_by`,
-`delete_when` and `except_columns` are settings of the `streaming-table`
-method. Without a key or `sequence_by`, loading fails with `METHOD_CHECK`.
+received. Keys come from the contract's `primaryKey` columns. `history`,
+`sequence` and `load.deletes.when` are intent; `except_columns`, the columns
+left out of the kept table, is a setting of the `streaming-table` method.
+Without a key or a `sequence`, loading fails with `METHOD_CHECK`.
diff --git a/examples/change-data/flows/crm.yaml b/examples/change-data/flows/crm.yaml
index 17aad4b..ead942b 100644
--- a/examples/change-data/flows/crm.yaml
+++ b/examples/change-data/flows/crm.yaml
@@ -4,22 +4,20 @@ id: crm
source: crm
ingest:
with: { format: json }
+sequence: changed_at # the newest change to a key wins
tables:
accounts:
object: accounts
contract: crm.accounts
+ # Only the current version of each account.
+ history: latest
+ load: { deletes: { when: "operation = 'DELETE'" } }
store:
- # Only the current version of each account.
- with:
- {
- mode: upsert,
- sequence_by: changed_at,
- delete_when: "operation = 'DELETE'",
- except_columns: [operation],
- }
+ with: { except_columns: [operation] }
contacts:
object: contacts
contract: crm.contacts
+ # Every version, with __START_AT and __END_AT.
+ history: full
store:
- # Every version, with __START_AT and __END_AT.
- with: { mode: history, sequence_by: changed_at, except_columns: [operation] }
+ with: { except_columns: [operation] }
diff --git a/examples/connection-first-adf/sample.yaml b/examples/connection-first-adf/sample.yaml
index 5bcb150..92287ff 100644
--- a/examples/connection-first-adf/sample.yaml
+++ b/examples/connection-first-adf/sample.yaml
@@ -1,3 +1,3 @@
title: Connection-first ADF project
summary: One PostgreSQL connection, two packs, one standard and configurable ADF names.
-order: 12
+order: 13
diff --git a/examples/connection-first-files/sample.yaml b/examples/connection-first-files/sample.yaml
index 6f84cd7..ffe5d59 100644
--- a/examples/connection-first-files/sample.yaml
+++ b/examples/connection-first-files/sample.yaml
@@ -1,3 +1,3 @@
title: Connection-first files project
summary: Create a files-to-Databricks project through the connection-first workflow.
-order: 11
+order: 12
diff --git a/examples/data-products/README.md b/examples/data-products/README.md
index 25346a1..84af2ed 100644
--- a/examples/data-products/README.md
+++ b/examples/data-products/README.md
@@ -2,7 +2,8 @@
The web shop's orders and customers are copied from Azure SQL by ADF, and
DuckDB builds the tables (`packs: [duckdb@2, adf@2]`), as in the DuckDB
-example. On top, two **data products**, each an Open Data Product Standard
+example: `bronze` keeps the rows as they arrived and only counts its checks,
+and `silver` checks its rows. On top, two **data products**, each an Open Data Product Standard
file in `products/`:
- `customers` publishes the customers table (contract `shop.customers`), owned
@@ -32,5 +33,14 @@ which are sensitive, its quality checks, how fresh it is, who may use it, its
lineage as a diagram, and the governance rules it meets. It is generated with
the pipelines, so it is always up to date.
+DuckDB has no users or permissions, so it cannot grant `marketing-analysts`
+access to `customers`. Planning says so, as a `NOT_APPLIED` warning on the
+contract's role. The platform team has decided to live with it: who may read
+the published files is set on the storage account. So `platform.yaml` accepts
+the warning, with that reason, in its `accept` list. The plan then prints
+`1 accepted warning (platform.yaml)` in place of the warning. Accepting a
+warning does not grant anything: the passport's "Who may use it" still says
+`Not enforced on DuckDB: DuckDB has no access control.`
+
The `local` environment runs the same tables on a laptop with DuckDB, from a
load's files in `./samples/`, and gets its own passports.
diff --git a/examples/data-products/platform.yaml b/examples/data-products/platform.yaml
index dcf6663..20f1f8e 100644
--- a/examples/data-products/platform.yaml
+++ b/examples/data-products/platform.yaml
@@ -10,3 +10,7 @@ settings:
job: ${values.duckdb_job}
duckdb-table:
published: ${values.published}
+accept: # warnings the platform team has looked at, and why it lives with them
+ - code: NOT_APPLIED
+ about: { feature: grants }
+ reason: DuckDB has no access control. Who may read the published files is set on the storage account.
diff --git a/examples/data-products/standard.yaml b/examples/data-products/standard.yaml
index 9845c18..61dfe24 100644
--- a/examples/data-products/standard.yaml
+++ b/examples/data-products/standard.yaml
@@ -2,6 +2,10 @@ apiVersion: ingestron.io/v1
kind: Standard
id: shop
packs: [duckdb@2, adf@2] # DuckDB first: the first pack names and types the tables; ADF only copies
+layers:
+ - { name: bronze, does: raw } # every row as it arrived; its checks only count
+ - { name: silver, does: [check] }
+ - { name: gold, does: [check] }
ingest:
default: adf-copy # ADF copies from the source; DuckDB builds the tables
settings:
diff --git a/examples/duckdb-local/README.md b/examples/duckdb-local/README.md
index a3c0c4d..86fa299 100644
--- a/examples/duckdb-local/README.md
+++ b/examples/duckdb-local/README.md
@@ -4,7 +4,9 @@ ADF copies the ERP's invoices into storage, and DuckDB loads them into a table,
runs a transform and publishes both as Parquet: `packs: [duckdb@2, adf@2]` in
`standard.yaml`. Each load copies the whole table into one file,
`erp/erp.dbo.Invoices/erp.dbo.Invoices.parquet`, replacing the last one
-(`layout: snapshot`), so the table is always the latest load.
+(`layout: snapshot`), so the table is always the latest load. The standard's
+`layers` say what each does: `bronze` keeps the rows as they arrived and only
+counts its checks, and `silver` and `gold` check theirs.
The `local` environment names `runs-on: duckdb`, so it runs the same tables on
a laptop: ADF's copy becomes `duckdb-landed`, which reads a load's files from
@@ -16,7 +18,8 @@ a laptop: ADF's copy becomes `duckdb-landed`, which reads a load's files from
that runs it and then the DuckDB job, the parent's trigger, linked services
and datasets, and `parameters/dev.json`;
- `build/duckdb//run.sql`: every table in order, the same SQL in
- both environments apart from where the files are.
+ both environments apart from where the files are. It counts and prints the
+ invoices with no `invoice_id`, the contract's required column.
In `dev`, ADF's `after_copy` starts the Container Apps job `sales-dev-duckdb`
after each schedule's copies: the parent pipeline, `pl_sales_every_24_hour`,
diff --git a/examples/duckdb-local/standard.yaml b/examples/duckdb-local/standard.yaml
index 736186d..9b7315e 100644
--- a/examples/duckdb-local/standard.yaml
+++ b/examples/duckdb-local/standard.yaml
@@ -2,6 +2,10 @@ apiVersion: ingestron.io/v1
kind: Standard
id: sales
packs: [duckdb@2, adf@2] # DuckDB first: the first pack names and types the tables; ADF only copies
+layers:
+ - { name: bronze, does: raw } # every row as it arrived; its checks only count
+ - { name: silver, does: [check] }
+ - { name: gold, does: [check] }
ingest:
settings:
adf-copy:
diff --git a/examples/environments/platform.yaml b/examples/environments/platform.yaml
index 3a21751..c9ff292 100644
--- a/examples/environments/platform.yaml
+++ b/examples/environments/platform.yaml
@@ -3,11 +3,12 @@
apiVersion: ingestron.io/v1
kind: Platform
id: lakehouse
-workspace:
- host: ${values.host}
- root_path: /Workspace/Shared/.bundle/finance/${environment.name}
schedule:
cron: 0 30 6 * * ?
timezone: Europe/London
tags:
environment: ${environment.name}
+databricks:
+ workspace:
+ host: ${values.host}
+ root_path: /Workspace/Shared/.bundle/finance/${environment.name}
diff --git a/examples/governed-databricks/README.md b/examples/governed-databricks/README.md
new file mode 100644
index 0000000..62eef3c
--- /dev/null
+++ b/examples/governed-databricks/README.md
@@ -0,0 +1,49 @@
+# Grant, mask and filter on Databricks
+
+ADF copies a web shop's customers and orders from Azure SQL, and Databricks
+keeps the tables (`packs: [databricks@2, adf@2]`). Each table says who may
+read it, the sensitive columns are masked, and each reader sees only the rows
+of their own region.
+
+Who may read a table:
+
+- Each contract's `roles` name who reads its tables. `regional_sales`, in
+ `silver`, is read by `finance-analysts` and `marketing-analysts`.
+- A layer can change that. `bronze` holds the raw rows, so it grants none of
+ its contracts' roles (`contract: false`) and grants `data-engineers` on
+ every table in it. `marketing-analysts`, the role of `customers`, is not
+ granted there.
+- `principals` in `platform.yaml` says which group each role is in each
+ environment, from the values in `project.yaml`, such as
+ `shop-marketing-analysts-dev` in `dev`.
+- `governance.access.grants: exact` in the standard takes away any other
+ privilege granted directly on these tables. The run_as service principal
+ and `shop-data-stewards` keep theirs.
+
+What readers see:
+
+- `email` is `confidential`, so readers see a hash of it; `phone` is
+ `restricted`, so readers see no value (`governance.masks`).
+- `region` is tagged `region` in each contract, so the filter `region_rows`
+ passes it to the team's own function `region_filter`, which says which rows
+ each reader sees (`governance.filters`).
+- The run_as service principal and `shop-data-stewards`
+ (`databricks.governance.unmasked`) see every value and every row.
+
+Each of these is switched on in the platform's `databricks.manage`: `grants`,
+`masks`, `filters`, and `tags`, which masks and filters match.
+`manage.schemas` is on too, so the bundle creates the schemas and the
+policies go on each schema.
+
+`ingestron apply` writes `build/databricks/`. After each refresh,
+`tasks/shop/apply_contract.py` sets the tags, creates the mask functions in
+`dev_governance.functions` (`prod_governance.functions` in `prod`), makes the
+policies, then grants and revokes, so no one it grants reads real values if a
+policy cannot be made. `prerequisites/.sql` gives the
+run_as service principal what that needs, and `ingestron request --env dev`
+lists what must exist first: each group, the function `region_filter`, and
+the governed tags `classification` and `region`.
+
+The masks and filters apply to the tables, not to the files ADF lands:
+anyone who can read the landing container reads `email` and `phone` in full.
+Set access to the landed files on the storage account.
diff --git a/examples/governed-databricks/connections/shop.yaml b/examples/governed-databricks/connections/shop.yaml
new file mode 100644
index 0000000..0d95dfc
--- /dev/null
+++ b/examples/governed-databricks/connections/shop.yaml
@@ -0,0 +1,10 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: shop
+description: The web shop's database, in Azure SQL.
+connector: azure-sql-database
+connectivity: public
+settings:
+ server: shop-sql.database.windows.net
+ user: shop_reader
+ password: ${secret:shop-sql-password} # a Key Vault secret name
diff --git a/examples/governed-databricks/contracts/customers.yaml b/examples/governed-databricks/contracts/customers.yaml
new file mode 100644
index 0000000..9d4f6ef
--- /dev/null
+++ b/examples/governed-databricks/contracts/customers.yaml
@@ -0,0 +1,18 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: shop.customers
+version: 1.0.0
+status: active
+schema:
+ - name: customers
+ logicalType: object
+ description: Everyone who has bought from the web shop.
+ properties:
+ - { name: id, logicalType: integer, primaryKey: true, classification: internal }
+ - { name: email, logicalType: string, required: true, classification: confidential }
+ - { name: phone, logicalType: string, classification: restricted }
+ - { name: region, logicalType: string, classification: internal, tags: [region] }
+roles:
+ - role: marketing-analysts
+ access: read
+ description: Campaign reporting by region
diff --git a/examples/governed-databricks/contracts/orders.yaml b/examples/governed-databricks/contracts/orders.yaml
new file mode 100644
index 0000000..3c6b16c
--- /dev/null
+++ b/examples/governed-databricks/contracts/orders.yaml
@@ -0,0 +1,15 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: shop.orders
+version: 1.0.0
+status: active
+schema:
+ - name: orders
+ logicalType: object
+ description: Orders placed in the web shop.
+ properties:
+ - { name: order_id, logicalType: integer, primaryKey: true }
+ - { name: customer_id, logicalType: integer, required: true }
+ - { name: region, logicalType: string, tags: [region] }
+ - { name: amount, logicalType: number, physicalType: "DECIMAL(18,2)" }
+ - { name: ordered_at, logicalType: timestamp, required: true }
diff --git a/examples/governed-databricks/contracts/regional_sales.yaml b/examples/governed-databricks/contracts/regional_sales.yaml
new file mode 100644
index 0000000..0c21f11
--- /dev/null
+++ b/examples/governed-databricks/contracts/regional_sales.yaml
@@ -0,0 +1,21 @@
+apiVersion: v3.1.0
+kind: DataContract
+id: shop.regional_sales
+version: 1.0.0
+status: active
+schema:
+ - name: regional_sales
+ logicalType: object
+ description: Revenue and orders per day and region.
+ properties:
+ - { name: order_day, logicalType: date, primaryKey: true }
+ - { name: region, logicalType: string, primaryKey: true, tags: [region] }
+ - { name: orders, logicalType: integer }
+ - { name: revenue, logicalType: number, physicalType: "DECIMAL(18,2)" }
+roles:
+ - role: finance-analysts
+ access: read
+ description: Revenue reporting
+ - role: marketing-analysts
+ access: read
+ description: Campaign results by region
diff --git a/examples/governed-databricks/flows/shop.yaml b/examples/governed-databricks/flows/shop.yaml
new file mode 100644
index 0000000..d9687ab
--- /dev/null
+++ b/examples/governed-databricks/flows/shop.yaml
@@ -0,0 +1,8 @@
+apiVersion: ingestron.io/v1
+kind: Flow
+id: shop
+description: Customers and orders from the web shop.
+source: shop
+tables:
+ customers: { object: shop.dbo.Customers, contract: shop.customers }
+ orders: { object: shop.dbo.Orders, contract: shop.orders }
diff --git a/examples/governed-databricks/platform.yaml b/examples/governed-databricks/platform.yaml
new file mode 100644
index 0000000..d623294
--- /dev/null
+++ b/examples/governed-databricks/platform.yaml
@@ -0,0 +1,26 @@
+apiVersion: ingestron.io/v1
+kind: Platform
+id: azure
+principals: # the group each contract and layer role is, per environment
+ marketing-analysts: ${values.marketing}
+ finance-analysts: ${values.finance}
+ data-engineers: ${values.engineers}
+settings:
+ adf-copy: # shared infrastructure, referenced by name
+ landing: ${values.landing}/shop
+ key_vault: ${values.key_vault}
+ integration_runtime: AutoResolveIntegrationRuntime
+databricks:
+ workspace:
+ host: ${values.host}
+ run_as: # the pipelines run as this service principal, which always sees real values
+ service_principal_name: 00000000-0000-0000-0000-000000000001
+ manage:
+ schemas: true # the bundle creates the schemas, so policies go on each schema
+ tags: true # set each column's classification and tags, which the policies match
+ grants: true # grant each table's roles after each refresh
+ masks: true # mask the columns whose classification the standard masks
+ filters: true # filter the rows of the tables with a region column
+ governance:
+ functions: ${values.functions}
+ unmasked: [shop-data-stewards] # also sees real values
diff --git a/examples/governed-databricks/project.yaml b/examples/governed-databricks/project.yaml
new file mode 100644
index 0000000..5558a6d
--- /dev/null
+++ b/examples/governed-databricks/project.yaml
@@ -0,0 +1,30 @@
+# ADF copies a web shop's customers and orders from Azure SQL, and Databricks
+# keeps the tables. Each table says who may read it, sensitive columns are
+# masked, and each reader sees only their region's rows.
+id: shop
+environments:
+ dev: {}
+ prod:
+ values:
+ landing: abfss://landing@shopprod.dfs.core.windows.net
+ key_vault: https://kv-shop-prod.vault.azure.net
+ host: https://adb-2222222222222222.2.azuredatabricks.net
+ functions: prod_governance.functions
+ marketing: shop-marketing-analysts
+ finance: shop-finance-analysts
+ engineers: shop-data-engineers
+values:
+ landing: abfss://landing@shopdev.dfs.core.windows.net
+ key_vault: https://kv-shop-dev.vault.azure.net
+ host: https://adb-1111111111111111.1.azuredatabricks.net
+ functions: dev_governance.functions # the schema that holds the mask and filter functions
+ marketing: shop-marketing-analysts-dev # the group each role is, per environment
+ finance: shop-finance-analysts-dev
+ engineers: shop-data-engineers-dev
+imports:
+ - standard.yaml
+ - platform.yaml
+ - connections/*.yaml
+ - flows/*.yaml
+ - transforms/*.yaml
+ - contracts/*.yaml
diff --git a/examples/governed-databricks/sample.yaml b/examples/governed-databricks/sample.yaml
new file mode 100644
index 0000000..ee6a351
--- /dev/null
+++ b/examples/governed-databricks/sample.yaml
@@ -0,0 +1,4 @@
+# A guide: a small project that shows one idea; the docs embed it.
+title: Grant, mask and filter on Databricks
+summary: Contract and layer roles granted exactly, two classifications masked and one row filter, with each role's group per environment.
+order: 11
diff --git a/examples/governed-databricks/standard.yaml b/examples/governed-databricks/standard.yaml
new file mode 100644
index 0000000..78236c5
--- /dev/null
+++ b/examples/governed-databricks/standard.yaml
@@ -0,0 +1,26 @@
+apiVersion: ingestron.io/v1
+kind: Standard
+id: shop
+packs: [databricks@2, adf@2] # Databricks names, types and keeps the tables; ADF copies
+layers:
+ - name: bronze # raw rows: only the engineers who load them read them
+ access: { contract: false, roles: [{ role: data-engineers }] }
+ - silver # each table's contract roles are granted
+ingest:
+ default: adf-copy
+ settings:
+ adf-copy:
+ layout: snapshot
+ start: 2026-11-01T00:00:00Z
+governance:
+ classification:
+ values: [public, internal, confidential, restricted]
+ access:
+ grants: exact # also take away what the contracts and layers do not name
+ masks:
+ confidential: hash # readers see a hash of the text
+ restricted: "null" # readers see no value
+ filters:
+ region_rows: # each reader sees the rows of their own region
+ function: region_filter # a BOOLEAN function the team owns, in databricks.governance.functions
+ match: { tag: region }
diff --git a/examples/governed-databricks/transforms/regional_sales.yaml b/examples/governed-databricks/transforms/regional_sales.yaml
new file mode 100644
index 0000000..2bf8d4e
--- /dev/null
+++ b/examples/governed-databricks/transforms/regional_sales.yaml
@@ -0,0 +1,13 @@
+apiVersion: ingestron.io/v1
+kind: Transform
+id: regional_sales
+description: Revenue and orders per day and region.
+inputs: [orders]
+code: |
+ SELECT CAST(ordered_at AS DATE) AS order_day,
+ region,
+ COUNT(*) AS orders,
+ SUM(amount) AS revenue
+ FROM {{ tables.orders }}
+ GROUP BY CAST(ordered_at AS DATE), region
+contract: shop.regional_sales
diff --git a/examples/layers-and-transforms/README.md b/examples/layers-and-transforms/README.md
index ccf0e1b..110163f 100644
--- a/examples/layers-and-transforms/README.md
+++ b/examples/layers-and-transforms/README.md
@@ -6,8 +6,7 @@
`transforms/store_sales.sql`, where `{{ tables.sales }}` became
`` `bronze`.`sales` ``, in the pipeline's catalog.
- `region_sales` reads `store_sales` and says `layer: gold`.
- The standard's rule for `layer: gold` turns clustering off and adds a
- retention property: compare [gold](build/databricks/src/sales/gold/region_sales.sql)
+ The standard's rule for `layer: gold` adds a retention property: compare [gold](build/databricks/src/sales/gold/region_sales.sql)
with silver.
Transforms are ordered by their inputs, not by file name, and one pipeline
diff --git a/examples/layers-and-transforms/standard.yaml b/examples/layers-and-transforms/standard.yaml
index 969fa7e..5cd445d 100644
--- a/examples/layers-and-transforms/standard.yaml
+++ b/examples/layers-and-transforms/standard.yaml
@@ -4,7 +4,7 @@ id: sales
packs: [databricks@2]
store:
when:
- # Gold tables are small and read whole: no clustering, longer retention.
+ # Gold tables are small and read whole: kept longer, as materialised views.
- layer: gold
use: materialised-view
- with: { cluster_by: none, properties: { delta.logRetentionDuration: "interval 90 days" } }
+ with: { properties: { delta.logRetentionDuration: "interval 90 days" } }
diff --git a/examples/northwind/platform.yaml b/examples/northwind/platform.yaml
index a8042ac..07f6b29 100644
--- a/examples/northwind/platform.yaml
+++ b/examples/northwind/platform.yaml
@@ -1,11 +1,12 @@
apiVersion: ingestron.io/v1
kind: Platform
id: azure
-workspace:
- host: ${values.host}
settings:
adf-copy: # shared infrastructure, referenced by name
landing: ${values.landing}/northwind
key_vault: ${values.key_vault}
integration_runtime: ${values.runtime}
runtime_kind: azure
+databricks:
+ workspace:
+ host: ${values.host}
diff --git a/examples/northwind/sample.yaml b/examples/northwind/sample.yaml
index 686d85c..a1b6344 100644
--- a/examples/northwind/sample.yaml
+++ b/examples/northwind/sample.yaml
@@ -1,4 +1,4 @@
# A guide: a small project that shows one idea; the docs embed it.
title: Northwind on Azure
summary: ADF copies four Northwind tables with one ForEach, audit columns and a retry policy, and Databricks builds the tables and a daily sales view from the landed files.
-order: 13
+order: 14
diff --git a/examples/platform-setup/README.md b/examples/platform-setup/README.md
index 4c4ec6a..6037880 100644
--- a/examples/platform-setup/README.md
+++ b/examples/platform-setup/README.md
@@ -2,14 +2,14 @@
`platform.yaml` holds what a platform team sets up for a project:
-| Setting | Generated in `build/databricks/` |
-| --------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
-| `workspace.host`, `run_as` | each target of [databricks.yml](build/databricks/databricks.yml) |
-| `permissions` | each target's permissions, applied to every resource |
-| `compute.budget_policy_id`, `tags` | [the pipeline](build/databricks/resources/hr.pipeline.yml) |
-| `schedule`, `notifications` | [the refresh job](build/databricks/resources/hr_refresh.job.yml) |
-| `manage.schemas: true`, `grants.bronze` | [schemas.yml](build/databricks/resources/schemas.yml); the pipeline refers to its schema so the bundle creates it first |
-| `manage.schemas: true` | [prerequisites/prod.sql](build/databricks/prerequisites/prod.sql): the grants an administrator runs once in prod |
+| Setting | Generated in `build/databricks/` |
+| ------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
+| `databricks.workspace.host`, `databricks.run_as` | each target of [databricks.yml](build/databricks/databricks.yml) |
+| `databricks.permissions` | each target's permissions, applied to every resource |
+| `databricks.compute.budget_policy_id`, `tags` | [the pipeline](build/databricks/resources/hr.pipeline.yml) |
+| `schedule`, `alerts` | [the refresh job](build/databricks/resources/hr_refresh.job.yml) |
+| `databricks.manage.schemas: true`, `databricks.grants.bronze` | [schemas.yml](build/databricks/resources/schemas.yml); the pipeline refers to its schema so the bundle creates it first |
+| `databricks.manage.schemas: true` | [prerequisites/prod.sql](build/databricks/prerequisites/prod.sql): the grants an administrator runs once in prod |
-Without `manage.schemas`, no schema is generated, because schemas are often
+Without `databricks.manage.schemas`, no schema is generated, because schemas are often
shared infrastructure that one project must not change.
diff --git a/examples/platform-setup/platform.yaml b/examples/platform-setup/platform.yaml
index 62183d5..06fd545 100644
--- a/examples/platform-setup/platform.yaml
+++ b/examples/platform-setup/platform.yaml
@@ -1,24 +1,25 @@
apiVersion: ingestron.io/v1
kind: Platform
id: hr-lakehouse
-workspace:
- host: https://adb-3000000000000003.3.azuredatabricks.net
-run_as:
- service_principal_name: 0b6c3d2e-0000-4000-8000-000000000007
-compute:
- budget_policy_id: 4e1d0c5b-hr-budget
tags:
cost_centre: "4410"
data_classification: confidential
-permissions:
- - { group_name: hr-data-engineers, level: CAN_MANAGE }
- - { group_name: hr-analysts, level: CAN_VIEW }
schedule:
cron: 0 0 2 * * ?
-notifications:
- on_failure: [hr-data@example.com]
-manage:
- schemas: true # this project owns its schemas
-grants:
- bronze:
- - { principal: hr-data-engineers, privileges: [USE_SCHEMA, SELECT, MODIFY] }
+alerts:
+ - { kind: email, to: [hr-data@example.com], on: [failure] }
+databricks:
+ workspace:
+ host: https://adb-3000000000000003.3.azuredatabricks.net
+ run_as:
+ service_principal_name: 0b6c3d2e-0000-4000-8000-000000000007
+ compute:
+ budget_policy_id: 4e1d0c5b-hr-budget
+ permissions:
+ - { group_name: hr-data-engineers, level: CAN_MANAGE }
+ - { group_name: hr-analysts, level: CAN_VIEW }
+ manage:
+ schemas: true # this project owns its schemas
+ grants:
+ bronze:
+ - { principal: hr-data-engineers, privileges: [USE_SCHEMA, SELECT, MODIFY] }
diff --git a/examples/private-sql-server-filled/flows/erp.yaml b/examples/private-sql-server-filled/flows/erp.yaml
index bafddf9..cce4eb7 100644
--- a/examples/private-sql-server-filled/flows/erp.yaml
+++ b/examples/private-sql-server-filled/flows/erp.yaml
@@ -2,6 +2,10 @@ apiVersion: ingestron.io/v1
kind: Flow
id: erp
source: erp
+# When a row last changed: each copy reads the rows past it,
+# and the tables keep the latest row of each key by it
+load: { cursor: modified_at }
+sequence: modified_at
tables:
customers:
# The source table: its database and schema, database.schema.Customers
@@ -11,7 +15,5 @@ tables:
# The source table: its database and schema, database.schema.Invoices
object: erp.dbo.Invoices
contract: erp.invoices
- ingest:
- with: { watermark_column: updated_at }
- store:
- with: { sequence_by: updated_at }
+ load: { cursor: updated_at }
+ sequence: updated_at
diff --git a/examples/private-sql-server-filled/sample.yaml b/examples/private-sql-server-filled/sample.yaml
index 4fd8061..4d6f428 100644
--- a/examples/private-sql-server-filled/sample.yaml
+++ b/examples/private-sql-server-filled/sample.yaml
@@ -1,4 +1,4 @@
# A guide: a small project that shows one idea; the docs embed it.
title: Private SQL Server, with its values written
summary: The private-sql-server starter after init --set wrote each value it left as REPLACE_ME, so its factory and bundle deploy to dev and prod.
-order: 15
+order: 16
diff --git a/examples/private-sql-server-filled/standard.yaml b/examples/private-sql-server-filled/standard.yaml
index f7236f3..f6c0b6e 100644
--- a/examples/private-sql-server-filled/standard.yaml
+++ b/examples/private-sql-server-filled/standard.yaml
@@ -5,16 +5,17 @@ packs: [databricks@2, adf@2] # Databricks first: the first pack names and types
naming:
# One catalog per environment, such as erp_dev
catalog: erp_{{ environment }}
+layers:
+ # Each window's file holds the rows changed in it; keep the current version.
+ - { name: bronze, history: latest }
+ - silver
+ - gold
+load:
+ default: incremental # copy the rows changed since the last copy, by each flow's cursor
ingest:
settings:
adf-copy:
# the schedule; where it lands is the platform's
- mode: incremental
- watermark_column: modified_at
every: { frequency: Hour, interval: 1 }
# Start of the first window (UTC): midnight on the day this project was created
start: 2026-11-06T00:00:00Z
-store:
- settings:
- # Each window's file holds the rows changed in it; keep the current version.
- streaming-table: { mode: upsert, sequence_by: modified_at }
diff --git a/examples/quality/README.md b/examples/quality/README.md
index 366b93d..227d1f8 100644
--- a/examples/quality/README.md
+++ b/examples/quality/README.md
@@ -1,14 +1,40 @@
-# Quality
+# Quality checks
-The standard sets `quality.required: drop` and `quality.default: warn`.
-In [the generated table](build/databricks/src/iot/bronze/readings.sql):
+One contract's checks, generated for three engines. Each environment builds the
+readings table on a different one, so you can compare what each does with the
+same rules: `databricks` keeps it as a streaming table, `duckdb` as a DuckDB
+table, and `synapse` as a Synapse serverless view over the files Azure Data
+Factory lands. `project.yaml` gives each environment its methods, and the
+standard uses them as its defaults. A real project builds its tables on one
+engine.
-| Check | From | Action |
-| ---------------------------------------- | -------------------------------- | ---------------------------------------------- |
-| `sensor_id_not_null`, `read_at_not_null` | `required: true` in the contract | drop (standard) |
-| `plausible_temperature` | the flow, no action | warn (standard default): rows kept and counted |
-| `known_unit` | the flow, `action: drop` | drop |
-| `not_from_future` | the flow, `action: fail` | the update stops |
+The checks:
-The flow also reads CSV with a header and `;` delimiter through
-`ingest.with.options`.
+| Check | From | Action |
+| ---------------------------------------- | ---------------------------------------- | --------------------------------------- |
+| `sensor_id_not_null`, `read_at_not_null` | `required: true` in the contract | `drop`, from the standard's `required` |
+| `sensor_id_pattern` | the contract's `pattern` for `sensor_id` | `warn`, from the standard's `default` |
+| `known_unit` | the contract's `known_unit` rule | `warn`, from the standard's `default` |
+| `plausible_temperature` | the flow's own expectation | `fail`, written on the expectation |
+| `has_readings` | the contract's table rule, `rowCount` | `warn`: a table check never drops a row |
+
+The `bronze` layer does `check` and `quarantine`, so every engine checks its
+rows there, and the standard's `quarantine: true` and `results: true` reach
+it.
+
+`ingestron apply` writes:
+
+- `build/databricks/`: [the table](build/databricks/src/iot/bronze/readings.sql)
+ with a constraint per check and `readings_quarantine` beside it, and
+ `tasks/iot/apply_contract.py`, which counts `has_readings` after each
+ refresh because the platform turns on `databricks.manage.checks`;
+- `build/duckdb/duckdb/run.sql`: the checks counted and printed in the run's
+ transaction, the rows kept apart in `bronze.readings_quarantine`, and each
+ result in `bronze._ingestron_checks`;
+- `build/synapse/views.sql`: a view that leaves out the rows a `drop` check
+ fails, and `build/synapse/checks.sql`, which counts every check and raises an
+ error when `plausible_temperature` fails.
+
+Synapse cannot apply four of these, and `ingestron plan` says so under
+`Not applied on this platform`: the `pattern` rule, the `fail` action,
+quarantine and results.
diff --git a/examples/quality/connections/drop.yaml b/examples/quality/connections/drop.yaml
new file mode 100644
index 0000000..277d3a0
--- /dev/null
+++ b/examples/quality/connections/drop.yaml
@@ -0,0 +1,8 @@
+apiVersion: ingestron.io/v1
+kind: Connection
+id: drop
+description: The same files, which Azure Data Factory lands for Synapse to read.
+connector: adls-gen2
+connectivity: public
+settings:
+ url: https://iotlab.dfs.core.windows.net
diff --git a/examples/quality/connections/sensors.yaml b/examples/quality/connections/sensors.yaml
index f71ed7e..744df5b 100644
--- a/examples/quality/connections/sensors.yaml
+++ b/examples/quality/connections/sensors.yaml
@@ -1,7 +1,8 @@
apiVersion: ingestron.io/v1
kind: Connection
id: sensors
+description: The CSV files the sensor gateways drop, read by Databricks and DuckDB.
connector: files
connectivity: public
settings:
- path: /Volumes/dev/landing/iot
+ path: abfss://drop@iotlab.dfs.core.windows.net/sensors
diff --git a/examples/quality/contracts/readings.yaml b/examples/quality/contracts/readings.yaml
index e8e0d7e..a706525 100644
--- a/examples/quality/contracts/readings.yaml
+++ b/examples/quality/contracts/readings.yaml
@@ -8,7 +8,19 @@ schema:
logicalType: object
description: One reading per sensor and time.
properties:
- - { name: sensor_id, logicalType: string, required: true }
+ - name: sensor_id
+ logicalType: string
+ required: true
+ logicalTypeOptions: { pattern: "^S[0-9]+$" }
- { name: read_at, logicalType: timestamp, required: true }
- { name: temperature, logicalType: number }
- - { name: unit, logicalType: string }
+ - name: unit
+ logicalType: string
+ quality:
+ - name: known_unit
+ type: library
+ metric: invalidValues
+ arguments: { validValues: [C, F] }
+ mustBe: 0
+ quality:
+ - { name: has_readings, type: library, metric: rowCount, mustBeGreaterThan: 0 }
diff --git a/examples/quality/flows/readings.yaml b/examples/quality/flows/readings.yaml
index 2f32ae4..38622d5 100644
--- a/examples/quality/flows/readings.yaml
+++ b/examples/quality/flows/readings.yaml
@@ -1,14 +1,11 @@
apiVersion: ingestron.io/v1
kind: Flow
id: readings
-source: sensors
+source: ${values.source}
tables:
readings:
object: readings
contract: iot.readings
- ingest:
- with: { format: csv, options: { header: true, delimiter: ";" } }
expectations:
- - { name: plausible_temperature, check: temperature BETWEEN -60 AND 80 }
- - { name: known_unit, check: "unit IN ('C', 'F')", action: drop }
- - { name: not_from_future, check: read_at <= current_timestamp(), action: fail }
+ # SQL that Databricks, DuckDB and Synapse each read the same way
+ - { name: plausible_temperature, check: temperature BETWEEN -60 AND 80, action: fail }
diff --git a/examples/quality/platform.yaml b/examples/quality/platform.yaml
new file mode 100644
index 0000000..c94d604
--- /dev/null
+++ b/examples/quality/platform.yaml
@@ -0,0 +1,14 @@
+apiVersion: ingestron.io/v1
+kind: Platform
+id: lab
+databricks:
+ manage: { checks: true } # Databricks runs table checks after each refresh
+settings:
+ adf-files:
+ landing: abfss://landing@iotlab.dfs.core.windows.net/iot
+ integration_runtime: AutoResolveIntegrationRuntime
+ synapse-view: # the external data source the view reads through
+ data_source: landing
+ data_source_location: abfss://landing@iotlab.dfs.core.windows.net
+ duckdb-table:
+ published: abfss://published@iotlab.dfs.core.windows.net/iot
diff --git a/examples/quality/project.yaml b/examples/quality/project.yaml
index f7a2366..668f65a 100644
--- a/examples/quality/project.yaml
+++ b/examples/quality/project.yaml
@@ -1,10 +1,17 @@
-# Data quality: required columns from the contract and your own checks,
-# each with warn, drop or fail.
+# One set of checks on three engines: each environment builds the readings
+# table on a different one, so you can compare what each does with the same
+# rules. A real project builds its tables on one engine.
id: iot
environments:
- dev: {}
+ databricks:
+ values: { ingest: files, store: streaming-table, source: sensors }
+ duckdb:
+ values: { ingest: duckdb-files, store: duckdb-table, source: sensors }
+ synapse:
+ values: { ingest: adf-files, store: synapse-view, source: drop }
imports:
- standard.yaml
+ - platform.yaml
- connections/*.yaml
- flows/*.yaml
- contracts/*.yaml
diff --git a/examples/quality/sample.yaml b/examples/quality/sample.yaml
index 98ca825..691f695 100644
--- a/examples/quality/sample.yaml
+++ b/examples/quality/sample.yaml
@@ -1,4 +1,4 @@
# A guide: a small project that shows one idea; the docs embed it.
title: Quality checks
-summary: Expectations from required columns and your own checks, with warn, drop and fail.
+summary: One contract's checks on Databricks, DuckDB and Synapse, with warn, drop and fail, a table check, quarantine and results.
order: 3
diff --git a/examples/quality/standard.yaml b/examples/quality/standard.yaml
index 90dfe0e..2944df6 100644
--- a/examples/quality/standard.yaml
+++ b/examples/quality/standard.yaml
@@ -1,7 +1,21 @@
apiVersion: ingestron.io/v1
kind: Standard
id: iot
-packs: [databricks@2]
+packs: [databricks@2, duckdb@2, synapse@2, adf@2]
+layers:
+ - name: bronze
+ does: [check, quarantine] # checks its rows, and keeps the ones a check leaves out
+ - silver
quality:
- required: drop # a row missing a required value is dropped and counted
- default: warn # own checks only count failures unless they say otherwise
+ required: drop # a row missing a required value is left out
+ default: warn # every other check counts the rows that fail it
+ quarantine: true # rows a drop check leaves out go to
_quarantine
+ results: true # each check's result is kept where the engine can keep it
+ingest:
+ default: ${values.ingest} # the engine each environment builds on, from project.yaml
+ settings:
+ files: { format: csv }
+ duckdb-files: { format: csv }
+ adf-files: { format: csv, start: 2026-10-01T00:00:00Z }
+store:
+ default: ${values.store}
diff --git a/examples/retail-filled/flows/shop.yaml b/examples/retail-filled/flows/shop.yaml
index 87cadb1..9dd49ba 100644
--- a/examples/retail-filled/flows/shop.yaml
+++ b/examples/retail-filled/flows/shop.yaml
@@ -7,8 +7,11 @@ tables:
orders:
object: orders
contract: shop.orders
+ history: latest
+ sequence: updated_at
+ load: { deletes: { when: "op = 'D'" } }
store:
- with: { mode: upsert, sequence_by: updated_at, delete_when: "op = 'D'", except_columns: [op] }
+ with: { except_columns: [op] }
expectations:
- { name: positive_total, check: total >= 0 }
customers:
diff --git a/examples/retail-filled/platform.yaml b/examples/retail-filled/platform.yaml
index e00553c..ab4bd38 100644
--- a/examples/retail-filled/platform.yaml
+++ b/examples/retail-filled/platform.yaml
@@ -2,20 +2,21 @@
apiVersion: ingestron.io/v1
kind: Platform
id: lakehouse
-workspace:
- host: ${values.host}
-run_as:
- service_principal_name: ${values.run_as}
tags:
team: sales
environment: ${environment.name}
-permissions:
- - group_name: ${values.engineers}
- level: CAN_MANAGE
- - group_name: ${values.analysts}
- level: CAN_VIEW
schedule:
cron: 0 0 5 * * ? # Quartz cron: the refresh job runs once a day
timezone: UTC # the time zone the cron is read in
-notifications:
- on_failure: ["${values.on_failure}"]
+alerts:
+ - { kind: email, to: ["${values.on_failure}"], on: [failure] }
+databricks:
+ workspace:
+ host: ${values.host}
+ run_as:
+ service_principal_name: ${values.run_as}
+ permissions:
+ - group_name: ${values.engineers}
+ level: CAN_MANAGE
+ - group_name: ${values.analysts}
+ level: CAN_VIEW
diff --git a/examples/retail-filled/sample.yaml b/examples/retail-filled/sample.yaml
index d1173e8..aa97d42 100644
--- a/examples/retail-filled/sample.yaml
+++ b/examples/retail-filled/sample.yaml
@@ -1,4 +1,4 @@
# A guide: a small project that shows one idea; the docs embed it.
title: Retail, with its values written
summary: The retail starter after init --set wrote each value it left as REPLACE_ME, so it deploys to its dev and prod workspaces.
-order: 14
+order: 15
diff --git a/examples/synapse-views/README.md b/examples/synapse-views/README.md
index 6c6c3a7..356bbc2 100644
--- a/examples/synapse-views/README.md
+++ b/examples/synapse-views/README.md
@@ -10,15 +10,29 @@ copies.
- `build/adf/`: the copy pipeline, its trigger, linked services and datasets,
and a parameter file for each environment;
+- `build/synapse/checks.sql`, with `checks/.sql`: the checks of
+ the contracts' required columns, counted each time it runs. `bronze` keeps
+ the rows as they arrived and only counts its checks, and `silver` and `gold`
+ check theirs;
- `build/synapse/views.sql`: a view per table over its file,
`[bronze].[customers]` and `[bronze].[invoices]`, and
`[silver].[customer_totals]` over the invoices view, from
- `transforms/customer_totals.yaml`, with `environments/.sql`
- setting each environment's database.
+ `transforms/customer_totals.yaml`. Each view is followed by a `GRANT SELECT`
+ to its roles, in its own database. `environments/.sql` sets
+ each environment's database and the database user each role is.
+
+Who may read each view: `synapse.manage.grants: true` in `platform.yaml`
+turns grants on. The contract of `customer_totals` names the role
+`finance-analysts`, and the standard's `bronze` layer grants
+`data-engineers` on every view in it. `principals` in `platform.yaml` says
+which database user each role is in each environment, from the values in
+`project.yaml`, such as `sales-finance-analysts-dev` in `dev`. Synapse
+serverless has no data masking or row-level security, so a masked
+classification or a row filter would be a `NOT_APPLIED` warning here.
Run `sqlcmd -i environments/.sql` from `build/synapse/` after
each change to the definitions; loads need nothing more. `ingestron request`
lists what the platform team creates: the database, a master key if it has
none, the external data source `landing` with its credential, Storage Blob Data
Reader on the landing storage for the Synapse workspace's managed identity, and
-report users' access.
+each role's database user, with the rights views over files need.
diff --git a/examples/synapse-views/contracts/customer_totals.yaml b/examples/synapse-views/contracts/customer_totals.yaml
index b054498..792be70 100644
--- a/examples/synapse-views/contracts/customer_totals.yaml
+++ b/examples/synapse-views/contracts/customer_totals.yaml
@@ -10,3 +10,7 @@ schema:
properties:
- { name: customer_id, logicalType: integer, required: true, primaryKey: true }
- { name: total, logicalType: number, physicalType: "DECIMAL(18,2)" }
+roles:
+ - role: finance-analysts
+ access: read
+ description: Monthly revenue reporting
diff --git a/examples/synapse-views/platform.yaml b/examples/synapse-views/platform.yaml
index e1c2c8f..1482ae6 100644
--- a/examples/synapse-views/platform.yaml
+++ b/examples/synapse-views/platform.yaml
@@ -1,6 +1,9 @@
apiVersion: ingestron.io/v1
kind: Platform
id: azure
+principals: # the database user each role is, per environment
+ finance-analysts: ${values.finance_analysts}
+ data-engineers: ${values.data_engineers}
settings:
adf-copy: # shared infrastructure, referenced by name
landing: ${values.landing}
@@ -9,3 +12,5 @@ settings:
synapse-view: # the external data source the views read through
data_source: landing
data_source_location: ${values.storage}
+synapse:
+ manage: { grants: true } # views.sql grants SELECT on each view to its roles
diff --git a/examples/synapse-views/project.yaml b/examples/synapse-views/project.yaml
index 5a2441e..8324eec 100644
--- a/examples/synapse-views/project.yaml
+++ b/examples/synapse-views/project.yaml
@@ -9,10 +9,14 @@ environments:
landing: abfss://landing@saleslakeprod.dfs.core.windows.net/erp
storage: abfss://landing@saleslakeprod.dfs.core.windows.net
key_vault: https://kv-sales-prod.vault.azure.net
+ finance_analysts: sales-finance-analysts
+ data_engineers: sales-data-engineers
values:
landing: abfss://landing@saleslakedev.dfs.core.windows.net/erp
storage: abfss://landing@saleslakedev.dfs.core.windows.net
key_vault: https://kv-sales-dev.vault.azure.net
+ finance_analysts: sales-finance-analysts-dev
+ data_engineers: sales-data-engineers-dev
imports:
- standard.yaml
- platform.yaml
diff --git a/examples/synapse-views/standard.yaml b/examples/synapse-views/standard.yaml
index a880779..529dc73 100644
--- a/examples/synapse-views/standard.yaml
+++ b/examples/synapse-views/standard.yaml
@@ -2,6 +2,12 @@ apiVersion: ingestron.io/v1
kind: Standard
id: sales
packs: [synapse@2, adf@2] # Synapse first: the first pack names and types the views; ADF only copies
+layers:
+ - name: bronze # every row as it arrived; its checks only count
+ does: raw
+ access: { roles: [{ role: data-engineers }] } # beside the contracts' roles
+ - { name: silver, does: [check] }
+ - { name: gold, does: [check] }
store:
default: synapse-view # each table is a view over the files ADF lands
ingest:
diff --git a/lib/areas.ts b/lib/areas.ts
index 9a084b8..0eef94d 100644
--- a/lib/areas.ts
+++ b/lib/areas.ts
@@ -47,8 +47,8 @@ export const areaCardSettings: { area: string; icon: string; links: string[] }[]
icon: "standard",
links: [
"/docs/how-to/configure-a-standard",
+ "/docs/how-to/choose-a-published-standard",
"/docs/how-to/ask-the-platform-team",
- "/docs/how-to/upgrade",
],
},
{
diff --git a/lib/connector-page.ts b/lib/connector-page.ts
index c394abf..447f095 100644
--- a/lib/connector-page.ts
+++ b/lib/connector-page.ts
@@ -194,13 +194,13 @@ function reading(id: string, entry: ConnectorEntry): Part {
...part,
title: "Full copies and incremental windows",
before: [
- `\`mode: full\` copies the whole table each ${windows}. \`mode: incremental\` copies rows whose \`watermark_column\` falls within that window. Choose a column that records when a row changes, such as \`modified_at\`; an integer identity is not a time-window watermark.`,
+ `\`load: full\` copies the whole table each ${windows}. \`load: incremental\` copies the rows whose cursor column falls within that window, the column a flow or a table names in \`load: { cursor }\`; the method writes them as its \`mode\` and \`watermark_column\` settings. Choose a column that records when a row changes, such as \`modified_at\`; an integer identity is not a time-window watermark.`,
],
after: [
- "The watermark is configured in the flow's `ingest.with`, or as a default in the standard's `ingest.settings.adf-copy`. It is not a contract property. Each table can override the default with its own `mode` and `watermark_column`; the contract describes the columns and keys of that table.",
+ "Write the load mode once in the standard, as `load: { default: incremental }`, and the cursor on each flow, as `load: { cursor: modified_at }`. Each table can give its own `load`. They are [intent](/docs/reference/intent), not contract properties; the contract describes the columns and keys of that table.",
"The filter is always the trigger window's inclusive start and exclusive end. Ingestron does not track the greatest value seen in an earlier run. A change outside the selected window is not copied by that window. Arbitrary SQL queries, extra `WHERE` criteria and per-table window schedules are not supported by `adf-copy`.",
- "Use the source column name for `watermark_column`. A contract's `physicalName` can rename a selected column, but it does not rewrite the watermark filter.",
- "All tables of one connection and source database share runtime, encryption, schedule, retry and paused settings. Those values must agree even when the tables use different watermarks.",
+ "Use the source column name for the cursor. A contract's `physicalName` can rename a selected column, but it does not rewrite the window filter.",
+ "All tables of one connection and source database share runtime, encryption, schedule, retry and paused settings. Those values must agree even when the tables use different cursors.",
],
};
case "adf-files":
@@ -253,9 +253,9 @@ function current(id: string, entry: ConnectorEntry): Part | undefined {
const databricks =
connector.method === "adf-files"
? "Auto Loader reads each file once by default. Choose how the table keeps the rows inside those files, separately from which files ADF copies; see [On Databricks](/docs/how-to/keep-a-table-current#on-databricks)."
- : "Databricks reads the landed files into a streaming table. With `mode: append`, repeated copies add repeated rows; `upsert` keeps the current row per key and `history` keeps every version, ordered by a column you choose. See [On Databricks](/docs/how-to/keep-a-table-current#on-databricks)." +
+ : "Databricks reads the landed files into a streaming table. With `history: none`, its default, repeated copies add repeated rows; `history: latest` keeps the current row per key and `history: full` keeps every version, ordered by the `sequence` column you choose. See [On Databricks](/docs/how-to/keep-a-table-current#on-databricks)." +
(connector.method === "adf-copy"
- ? " The store's `sequence_by` decides which version wins; it is separate from the extraction watermark. ADF and Databricks run on separate schedules."
+ ? " `sequence` decides which version wins; it is separate from the copy's cursor, though often the same column. ADF and Databricks run on separate schedules."
: "");
return {
kind: "current",
@@ -267,7 +267,7 @@ function current(id: string, entry: ConnectorEntry): Part | undefined {
"adf-synapse":
"A view reads the landed files each time it is queried and shows them as they are: it cannot keep a current row per key. See [On Synapse](/docs/how-to/keep-a-table-current#on-synapse).",
"adf-duckdb":
- "The DuckDB table's `mode` decides what it keeps; see [On DuckDB](/docs/how-to/keep-a-table-current#on-duckdb).",
+ "The DuckDB table's `history` decides what it keeps; see [On DuckDB](/docs/how-to/keep-a-table-current#on-duckdb).",
"adf-landing": "The landed files are the result: nothing merges them.",
}),
after: [],
@@ -332,9 +332,9 @@ function readSection(id: string, entry: ConnectorEntry): Section {
part: {
kind: "watermarks",
id: `${pack}-watermarks`,
- title: "Different watermarks per table",
+ title: "A different cursor per table",
before: [
- "In this example, the standard reads changed rows once an hour using `modified_at`. Customers inherit that default. Invoices override the watermark with `updated_at` in the flow and set the store's ordering column to match. The same hourly trigger serves both tables.",
+ "In this example, the standard copies changed rows once an hour, and the flow reads them by `modified_at`. Customers take the flow's cursor. Invoices give their own, `updated_at`, and order their changes by it too. The same hourly trigger serves both tables.",
...watermarkNote(id),
],
after: [],
diff --git a/lib/kits.ts b/lib/kits.ts
new file mode 100644
index 0000000..830228d
--- /dev/null
+++ b/lib/kits.ts
@@ -0,0 +1,155 @@
+// Published standards (kits), read at build time from the installed
+// @ingestron/standards package, so each kit's page shows the kit's own files.
+// What a REPLACE_ME stands for comes from the same field reference the
+// editor's hover uses (`ProjectSnapshot.replaceMeHelp`).
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { ProjectSnapshot, type ReplaceMeHelp } from "@ingestron/authoring";
+import { installedKits, packs as corePacks, type Kit } from "@ingestron/core";
+import { loadDefinitions, type Origin } from "@ingestron/definitions";
+import { isMap, isScalar, isSeq, parseDocument, type Node } from "yaml";
+
+/** The `kits` folder of the installed @ingestron/standards package. */
+export const kitsFolder = join(process.cwd(), "node_modules/@ingestron/standards/kits");
+
+let found: Kit[] | undefined;
+
+/** Every installed kit version, by id and then version, oldest first. */
+export function kits(): Kit[] {
+ found ??= installedKits(kitsFolder);
+ return found;
+}
+
+/** The newest version of each kit that is not a preview, in order of id. */
+export function latestKits(): Kit[] {
+ const ids = [...new Set(kits().map((kit) => kit.id))];
+ return ids.map(kitById);
+}
+
+/** A kit's newest installed version that is not a preview, as `ingestron init --kit ` takes it offline. */
+export function kitById(id: string): Kit {
+ const kit = kits()
+ .filter((item) => item.id === id && item.features === "ga")
+ .at(-1);
+ if (!kit) throw new Error(`No kit "${id}" is installed.`);
+ return kit;
+}
+
+/** A kit's page: `/docs/how-to/kits/duckdb-local-medallion` for `duckdb/local-medallion`. */
+export function kitPage(id: string): string {
+ return `/docs/how-to/kits/${id.replace("/", "-")}`;
+}
+
+/** A file of a kit, as it is in the package. */
+export function kitFile(kit: Kit, file: string): string {
+ return readFileSync(join(kit.folder, file), "utf8");
+}
+
+/** A value still `REPLACE_ME` in a kit's file, and what it stands for. */
+export interface Blank {
+ /** Where `REPLACE_ME` starts in the file's text. */
+ offset: number;
+ /** The file's line, from 1. */
+ line: number;
+ /** The value's JSON Pointer, such as `/databricks/run_as/service_principal_name`. */
+ pointer: string;
+ /** The field it fills and the help the field gives; undefined when no field describes it. */
+ help?: ReplaceMeHelp;
+}
+
+const snapshots = new Map();
+
+/** The kit's folder opened as a project, as the editor opens it, once. */
+function snapshotOf(kit: Kit): ProjectSnapshot {
+ let snapshot = snapshots.get(kit.folder);
+ if (!snapshot) {
+ snapshot = ProjectSnapshot.open(kit.folder);
+ snapshots.set(kit.folder, snapshot);
+ }
+ return snapshot;
+}
+
+/**
+ * Each `REPLACE_ME` value in a YAML file of a kit, in file order, with the
+ * help the editor's hover gives for it. A value under
+ * `environments.` of project.yaml is explained for that environment.
+ */
+export function kitBlanks(kit: Kit, file: string): Blank[] {
+ const text = kitFile(kit, file);
+ const snapshot = snapshotOf(kit);
+ const blanks: Blank[] = [];
+ const visit = (node: Node | null | undefined, path: string[]) => {
+ if (isMap(node))
+ for (const pair of node.items)
+ visit(pair.value as Node | null, [
+ ...path,
+ String(isScalar(pair.key) ? pair.key.value : pair.key),
+ ]);
+ else if (isSeq(node))
+ node.items.forEach((item, index) => visit(item as Node | null, [...path, String(index)]));
+ else if (isScalar(node) && node.value === "REPLACE_ME" && node.range) {
+ const pointer = `/${path.map((key) => key.replaceAll("~", "~0").replaceAll("/", "~1")).join("/")}`;
+ const environment =
+ file === "project.yaml" && path[0] === "environments" ? path[1] : undefined;
+ const help = snapshot.replaceMeHelp(file, pointer, environment);
+ blanks.push({
+ offset: node.range[0],
+ line: text.slice(0, node.range[0]).split("\n").length,
+ pointer,
+ ...(help && { help }),
+ });
+ }
+ };
+ visit(parseDocument(text).contents as Node | null, []);
+ return blanks;
+}
+
+/** One value of a table's intent in a kit, and where it came from. */
+export interface KitIntent {
+ table: string;
+ layer: string;
+ /** The intent path, such as `history`. */
+ path: string;
+ /** The value as the editor's hover shows it, such as `latest` or `[loaded_at, source_file]`. */
+ value: string;
+ origin: Origin;
+ /** Whether the table is a flow's or a transform's, for naming where a value came from. */
+ kind: "Flow" | "Transform";
+}
+
+/**
+ * Each table's intent, resolved for the kit's first environment as `plan`
+ * resolves it, in the order of the tables and of their intent paths. Audit
+ * columns are shown by their values.
+ */
+export function kitIntent(kit: Kit): KitIntent[] {
+ const environment = snapshotOf(kit).environments[0];
+ const result = loadDefinitions(kit.folder, { packs: corePacks, environment });
+ if (!result.model) throw new Error(`The kit ${kit.id}@${kit.version} does not resolve.`);
+ return result.model.tables.flatMap((table) =>
+ Object.entries(table.intentOrigins).map(([path, origin]) => ({
+ table: table.name,
+ layer: table.layer,
+ path,
+ value: written(valueAt(table.intent, path)),
+ origin,
+ kind: table.origin.kind === "Transform" ? "Transform" : "Flow",
+ })),
+ );
+}
+
+function valueAt(intent: object, path: string): unknown {
+ return path
+ .split(".")
+ .reduce(
+ (node, key) =>
+ node && typeof node === "object" ? (node as Record)[key] : undefined,
+ intent,
+ );
+}
+
+function written(value: unknown): string {
+ if (Array.isArray(value))
+ return `[${value.map((item) => (item && typeof item === "object" && "value" in item ? String(item.value) : written(item))).join(", ")}]`;
+ return String(value);
+}
diff --git a/lib/reference.ts b/lib/reference.ts
index 4f386c7..8521263 100644
--- a/lib/reference.ts
+++ b/lib/reference.ts
@@ -4,10 +4,14 @@ import { join } from "node:path";
import { projectRecipe, projectRoutes, projectSources, type ProjectRoute } from "@ingestron/core";
import {
documentReference,
+ intentPaths,
+ intentUseOf,
methodReference,
+ platformSectionReference,
referenceKinds,
type Connector,
type FieldInfo,
+ type IntentUse,
type Pack,
} from "@ingestron/definitions";
import { adf } from "@ingestron/pack-adf";
@@ -20,6 +24,13 @@ import { contentFile, contentFolder } from "./content-files";
/** The packs the CLI provides, in the order a standard usually lists them. */
export const packs: Pack[] = [databricks, adf, synapse, duckdb];
+/** A pack the CLI provides, by id; throws for any other. */
+export function packOf(id: string): Pack {
+ const found = packs.find((candidate) => candidate.id === id);
+ if (!found) throw new Error(`No pack "${id}".`);
+ return found;
+}
+
/** Each pack's reference page. */
export function packPages(): { pack: string; url: string }[] {
return packs.map((pack) => ({ pack: pack.id, url: `/docs/reference/packs/${pack.id}` }));
@@ -134,18 +145,351 @@ export function shown(value: unknown): string {
/**
* A pack's platform defaults: each Platform field the pack declares it reads
* (its `platformFields`), in the pack's order, with the default the field's
- * description states. Fields with no default, such as `run_as`, are left
- * out. Throws for a pack that reads no Platform field, or for a field the
- * Platform document does not describe.
+ * description states. A field the pack's own section of the Platform has
+ * (`databricks.mode` for `mode`) is shown there, where it is written now.
+ * Fields with no default, such as `run_as`, are left out. Throws for a pack
+ * that reads no Platform field, or for a field neither describes.
*/
export function platformDefaults(pack: string): { field: FieldInfo; default: string }[] {
- const read = packs.find((candidate) => candidate.id === pack)?.platformFields;
+ const found = packOf(pack);
+ const read = found.platformFields;
if (!read?.length) throw new Error(`The ${pack} pack reads no platform defaults.`);
const fields = documentReference("Platform");
+ const section = platformSectionFields(pack);
return read.flatMap((path) => {
- const field = fields.find((candidate) => fieldPath(candidate) === path);
+ const field =
+ section.find((candidate) => fieldPath(candidate) === `${pack}.${path}`) ??
+ section.find((candidate) => fieldPath(candidate) === path) ??
+ fields.find((candidate) => fieldPath(candidate) === path);
if (!field) throw new Error(`The Platform document has no field "${path}".`);
const stated = /Default: (.+?)\.?$/.exec(field.description ?? "")?.[1];
return stated ? [{ field, default: stated }] : [];
});
}
+
+/** The fields of a pack's own section of the Platform, such as `databricks.run_as`; none when it has none. */
+export function platformSectionFields(pack: string): FieldInfo[] {
+ return platformSectionReference(packOf(pack));
+}
+
+/** One row of a pack's intent support: a path, the values that share one entry, and the entry. */
+export interface IntentRow {
+ path: string;
+ /** The values the entry covers, in the vocabulary's order; `["*"]` for a path that takes any text. */
+ values: string[];
+ use: IntentUse;
+}
+
+/**
+ * What a pack does with each intent path and value, from the pack's own
+ * `intent` (each value takes its own entry, else its path's, else a parent
+ * path's, as resolving does). Values of one path that share an entry are one
+ * row. Throws when a value has no entry, so a pack's gap fails the build.
+ */
+export function intentRows(pack: string): IntentRow[] {
+ const found = packOf(pack);
+ const rows: IntentRow[] = [];
+ for (const [path, values] of Object.entries(intentPaths)) {
+ for (const value of values) {
+ const use = intentUseOf(found, path, value === "*" ? undefined : value);
+ if (!use) throw new Error(`The ${pack} pack says nothing about ${path}: ${value}.`);
+ const last = rows.at(-1);
+ if (last?.path === path && JSON.stringify(last.use) === JSON.stringify(use))
+ last.values.push(value);
+ else rows.push({ path, values: [value], use });
+ }
+ }
+ return rows;
+}
+
+/** One entry of a method's own `intent`: where the method differs from its pack. */
+export interface MethodIntentRow {
+ method: string;
+ path: string;
+ /** Undefined when the entry covers the whole path. */
+ value?: string;
+ use: IntentUse;
+}
+
+/** Each method's own intent entries, in the pack's method order, then the vocabulary's. */
+export function methodIntentRows(pack: string): MethodIntentRow[] {
+ const listed = Object.keys(intentPaths);
+ const rank = (path: string) => {
+ const at = listed.findIndex((item) => item === path || item.startsWith(`${path}.`));
+ return at < 0 ? listed.length : at;
+ };
+ return Object.entries(packOf(pack).methods).flatMap(([method, info]) =>
+ Object.entries(info.intent ?? {})
+ .map(([key, use]) => {
+ const [path, value] = key.split("=") as [string, string | undefined];
+ return { method, path, ...(value !== undefined && { value }), use };
+ })
+ .sort((a, b) => {
+ const values = intentPaths[a.path] ?? [];
+ return (
+ rank(a.path) - rank(b.path) ||
+ values.indexOf(a.value ?? "") - values.indexOf(b.value ?? "")
+ );
+ }),
+ );
+}
+
+/** Intent written as `path: value` pairs, such as `["load.mode: files"]`; a list is one value. */
+export function intentPairs(intent: object, prefix = ""): string[] {
+ return Object.entries(intent).flatMap(([key, value]) => {
+ const path = prefix ? `${prefix}.${key}` : key;
+ return value && typeof value === "object" && !Array.isArray(value)
+ ? intentPairs(value, path)
+ : [`${path}: ${shown(value)}`];
+ });
+}
+
+/**
+ * A method's settings that are other names for intent, each as a sentence
+ * part: `` `sequence_by` is `sequence` `` or `` `mode: upsert` is `history: latest` ``.
+ */
+export function aliasTexts(pack: string, method: string): string[] {
+ const aliases = packOf(pack).methods[method]?.aliases ?? {};
+ return Object.entries(aliases).flatMap(([setting, alias]) =>
+ "path" in alias
+ ? [`\`${setting}\` is \`${alias.path}\``]
+ : Object.entries(alias.values).map(
+ ([value, intent]) => `\`${setting}: ${value}\` is \`${intentPairs(intent).join(", ")}\``,
+ ),
+ );
+}
+
+/** A document kind whose fields can write intent, or a contract, which writes the checks. */
+type WritingKind = "Standard" | "Flow" | "Transform" | "DataContract";
+
+/** The places that can write a path, in the order they win. */
+type Place = "standard" | "layer" | "flow" | "flow table" | "transform" | "contract";
+
+/** Where a path's meaning is described, and, for a path no intent field writes as it is named, where it is written. */
+interface Described {
+ /** The field whose description and values explain the path. */
+ field: [WritingKind, string];
+ /** A description for a path whose field explains only part of it. */
+ description?: string;
+ /** Meanings for values the field does not describe. */
+ meanings?: Record;
+ /** The fields that write it, by place; without it, the path's own field in each document. */
+ written?: Partial>;
+}
+
+/** The fields a cleansing operation is written in: one column of a flow table or a transform, and the standard's default when it has one. */
+const operation = (name: string, standard?: string): Described => ({
+ field: standard ? ["Standard", standard] : ["Flow", `tables.*.cleanse.columns.*.${name}`],
+ written: {
+ ...(standard && { standard: ["Standard", [standard]] as [WritingKind, string[]] }),
+ "flow table": ["Flow", [`tables.*.cleanse.columns.*.${name}`]],
+ transform: ["Transform", [`cleanse.columns.*.${name}`]],
+ },
+});
+
+/** Where each intent path's meaning is described: the field of a document that writes it. */
+const intentFields: Record = {
+ "load.mode": { field: ["Standard", "load.default"] },
+ "load.cursor": { field: ["Flow", "load.cursor"] },
+ "load.deletes.mode": { field: ["Standard", "load.deletes"] },
+ "load.deletes.when": { field: ["Flow", "load.deletes.when"] },
+ history: { field: ["Standard", "layers.*.history"] },
+ sequence: { field: ["Flow", "sequence"] },
+ audit: { field: ["Standard", "audit.*.value"] },
+ rescued: { field: ["Standard", "rescued"] },
+ "retention.time_travel": { field: ["Standard", "retention.time_travel"] },
+ "retention.log": { field: ["Standard", "retention.log"] },
+ "retention.landed_files": { field: ["Standard", "retention.landed_files"] },
+ "storage.clustering": { field: ["Standard", "storage.clustering"] },
+ "storage.features": { field: ["Standard", "storage.features"] },
+ "operations.schedule": { field: ["Standard", "operations.schedule.per"] },
+ "operations.trigger": { field: ["Standard", "operations.trigger"] },
+ "operations.retries": { field: ["Standard", "operations.retries"] },
+ "operations.timeout": { field: ["Standard", "operations.timeout"] },
+ "operations.alerts": { field: ["Standard", "operations.alerts.on"] },
+ "operations.concurrency": { field: ["Standard", "operations.concurrency"] },
+ "quality.results": { field: ["Standard", "quality.results"] },
+ // The checks a contract gives each row, by the kind of rule each becomes.
+ "quality.rules": {
+ field: ["DataContract", "schema.*.properties.*.quality"],
+ description:
+ "The rules a contract gives each row, by kind: a required column, a column's limits in `logicalTypeOptions`, and its `quality` rules that allow no failing row.",
+ meanings: {
+ notNull: "Has a value: `required: true`, `nullValues`, or `missingValues` listing only null.",
+ in: "Is one of a list: `invalidValues` with `validValues`.",
+ notIn: "Is none of a list: `missingValues`.",
+ pattern:
+ "Matches a regular expression: `invalidValues` with `pattern`, or the option `pattern`.",
+ length: "Text no shorter or longer than a limit: the options `minLength` and `maxLength`.",
+ range:
+ "No lower or higher than a limit: the options `minimum`, `maximum`, `exclusiveMinimum` and `exclusiveMaximum`.",
+ multipleOf: "A multiple of a number: the option `multipleOf`.",
+ items: "A list's number of items, or unique items: `minItems`, `maxItems` and `uniqueItems`.",
+ },
+ written: {
+ contract: [
+ "DataContract",
+ [
+ "schema.*.properties.*.required",
+ "schema.*.properties.*.logicalTypeOptions",
+ "schema.*.properties.*.quality",
+ ],
+ ],
+ },
+ },
+ "quality.actions": {
+ field: ["Standard", "quality.default"],
+ description:
+ "What a check does with a row that fails it: the standard's or a layer's `quality.required` and `quality.default`, or an expectation's own `action`.",
+ written: {
+ standard: ["Standard", ["quality.required", "quality.default"]],
+ layer: ["Standard", ["layers.*.quality.required", "layers.*.quality.default"]],
+ "flow table": ["Flow", ["tables.*.expectations.*.action"]],
+ transform: ["Transform", ["expectations.*.action"]],
+ },
+ },
+ "quality.table-checks": {
+ field: ["DataContract", "schema.*.quality"],
+ description:
+ "A contract rule that compares a count over all of a table's rows, such as `rowCount`, or a column's `unique: true`. It is counted after the table is built, with the action `quality.default` gives it; `drop` only warns.",
+ written: {
+ contract: ["DataContract", ["schema.*.quality", "schema.*.properties.*.unique"]],
+ },
+ },
+ "quality.quarantine": { field: ["Standard", "quality.quarantine"] },
+ "cleanse.trim": operation("trim", "cleanse.text.trim"),
+ "cleanse.empty": operation("empty", "cleanse.text.empty"),
+ "cleanse.map": operation("map"),
+ "cleanse.case": operation("case"),
+ "cleanse.default": operation("default"),
+ "cleanse.timezone": {
+ field: ["Flow", "tables.*.cleanse.columns.*.timezone"],
+ written: {
+ standard: ["Standard", ["cleanse.timestamps.to"]],
+ "flow table": ["Flow", ["tables.*.cleanse.columns.*.timezone"]],
+ transform: ["Transform", ["cleanse.columns.*.timezone"]],
+ },
+ },
+ deduplicate: {
+ field: ["Flow", "tables.*.cleanse.deduplicate"],
+ written: {
+ "flow table": ["Flow", ["tables.*.cleanse.deduplicate"]],
+ transform: ["Transform", ["cleanse.deduplicate"]],
+ },
+ },
+ // Who may use a table, and which of its columns and rows are hidden.
+ "access.grants": {
+ field: ["DataContract", "roles.*.access"],
+ description:
+ "A role granted on a table: each of its contract's `roles`, and each of its layer's `access.roles`. The platform's `principals` says who the role is.",
+ meanings: { read: "Read the table.", write: "Read and change the table." },
+ written: {
+ layer: ["Standard", ["layers.*.access.roles", "layers.*.access.contract"]],
+ contract: ["DataContract", ["roles"]],
+ },
+ },
+ "access.exact": {
+ field: ["Standard", "governance.access.grants"],
+ description:
+ "Exact grants, the standard's `governance.access.grants: exact`: a privilege granted directly on a table to anyone its roles do not name is taken away.",
+ meanings: { true: "Grants are exact." },
+ written: { standard: ["Standard", ["governance.access.grants"]] },
+ },
+ "access.masks": {
+ field: ["Standard", "governance.masks.*"],
+ description:
+ "A column's mask: the standard's `governance.masks` gives one for each classification, and the contract classifies the column.",
+ meanings: {
+ null: "Shown as no value, for a column of any type.",
+ hash: "Text shown as a hash of its value, so equal values still match.",
+ redact: "Text replaced by a fixed value.",
+ function: "Masked by a function your team owns.",
+ },
+ written: {
+ standard: ["Standard", ["governance.masks"]],
+ contract: ["DataContract", ["schema.*.properties.*.classification"]],
+ },
+ },
+ "access.filters": {
+ field: ["Standard", "governance.filters"],
+ description:
+ "A row filter of the standard's `governance.filters`, on each table with a column its `match` names.",
+ meanings: { true: "The table has the filter." },
+ written: { standard: ["Standard", ["governance.filters"]] },
+ },
+};
+
+/** One intent path: what it means, its values with theirs, and the places that can write it. */
+export interface IntentPathInfo {
+ path: string;
+ /** The field that describes it, in the documents reference. */
+ field: FieldInfo;
+ /** What it means, when the field's own description says only part of it. */
+ description?: string;
+ values: { value: string; description?: string }[];
+ /** The places that write it, in the order they win: `standard`, `layer`, `flow`, `flow table`, `transform`, or `contract`. */
+ places: string[];
+}
+
+/**
+ * Each intent path of the vocabulary, in its order, described by the
+ * documents' own fields. Throws for a path no field describes, and for a field
+ * a path names that the documents do not have, so a new or renamed path fails
+ * the build until it is described.
+ */
+export function intentVocabulary(): IntentPathInfo[] {
+ const fields: Record = {
+ Standard: documentReference("Standard"),
+ Flow: documentReference("Flow"),
+ Transform: documentReference("Transform"),
+ DataContract: documentReference("DataContract"),
+ };
+ const find = (kind: WritingKind, path: string) =>
+ fields[kind].find((candidate) => fieldPath(candidate) === path);
+ const has = (kind: WritingKind, path: string) => find(kind, path) !== undefined;
+ return Object.entries(intentPaths).map(([path, values]) => {
+ const described = intentFields[path];
+ const field = described && find(...described.field);
+ if (!described || !field) throw new Error(`No field describes the intent path "${path}".`);
+ const meanings = new Map(
+ (field.values ?? []).map((item) => [String(item.value), item.description]),
+ );
+ for (const [value, meaning] of Object.entries(described.meanings ?? {}))
+ meanings.set(value, meaning);
+ let places: string[];
+ if (described.written) {
+ places = Object.entries(described.written).map(([place, [kind, paths]]) => {
+ for (const at of paths)
+ if (!has(kind, at))
+ throw new Error(`The ${kind} document has no field "${at}" (${path}).`);
+ return place;
+ });
+ } else {
+ // The standard writes the load mode as `load.default`, and deletes as
+ // `load.deletes`; a cursor or a delete condition only for one flow.
+ const inStandard = (
+ { "load.mode": "load.default", "load.deletes.mode": "load.deletes" } as Record<
+ string,
+ string
+ >
+ )[path];
+ places = [
+ has("Standard", inStandard ?? path) && "standard",
+ has("Standard", `layers.*.${path}`) && "layer",
+ has("Flow", path) && "flow",
+ has("Flow", `tables.*.${path}`) && "flow table",
+ has("Transform", path) && "transform",
+ ].filter((place): place is string => typeof place === "string");
+ }
+ return {
+ path,
+ field,
+ ...(described.description && { description: described.description }),
+ values: values.map((value) => ({
+ value,
+ ...(meanings.get(value) && { description: meanings.get(value) }),
+ })),
+ places,
+ };
+ });
+}
diff --git a/lib/search.ts b/lib/search.ts
index c87bf43..70ec00e 100644
--- a/lib/search.ts
+++ b/lib/search.ts
@@ -6,7 +6,16 @@ import { codeHelp } from "@ingestron/definitions";
import type { StructuredData } from "fumadocs-core/mdx-plugins";
import { ingestionRows } from "./connector-settings";
import { categoryAnchor, connectorSections, sourceCategories } from "./connector-page";
-import { anchorId, connectorEntries, fieldPath, packPages, packs, settingRows } from "./reference";
+import {
+ anchorId,
+ connectorEntries,
+ fieldPath,
+ intentRows,
+ packPages,
+ packs,
+ platformSectionFields,
+ settingRows,
+} from "./reference";
/** Headings and paragraphs to add to a page's search entries, by page URL. */
export function extraStructure(url: string): StructuredData {
@@ -30,19 +39,34 @@ function errorCodes(): StructuredData {
};
}
+/** A pack's page: its methods and settings, its section of the Platform, and its intent. */
function methods(pack: string): StructuredData {
const found = packs.find((candidate) => candidate.id === pack);
const all = Object.entries(found?.methods ?? {}).map(([id, method]) => ({ pack, id, method }));
+ const section = platformSectionFields(pack);
return {
- headings: all.map(({ id }) => ({ id, content: id })),
- contents: all.flatMap(({ pack, id, method }) => [
- { heading: id, content: method.description },
- ...settingRows(pack, id).map((field) => ({
- heading: id,
- content:
- `${field.path.join(".")}: ${field.description ?? ""} ${(field.values ?? []).map((item) => item.value).join(", ")}`.trim(),
+ headings: [
+ ...all.map(({ id }) => ({ id, content: id })),
+ ...section.map((field) => ({ id: anchorId(field), content: fieldPath(field) })),
+ ],
+ contents: [
+ ...all.flatMap(({ pack, id, method }) => [
+ { heading: id, content: method.description },
+ ...settingRows(pack, id).map((field) => ({
+ heading: id,
+ content:
+ `${field.path.join(".")}: ${field.description ?? ""} ${(field.values ?? []).map((item) => item.value).join(", ")}`.trim(),
+ })),
+ ]),
+ ...section.map((field) => ({
+ heading: anchorId(field),
+ content: `${fieldPath(field)}: ${field.description ?? ""} ${field.find ?? ""}`.trim(),
})),
- ]),
+ ...intentRows(pack).map(({ path, values, use }) => ({
+ heading: "intent",
+ content: `${path} ${values.join(", ")}: ${use.use === "used" ? use.does : use.reason}`,
+ })),
+ ],
};
}
diff --git a/package.json b/package.json
index 052d842..1992e38 100644
--- a/package.json
+++ b/package.json
@@ -23,14 +23,15 @@
"sync:vscode": "node scripts/sync-vscode-commands.mjs"
},
"dependencies": {
- "@ingestron/authoring": "^2.2.0",
- "@ingestron/cli": "^2.2.0",
- "@ingestron/core": "^1.0.0",
- "@ingestron/definitions": "^1.11.0",
- "@ingestron/pack-adf": "^2.1.0",
- "@ingestron/pack-databricks": "^2.1.0",
- "@ingestron/pack-duckdb": "^2.0.0",
- "@ingestron/pack-synapse": "^2.0.0",
+ "@ingestron/authoring": "^2.3.0",
+ "@ingestron/cli": "^2.3.0",
+ "@ingestron/core": "^1.1.0",
+ "@ingestron/definitions": "^1.13.0",
+ "@ingestron/pack-adf": "^2.3.0",
+ "@ingestron/pack-databricks": "^2.3.0",
+ "@ingestron/pack-duckdb": "^2.2.0",
+ "@ingestron/pack-synapse": "^2.2.0",
+ "@ingestron/standards": "^1.0.0",
"fflate": "0.8.2",
"fumadocs-core": "16.17.8",
"fumadocs-mdx": "15.4.10",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index ee9f2f9..886d88c 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -9,29 +9,32 @@ importers:
.:
dependencies:
'@ingestron/authoring':
- specifier: ^2.2.0
- version: 2.2.0
+ specifier: ^2.3.0
+ version: 2.3.0
'@ingestron/cli':
- specifier: ^2.2.0
- version: 2.2.0
+ specifier: ^2.3.0
+ version: 2.3.0
'@ingestron/core':
- specifier: ^1.0.0
- version: 1.0.0
+ specifier: ^1.1.0
+ version: 1.1.0
'@ingestron/definitions':
- specifier: ^1.11.0
- version: 1.11.0
+ specifier: ^1.13.0
+ version: 1.13.0
'@ingestron/pack-adf':
- specifier: ^2.1.0
- version: 2.1.0(@ingestron/definitions@1.11.0)
+ specifier: ^2.3.0
+ version: 2.3.0(@ingestron/definitions@1.13.0)
'@ingestron/pack-databricks':
- specifier: ^2.1.0
- version: 2.1.0(@ingestron/definitions@1.11.0)
+ specifier: ^2.3.0
+ version: 2.3.0(@ingestron/definitions@1.13.0)
'@ingestron/pack-duckdb':
- specifier: ^2.0.0
- version: 2.0.0(@ingestron/definitions@1.11.0)
+ specifier: ^2.2.0
+ version: 2.2.0(@ingestron/definitions@1.13.0)
'@ingestron/pack-synapse':
- specifier: ^2.0.0
- version: 2.0.0(@ingestron/definitions@1.11.0)
+ specifier: ^2.2.0
+ version: 2.2.0(@ingestron/definitions@1.13.0)
+ '@ingestron/standards':
+ specifier: ^1.0.0
+ version: 1.0.0
fflate:
specifier: 0.8.2
version: 0.8.2
@@ -786,86 +789,58 @@ packages:
cpu: [x64]
os: [win32]
- '@ingestron/authoring@2.2.0':
- resolution: {integrity: sha512-XkKNbuM8hR+X08oyvXJSEJmQZb/03O6WkQNI/vTfOxVT+L/y/hf3+o19v3ulTblrFkmr2CwGqXFHOQZnMULbpg==}
+ '@ingestron/authoring@2.3.0':
+ resolution: {integrity: sha512-ZtYvDOXzAglpLY607pCyTgYViSHAP98xJQr9TZ+nPxrT83oUg2khpS+e12j1lt3HJ3S1P9s84tcpiIspV+KRhA==}
engines: {node: '>=22.13.0'}
- '@ingestron/cli@2.2.0':
- resolution: {integrity: sha512-GSp3tNWykrHxRET87Sz25wfSkhajHl5wcedJQf3erknMEAcTjySnSM62BRzQg4GPOyx+gLExuqAoBo2bOhoEAg==}
+ '@ingestron/cli@2.3.0':
+ resolution: {integrity: sha512-KdoApSz8ycGNDBwMsy2qqnhwDchqY8UkbmLxQa4yuvplR3OHsnsUe3xAFpsshA3CTeYyu73/lu8xm7fdjh6aeA==}
engines: {node: '>=22.13.0'}
hasBin: true
- '@ingestron/config@1.5.0':
- resolution: {integrity: sha512-944AP9ehRUE4PL2GfUc8qEsadugYwWgCV3E9KNEcYRwGZYEU+iR8bgLUxjlETphV/dv2EkdP0B6x2Pq1c6BaRw==}
- engines: {node: '>=22.13.0'}
-
- '@ingestron/config@1.6.0':
- resolution: {integrity: sha512-Jz1XOgeh9uv5GgzNCNB3l1U44Bd5Gkf9UbyEWyBhJ4x85+OMV91Pjss8t4Mtz7RRGVtmYCV1uagUzOC+CLaHhg==}
+ '@ingestron/config@1.7.0':
+ resolution: {integrity: sha512-ybWLmq4xf4RARirY24MxqwzAyS0KDMLMuoWVW71zRxaCXzclDeuKjNx8UYt6Kfhcqwy6aXty2zhJQhj/7LMmbQ==}
engines: {node: '>=22.13.0'}
- '@ingestron/core@1.0.0':
- resolution: {integrity: sha512-1kTD6Tre/+3FUaEEwN7a5RIw4JffOXSSb10GzSzqmpPIyvTNLxyrc3zQjAtKQlans0/qqwW7LcIbdQSDknjXsw==}
+ '@ingestron/core@1.1.0':
+ resolution: {integrity: sha512-6e1u6sJNRRsjcMLLhryH0gjDqCl7N2Exvc3CVEo1blvhvriSJBFsjt78tfQDfHHeL/CIcovFKGR5czEPF5eUJg==}
engines: {node: '>=22.13.0'}
- '@ingestron/definitions@1.11.0':
- resolution: {integrity: sha512-+llUPmc2v9B7iIrajrDldxHETAa0swcrOAE4rzJWQ48gmK1wrpPOtmI4Nh8WFJQHBWeon3xM1zjmaiQhTOFDAg==}
- engines: {node: '>=22.13.0'}
-
- '@ingestron/definitions@1.12.0':
- resolution: {integrity: sha512-ZC1tQx46SsuwkqbMm2wDuExfNk6GN8rvi28PBQHyqUmtCi/ON+aASYZs7o9XtqLR1td5ajutJGT48fKR/TKmIw==}
+ '@ingestron/definitions@1.13.0':
+ resolution: {integrity: sha512-/hqZ10uvdlZtVWcyKEB1aPc9rx0WZhjJ3dmG8v3cbcgoAg1maDOT4z54fZpA+7eQY4Yxjj4XWBuwzSRtcjWACQ==}
engines: {node: '>=22.13.0'}
'@ingestron/generate@1.1.0':
resolution: {integrity: sha512-Ouhps0spdV4ueubGYbx2ZGenTkAyA8YvuhuDpHi19uX33RfxlM5ILRSUwnZhg91lPcmJmxYrHnGyfuU98bp2Vw==}
engines: {node: '>=22.13.0'}
- '@ingestron/pack-adf@2.1.0':
- resolution: {integrity: sha512-8PFU8QS3/inSU0htuaFrR6dDNo5gvPXVF3QQNpBo+p3zNYnGnjD66A2gYagQqNdUSxqxK15iGFkSqlRByoZsYw==}
- engines: {node: '>=22.13.0'}
- peerDependencies:
- '@ingestron/definitions': ^1.9.0
-
- '@ingestron/pack-adf@2.2.0':
- resolution: {integrity: sha512-1J+M4DlF/ybDkVmDxKyDLWv/VFUHNnuMUBg9oB99JrWwsxDO2BoLQCVRrJWVMGi4J8jxvc48bZuYlxVNW5mEHw==}
- engines: {node: '>=22.13.0'}
- peerDependencies:
- '@ingestron/definitions': ^1.9.0
-
- '@ingestron/pack-databricks@2.1.0':
- resolution: {integrity: sha512-kMRq2IO8uagIuQdaE+EgLVhL1tD4yD+Xxdz3dJToQS7Kyf42q78EBphA5mkfukVgZHvEUmpw9ZR9vQg5lRaQog==}
- engines: {node: '>=22.13.0'}
- peerDependencies:
- '@ingestron/definitions': ^1.9.0
-
- '@ingestron/pack-databricks@2.2.0':
- resolution: {integrity: sha512-mJbbqknqH6Ve+lKJQHrag6nsVozug4wSh5WZlw/OXvua7FXSXvmYCvIGoORWXYdW9+fH5SZKEwXpOCgK+INGHQ==}
+ '@ingestron/pack-adf@2.3.0':
+ resolution: {integrity: sha512-bMW32GInQ5L1KZb5F/tVDA0iJ/UtczH16n+pzOCFJoKILPr1uu2JIuGWRW4SWlERAHhcCq1/8B7SwBkeWz4CvQ==}
engines: {node: '>=22.13.0'}
peerDependencies:
- '@ingestron/definitions': ^1.9.0
+ '@ingestron/definitions': ^1.13.0
- '@ingestron/pack-duckdb@2.0.0':
- resolution: {integrity: sha512-GVQpprSyw7+vKdSF0xkj/uX6Z/1Cblco/qIgnGLvUXaRnYUE6UaMj4Q3qFP0wiA1DxguJIYw002kYm6SF5HIWA==}
+ '@ingestron/pack-databricks@2.3.0':
+ resolution: {integrity: sha512-LZ0+l/dGBWvxTKcOQgvvMWI10HNdWb2XAIab2CMlEMUrXt4ITMDMfsjCx8HfSdUbsWE/t6vnNozHOO+8ZBDczQ==}
engines: {node: '>=22.13.0'}
peerDependencies:
- '@ingestron/definitions': ^1.3.1
+ '@ingestron/definitions': ^1.13.0
- '@ingestron/pack-duckdb@2.1.0':
- resolution: {integrity: sha512-XmIGJ+vaYvT3cm/BQTGVZ0wNyhBKkPSsCuA/Pc4zehw0xbG4uKMV15EnFdo52dri3cqxnk0wIzyGE0a41rcsWQ==}
+ '@ingestron/pack-duckdb@2.2.0':
+ resolution: {integrity: sha512-x/C90ZTerzT2anRPRNlTy3hdmU+l+7cYCv5TaLoz/gbj5ZiiTXn75fxZE8WSV5tRaZC2qL+wpBvw+/YaU3vIkQ==}
engines: {node: '>=22.13.0'}
peerDependencies:
- '@ingestron/definitions': ^1.3.1
+ '@ingestron/definitions': ^1.13.0
- '@ingestron/pack-synapse@2.0.0':
- resolution: {integrity: sha512-R15xhzEt4ptoww23WKVw2JGDmurL4RbEHmv1hEtVefj/cStqicA5HnddGx0LU8ZA7Y+SVpLCW44OlyLenUh8Eg==}
+ '@ingestron/pack-synapse@2.2.0':
+ resolution: {integrity: sha512-6Xrfg+BZ+oh1nEmMbhou0X6mc5cxtYP3DuEuLqJa8vPs8q+nFaCo6B4XERbclyb+lBIzuu0WVXfcrlUQbdAAgA==}
engines: {node: '>=22.13.0'}
peerDependencies:
- '@ingestron/definitions': ^1.9.0
+ '@ingestron/definitions': ^1.13.0
- '@ingestron/pack-synapse@2.1.0':
- resolution: {integrity: sha512-uIOy0yqUwYSIIUVFH6n4xKnoUI2Lw0KTtvSZvg4l/qQTxwm1JvbQneJWVqTpCNh/RSfqc4bO6+HPva7LIQ10Qw==}
+ '@ingestron/standards@1.0.0':
+ resolution: {integrity: sha512-QXO4JfaWKZ5fdqYo3uxaPTk+zRLbFLT+++6Tvj+7zFdVBxt7U/SJKC4B8hYgTVQ8k0XjeCHsv4j6ePImNp92MA==}
engines: {node: '>=22.13.0'}
- peerDependencies:
- '@ingestron/definitions': ^1.9.0
'@jridgewell/gen-mapping@0.3.13':
resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==}
@@ -3359,94 +3334,67 @@ snapshots:
'@img/sharp-win32-x64@0.35.5':
optional: true
- '@ingestron/authoring@2.2.0':
+ '@ingestron/authoring@2.3.0':
dependencies:
- '@ingestron/config': 1.6.0
- '@ingestron/core': 1.0.0
- '@ingestron/definitions': 1.12.0
+ '@ingestron/config': 1.7.0
+ '@ingestron/core': 1.1.0
+ '@ingestron/definitions': 1.13.0
'@ingestron/generate': 1.1.0
yaml: 2.9.1
- '@ingestron/cli@2.2.0':
+ '@ingestron/cli@2.3.0':
dependencies:
- '@ingestron/config': 1.6.0
- '@ingestron/core': 1.0.0
- '@ingestron/definitions': 1.12.0
+ '@ingestron/config': 1.7.0
+ '@ingestron/core': 1.1.0
+ '@ingestron/definitions': 1.13.0
'@ingestron/generate': 1.1.0
yaml: 2.9.1
- '@ingestron/config@1.5.0':
- dependencies:
- ajv: 8.20.0
- yaml: 2.9.1
-
- '@ingestron/config@1.6.0':
+ '@ingestron/config@1.7.0':
dependencies:
ajv: 8.20.0
yaml: 2.9.1
- '@ingestron/core@1.0.0':
+ '@ingestron/core@1.1.0':
dependencies:
- '@ingestron/config': 1.6.0
- '@ingestron/definitions': 1.12.0
+ '@ingestron/config': 1.7.0
+ '@ingestron/definitions': 1.13.0
'@ingestron/generate': 1.1.0
- '@ingestron/pack-adf': 2.2.0(@ingestron/definitions@1.12.0)
- '@ingestron/pack-databricks': 2.2.0(@ingestron/definitions@1.12.0)
- '@ingestron/pack-duckdb': 2.1.0(@ingestron/definitions@1.12.0)
- '@ingestron/pack-synapse': 2.1.0(@ingestron/definitions@1.12.0)
- yaml: 2.9.1
-
- '@ingestron/definitions@1.11.0':
- dependencies:
- '@ingestron/config': 1.5.0
- ajv: 8.20.0
- liquidjs: 10.30.0
+ '@ingestron/pack-adf': 2.3.0(@ingestron/definitions@1.13.0)
+ '@ingestron/pack-databricks': 2.3.0(@ingestron/definitions@1.13.0)
+ '@ingestron/pack-duckdb': 2.2.0(@ingestron/definitions@1.13.0)
+ '@ingestron/pack-synapse': 2.2.0(@ingestron/definitions@1.13.0)
+ '@ingestron/standards': 1.0.0
yaml: 2.9.1
- '@ingestron/definitions@1.12.0':
+ '@ingestron/definitions@1.13.0':
dependencies:
- '@ingestron/config': 1.6.0
+ '@ingestron/config': 1.7.0
ajv: 8.20.0
liquidjs: 10.30.0
yaml: 2.9.1
'@ingestron/generate@1.1.0': {}
- '@ingestron/pack-adf@2.1.0(@ingestron/definitions@1.11.0)':
- dependencies:
- '@ingestron/definitions': 1.11.0
-
- '@ingestron/pack-adf@2.2.0(@ingestron/definitions@1.12.0)':
- dependencies:
- '@ingestron/definitions': 1.12.0
-
- '@ingestron/pack-databricks@2.1.0(@ingestron/definitions@1.11.0)':
+ '@ingestron/pack-adf@2.3.0(@ingestron/definitions@1.13.0)':
dependencies:
- '@ingestron/definitions': 1.11.0
- liquidjs: 10.30.0
- yaml: 2.9.1
+ '@ingestron/definitions': 1.13.0
- '@ingestron/pack-databricks@2.2.0(@ingestron/definitions@1.12.0)':
+ '@ingestron/pack-databricks@2.3.0(@ingestron/definitions@1.13.0)':
dependencies:
- '@ingestron/definitions': 1.12.0
+ '@ingestron/definitions': 1.13.0
liquidjs: 10.30.0
yaml: 2.9.1
- '@ingestron/pack-duckdb@2.0.0(@ingestron/definitions@1.11.0)':
+ '@ingestron/pack-duckdb@2.2.0(@ingestron/definitions@1.13.0)':
dependencies:
- '@ingestron/definitions': 1.11.0
+ '@ingestron/definitions': 1.13.0
- '@ingestron/pack-duckdb@2.1.0(@ingestron/definitions@1.12.0)':
+ '@ingestron/pack-synapse@2.2.0(@ingestron/definitions@1.13.0)':
dependencies:
- '@ingestron/definitions': 1.12.0
+ '@ingestron/definitions': 1.13.0
- '@ingestron/pack-synapse@2.0.0(@ingestron/definitions@1.11.0)':
- dependencies:
- '@ingestron/definitions': 1.11.0
-
- '@ingestron/pack-synapse@2.1.0(@ingestron/definitions@1.12.0)':
- dependencies:
- '@ingestron/definitions': 1.12.0
+ '@ingestron/standards@1.0.0': {}
'@jridgewell/gen-mapping@0.3.13':
dependencies:
diff --git a/public/images/vscode/completion-dark.png b/public/images/vscode/completion-dark.png
deleted file mode 100644
index 19b8bac..0000000
Binary files a/public/images/vscode/completion-dark.png and /dev/null differ
diff --git a/public/images/vscode/completion-light.png b/public/images/vscode/completion-light.png
deleted file mode 100644
index b6d3a10..0000000
Binary files a/public/images/vscode/completion-light.png and /dev/null differ
diff --git a/public/images/vscode/context-menu-dark.png b/public/images/vscode/context-menu-dark.png
index f76b70b..5046b05 100644
Binary files a/public/images/vscode/context-menu-dark.png and b/public/images/vscode/context-menu-dark.png differ
diff --git a/public/images/vscode/context-menu-light.png b/public/images/vscode/context-menu-light.png
index e37644e..9952886 100644
Binary files a/public/images/vscode/context-menu-light.png and b/public/images/vscode/context-menu-light.png differ
diff --git a/public/images/vscode/editor-dark.png b/public/images/vscode/editor-dark.png
deleted file mode 100644
index 36b0846..0000000
Binary files a/public/images/vscode/editor-dark.png and /dev/null differ
diff --git a/public/images/vscode/editor-light.png b/public/images/vscode/editor-light.png
deleted file mode 100644
index cdc517e..0000000
Binary files a/public/images/vscode/editor-light.png and /dev/null differ
diff --git a/public/images/vscode/environment-dark.png b/public/images/vscode/environment-dark.png
index 41d6f51..e9cd640 100644
Binary files a/public/images/vscode/environment-dark.png and b/public/images/vscode/environment-dark.png differ
diff --git a/public/images/vscode/environment-light.png b/public/images/vscode/environment-light.png
index 88e8baf..4855854 100644
Binary files a/public/images/vscode/environment-light.png and b/public/images/vscode/environment-light.png differ
diff --git a/public/images/vscode/first-project-dark.png b/public/images/vscode/first-project-dark.png
index ff4973f..8da90a6 100644
Binary files a/public/images/vscode/first-project-dark.png and b/public/images/vscode/first-project-dark.png differ
diff --git a/public/images/vscode/first-project-light.png b/public/images/vscode/first-project-light.png
index e893d28..1dea516 100644
Binary files a/public/images/vscode/first-project-light.png and b/public/images/vscode/first-project-light.png differ
diff --git a/public/images/vscode/hover-dark.png b/public/images/vscode/hover-dark.png
deleted file mode 100644
index 6943be5..0000000
Binary files a/public/images/vscode/hover-dark.png and /dev/null differ
diff --git a/public/images/vscode/hover-light.png b/public/images/vscode/hover-light.png
deleted file mode 100644
index 68b194e..0000000
Binary files a/public/images/vscode/hover-light.png and /dev/null differ
diff --git a/public/images/vscode/new-project-dark.png b/public/images/vscode/new-project-dark.png
deleted file mode 100644
index 5656426..0000000
Binary files a/public/images/vscode/new-project-dark.png and /dev/null differ
diff --git a/public/images/vscode/new-project-light.png b/public/images/vscode/new-project-light.png
deleted file mode 100644
index 58e1143..0000000
Binary files a/public/images/vscode/new-project-light.png and /dev/null differ
diff --git a/public/images/vscode/problems-dark.png b/public/images/vscode/problems-dark.png
index 3450c61..93033f1 100644
Binary files a/public/images/vscode/problems-dark.png and b/public/images/vscode/problems-dark.png differ
diff --git a/public/images/vscode/problems-light.png b/public/images/vscode/problems-light.png
index 56c131a..771f6dd 100644
Binary files a/public/images/vscode/problems-light.png and b/public/images/vscode/problems-light.png differ
diff --git a/public/images/vscode/project-view-dark.png b/public/images/vscode/project-view-dark.png
index 62dc2d8..b42df1b 100644
Binary files a/public/images/vscode/project-view-dark.png and b/public/images/vscode/project-view-dark.png differ
diff --git a/public/images/vscode/project-view-light.png b/public/images/vscode/project-view-light.png
index f251b2c..2ace8cb 100644
Binary files a/public/images/vscode/project-view-light.png and b/public/images/vscode/project-view-light.png differ
diff --git a/public/images/vscode/title-buttons-dark.png b/public/images/vscode/title-buttons-dark.png
index a0473cf..f8372ca 100644
Binary files a/public/images/vscode/title-buttons-dark.png and b/public/images/vscode/title-buttons-dark.png differ
diff --git a/public/images/vscode/title-buttons-light.png b/public/images/vscode/title-buttons-light.png
index b010538..ec44f47 100644
Binary files a/public/images/vscode/title-buttons-light.png and b/public/images/vscode/title-buttons-light.png differ
diff --git a/scripts/ci-templates.test.mjs b/scripts/ci-templates.test.mjs
index db4561a..858ebdb 100644
--- a/scripts/ci-templates.test.mjs
+++ b/scripts/ci-templates.test.mjs
@@ -58,7 +58,7 @@ test("every YAML template parses, and the folder holds no other kind of file", (
* The ingestron bundle the templates install: an exact version, the one the
* docs build with (test/standards-and-teams.test.ts checks the cli's version).
*/
-const bundle = "2.2.0";
+const bundle = "2.3.0";
test("each check installs an exact ingestron from npm, then runs ingestron check", () => {
for (const file of [...checks, "github-actions/deploy.yml", "azure-devops/azure-pipelines.yml"]) {
diff --git a/test/accept-a-warning.test.ts b/test/accept-a-warning.test.ts
new file mode 100644
index 0000000..ad7453f
--- /dev/null
+++ b/test/accept-a-warning.test.ts
@@ -0,0 +1,76 @@
+// "Accept a warning", run as the page shows it on a copy of the quality
+// example: each list the page writes is added, the plan is run, and what it
+// prints before the files is compared with the page.
+import { readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { parse } from "yaml";
+import { readExampleFile } from "../lib/examples.js";
+import { cli, copy, fences, read, section } from "./pages.js";
+
+const page = read("how-to/accept-a-warning.mdx");
+
+/** Plans a project, which must succeed, and returns what it prints. */
+function plan(cwd: string): string {
+ const result = cli(cwd, ["plan", "--no-diff"]);
+ expect(result.code, result.output).toBe(0);
+ return result.output;
+}
+
+describe("how-to/accept-a-warning", () => {
+ it("quotes the warning the quality example's synapse environment gives", () => {
+ const warning = fences(page.split("\n## ")[0]!, "text")[0]!;
+ expect(plan(copy("quality"))).toContain(warning);
+ });
+
+ it("accepts it for every environment, and reports the entry unused where it matches nothing", () => {
+ const steps = section(page, "## Accept one warning [#accept]");
+ const cwd = copy("quality");
+ const platform = join(cwd, "platform.yaml");
+ writeFileSync(platform, readFileSync(platform, "utf8") + fences(steps, "yaml")[0]!);
+ const output = plan(cwd);
+ expect(output.startsWith(fences(steps, "text")[0]!)).toBe(true);
+ expect(output).not.toContain("The fail action");
+ });
+
+ it("accepts it in one environment through a value, with nothing unused", () => {
+ const steps = section(page, "## Accept it in some environments only [#per-environment]");
+ const [list, project] = fences(steps, "yaml");
+ // The page's project.yaml is the example's, with the lists added.
+ const shown = parse(project!);
+ const example = parse(readExampleFile("quality", "project.yaml"));
+ const { accept, ...synapse } = shown.environments.synapse.values;
+ expect(accept).toHaveLength(1);
+ expect(synapse).toEqual(example.environments.synapse.values);
+ expect(shown.values).toEqual({ accept: [] });
+ expect({ ...shown, values: undefined, environments: undefined }).toEqual({
+ ...example,
+ values: undefined,
+ environments: undefined,
+ });
+ // The same entry the page accepted for every environment.
+ expect(accept).toEqual(
+ parse(fences(section(page, "## Accept one warning [#accept]"), "yaml")[0]!).accept,
+ );
+ const cwd = copy("quality");
+ const platform = join(cwd, "platform.yaml");
+ writeFileSync(platform, readFileSync(platform, "utf8") + list!);
+ writeFileSync(join(cwd, "project.yaml"), project!);
+ const output = plan(cwd);
+ expect(output.startsWith(fences(steps, "text")[0]!)).toBe(true);
+ expect(output).not.toContain("ACCEPT_UNUSED");
+ });
+
+ it("refuses a code that is no longer reported", () => {
+ const cwd = copy("quality");
+ const platform = join(cwd, "platform.yaml");
+ writeFileSync(
+ platform,
+ `${readFileSync(platform, "utf8")}accept:\n - code: CONTRACT_FIELD_UNUSED\n reason: Old.\n`,
+ );
+ const result = cli(cwd, ["plan", "--no-diff"]);
+ expect(result.code).toBe(1);
+ expect(result.output).toContain("error ACCEPT_CODE");
+ expect(section(page, "## Say which warnings [#about]")).toContain("`CONTRACT_FIELD_UNUSED`");
+ });
+});
diff --git a/test/areas.test.ts b/test/areas.test.ts
index 30346c6..3cb0640 100644
--- a/test/areas.test.ts
+++ b/test/areas.test.ts
@@ -33,7 +33,9 @@ const areas: Record = {
"/docs/how-to/write-a-transform",
"/docs/how-to/keep-a-table-current",
"/docs/how-to/add-quality-checks",
+ "/docs/how-to/cleanse-data",
"/docs/how-to/define-a-data-product",
+ "/docs/how-to/grant-access",
"/docs/how-to/add-an-environment",
"/docs/how-to/plan-and-generate",
"/docs/how-to/check-in-ci",
@@ -52,8 +54,14 @@ const areas: Record = {
],
"Standards and teams": [
"/docs/how-to/configure-a-standard",
+ "/docs/how-to/choose-a-published-standard",
+ "/docs/how-to/start-from-a-published-standard",
+ "/docs/how-to/upgrade-a-standard",
+ "/docs/how-to/kits/duckdb-local-medallion",
+ "/docs/how-to/kits/databricks-lakehouse-streaming",
"/docs/how-to/name-things-your-way",
"/docs/how-to/ask-the-platform-team",
+ "/docs/how-to/accept-a-warning",
"/docs/how-to/share-a-template",
"/docs/how-to/upgrade",
],
@@ -62,6 +70,7 @@ const areas: Record = {
"/docs/concepts/standards-and-packs",
"/docs/concepts/settings",
"/docs/concepts/layers",
+ "/docs/concepts/not-applied",
"/docs/concepts/environments-and-platform",
"/docs/concepts/yaml-references",
"/docs/concepts/generated-files",
@@ -69,6 +78,7 @@ const areas: Record = {
Connectors: ["/docs/connectors"],
Reference: [
"/docs/reference/documents",
+ "/docs/reference/intent",
"/docs/reference/packs",
"/docs/reference/packs/databricks",
"/docs/reference/packs/adf",
@@ -188,7 +198,7 @@ describe("page URLs", () => {
const built = readFileSync(join(process.cwd(), "test/page-urls.txt"), "utf8")
.trim()
.split("\n");
- expect(built).toHaveLength(127);
+ expect(built).toHaveLength(137);
expect(
source
.getPages()
diff --git a/test/build-a-project.test.ts b/test/build-a-project.test.ts
index f21e52e..44fd6d4 100644
--- a/test/build-a-project.test.ts
+++ b/test/build-a-project.test.ts
@@ -6,6 +6,7 @@ import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { run } from "@ingestron/cli";
+import { intentUseOf } from "@ingestron/definitions";
import { createElement } from "react";
import { renderToStaticMarkup } from "react-dom/server";
import { describe, expect, it } from "vitest";
@@ -58,6 +59,20 @@ describe("how-to/add-a-source", () => {
const contract = cli(cwd, command(commands(steps), "contract"));
expect(contract.code).toBe(0);
expect(contract.output.trim()).toBe(drafted!.trim());
+ // The key the page marks in the drafted contract.
+ expect(prose(steps)).toContain(
+ "mark `customer_id` with `primaryKey: true` and `required: true`",
+ );
+ const contractFile = join(cwd, "contracts/shop.customers.yaml");
+ const key = "{ name: customer_id, logicalType: integer }";
+ expect(readFileSync(contractFile, "utf8")).toContain(key);
+ writeFileSync(
+ contractFile,
+ readFileSync(contractFile, "utf8").replace(
+ key,
+ "{ name: customer_id, logicalType: integer, primaryKey: true, required: true }",
+ ),
+ );
// The line the page adds under the flow's `tables`.
merge(join(cwd, "flows/orders.yaml"), `tables:\n${fences(steps, "yaml")[0]!}`);
@@ -103,6 +118,25 @@ describe("how-to/add-a-source", () => {
});
});
+describe("how-to/write-a-transform", () => {
+ it("builds a silver copy of the first project's orders from a transform with no code", () => {
+ const steps = section(
+ read("how-to/write-a-transform.mdx"),
+ "## Leave out the code [#without-code]",
+ );
+ expect(prose(steps)).toContain("saved as `transforms/orders_clean.yaml`");
+ const cwd = shop();
+ writeFileSync(join(cwd, "transforms/orders_clean.yaml"), fences(steps, "yaml")[0]!);
+ const plan = cli(cwd, ["plan", "--no-diff"]);
+ expect(plan.code, plan.output).toBe(0);
+ expect(plan.output).toMatch(/^create +dev\/tables\/silver\.orders_clean\.sql$/m);
+ expect(cli(cwd, ["apply", "--no-diff"]).code).toBe(0);
+ const sql = readFileSync(join(cwd, "build/duckdb/dev/tables/silver.orders_clean.sql"), "utf8");
+ expect(sql).toContain(`nullif(trim(CAST("customer" AS VARCHAR)), '') AS "customer"`);
+ expect(sql).toContain(`SELECT * FROM "bronze"."orders"`);
+ });
+});
+
describe("how-to/keep-a-table-current", () => {
const page = read("how-to/keep-a-table-current.mdx");
@@ -117,33 +151,24 @@ describe("how-to/keep-a-table-current", () => {
expect(after).toContain('ORDER BY "ordered_at" DESC, "_ingestron_file" DESC');
});
- it("compares the modes each engine's store method accepts", () => {
+ it("compares what each engine does with the intent, from the packs' own coverage", () => {
const table = page.split("\n## ")[0]!;
- const row = (engine: string) =>
- table
- .split("\n")
- .find((line) => line.startsWith(`| ${engine} `))!
- .split("|")
- .slice(2, -1)
- .map((cell) => cell.trim());
- const duckdb = accepted("duckdb", "duckdb-table", "mode");
- expect(row("DuckDB")).toEqual(["`upsert`", "No", "No"]);
- expect(duckdb).toContain("upsert");
- expect(duckdb).not.toContain("history");
- expect(
- settingRows("duckdb", "duckdb-table").map((field) => field.path.join(".")),
- ).not.toContain("delete_when");
- const databricks = accepted("databricks", "streaming-table", "mode");
- expect(row("Databricks")).toEqual(["`upsert`", "`history`", "`delete_when`"]);
- expect(databricks).toEqual(expect.arrayContaining(["upsert", "history"]));
- expect(
- settingRows("databricks", "streaming-table").map((field) => field.path.join(".")),
- ).toContain("delete_when");
- // A Synapse view has no mode at all.
- expect(row("Synapse")).toEqual(["No", "No", "No"]);
- expect(
- settingRows("synapse", "synapse-view").map((field) => field.path.join(".")),
- ).not.toContain("mode");
+ expect(table).toContain("
+ intentUseOf(
+ packs.find((item) => item.id === pack)!,
+ path,
+ value,
+ )?.use;
+ expect(used("duckdb", "history", "latest")).toBe("used");
+ expect(used("duckdb", "history", "full")).not.toBe("used");
+ expect(used("databricks", "history", "full")).toBe("used");
+ expect(used("databricks", "load.deletes.when")).toBe("used");
+ expect(used("synapse", "history", "latest")).toBe("not-applicable");
+ // The settings each intent stands for, which the page names.
+ expect(accepted("duckdb", "duckdb-table", "mode")).toContain("upsert");
+ expect(page).toContain("`mode: upsert`");
});
});
@@ -222,7 +247,7 @@ describe("how-to/write-a-contract", () => {
it("checks rows by a column's logicalTypeOptions, and unique once checks are on", () => {
const text = prose(section(page, "## Rules about the data"));
expect(text).toContain("such as `customer_id_minimum`");
- expect(text).toContain("when the platform turns on `manage.checks`");
+ expect(text).toContain("when the platform turns on `databricks.manage.checks`");
const cwd = copy("minimal");
const contract = join(cwd, "contracts/orders.yaml");
writeFileSync(
@@ -243,7 +268,7 @@ describe("how-to/write-a-contract", () => {
// With checks on, unique becomes a check over all rows after the refresh.
writeFileSync(
join(cwd, "platform.yaml"),
- "apiVersion: ingestron.io/v1\nkind: Platform\nid: checks\nmanage: { checks: true }\n",
+ "apiVersion: ingestron.io/v1\nkind: Platform\nid: checks\ndatabricks: { manage: { checks: true } }\n",
);
const project = parse(readFileSync(join(cwd, "project.yaml"), "utf8"));
writeFileSync(
diff --git a/test/cleanse-data.test.ts b/test/cleanse-data.test.ts
new file mode 100644
index 0000000..c5f161b
--- /dev/null
+++ b/test/cleanse-data.test.ts
@@ -0,0 +1,130 @@
+// "Cleanse data in the right layer", run as the page shows it: the project is
+// created from the published standard with the page's command, each file the
+// page writes is written from its block, each command is run, and each output
+// block is compared with what the command prints.
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { cli, commands, fences, read, section } from "./pages.js";
+
+const page = read("how-to/cleanse-data.mdx");
+const prose = (text: string) => text.replace(/\s+/g, " ");
+const duckdb = spawnSync("duckdb", ["--version"], { encoding: "utf8" }).status === 0;
+const withDuckdb = duckdb ? "" : " (skipped: the duckdb command line is not on the PATH)";
+
+/** The words of a shell line, with double-quoted words kept whole. */
+const words = (line: string) =>
+ [...line.matchAll(/"([^"]*)"|(\S+)/g)].map((match) => match[1] ?? match[2]!);
+
+/** The project the page creates, in a new folder, with the page's own commands. */
+function created(): string {
+ const [init, cd] = commands(section(page, "## Create the project"));
+ expect(init).toMatch(/^ingestron init shop --kit duckdb\/local-medallion@/);
+ const parent = mkdtempSync(join(tmpdir(), "docs-cleanse-"));
+ expect(cli(parent, words(init!).slice(1)).code).toBe(0);
+ expect(cd).toBe("cd shop");
+ return join(parent, "shop");
+}
+
+/** Replaces text the file must hold. */
+function change(file: string, from: string, to: string): void {
+ const text = readFileSync(file, "utf8");
+ expect(text, `${file} holds ${from}`).toContain(from);
+ writeFileSync(file, text.replace(from, to));
+}
+
+/** The project after the page's changes: the columns, the zone and the transform. */
+function changed(): string {
+ const cwd = created();
+ const steps = section(page, "## Add the columns to cleanse");
+ writeFileSync(join(cwd, "samples/orders.csv"), fences(steps, "csv")[0]!);
+ const [columns, zone] = fences(steps, "yaml");
+ change(
+ join(cwd, "contracts/orders.yaml"),
+ " - { name: ordered_at, logicalType: date }\n",
+ columns!,
+ );
+ expect(prose(steps)).toContain("Add this line to `connections/shop.yaml`, under `connectivity`");
+ change(
+ join(cwd, "connections/shop.yaml"),
+ "connectivity: public\n",
+ `connectivity: public\n${zone}`,
+ );
+ const transform = section(page, "## Add a silver table without code");
+ expect(prose(transform)).toContain("Save this as `transforms/orders_clean.yaml`");
+ writeFileSync(join(cwd, "transforms/orders_clean.yaml"), fences(transform, "yaml")[0]!);
+ return cwd;
+}
+
+describe("how-to/cleanse-data", () => {
+ it("creates the project from the published standard with no warning", () => {
+ const cwd = created();
+ const plan = cli(cwd, ["plan", "--no-diff"]);
+ expect(plan.code).toBe(0);
+ expect(plan.output).not.toContain("warning");
+ });
+
+ it("plans the cleansed silver table the page shows, with no warning", () => {
+ const cwd = changed();
+ const steps = section(page, "## Plan, apply and run");
+ const plan = cli(cwd, words(commands(steps)[0]!).slice(1));
+ expect(plan.code).toBe(0);
+ expect(plan.output.trim()).toBe(fences(steps, "text")[0]!.trim());
+ });
+
+ it(`builds the tables and shows the rows the page shows${withDuckdb}`, ({ skip }) => {
+ if (!duckdb) skip();
+ const cwd = changed();
+ const steps = section(page, "## Plan, apply and run");
+ const lines = commands(steps).slice(1);
+ let shown = "";
+ for (const line of lines) {
+ const [command, ...args] = words(line);
+ if (command === "ingestron") {
+ expect(cli(cwd, args).code).toBe(0);
+ continue;
+ }
+ expect(command).toBe("duckdb");
+ const result = spawnSync("duckdb", ["-cmd", ".mode duckbox", ...args], {
+ cwd,
+ encoding: "utf8",
+ });
+ expect(result.status, result.stderr).toBe(0);
+ // The build's own output, its checks, is described, not shown.
+ if (!args.includes("-bail")) shown += result.stdout;
+ }
+ expect(shown).toBe(fences(steps, "text")[1]);
+ });
+
+ it("refuses cleansing in bronze with the page's error, and plans again once put back", () => {
+ const cwd = changed();
+ const steps = section(page, "## Ask a layer for a duty it does not do");
+ const [asked, original] = fences(steps, "yaml");
+ const flow = join(cwd, "flows/orders.yaml");
+ // What the page puts back is what the published standard wrote.
+ expect(readFileSync(flow, "utf8")).toBe(original);
+ writeFileSync(flow, asked!);
+ const plan = cli(cwd, words(commands(steps)[0]!).slice(1));
+ expect(plan.code).toBe(1);
+ expect(plan.output.trim()).toBe(fences(steps, "text")[0]!.trim());
+ writeFileSync(flow, original!);
+ expect(cli(cwd, ["plan", "--no-diff"]).code).toBe(0);
+ });
+
+ it("says deduplicate is redundant in a silver table kept at history: latest", () => {
+ const cwd = changed();
+ const steps = section(page, "## Keep one row per key");
+ const file = join(cwd, "transforms/orders_clean.yaml");
+ const text = readFileSync(file, "utf8").replace(
+ "cleanse:\n",
+ `cleanse:\n${fences(steps, "yaml")[0]!.replace("cleanse:\n", "")}`,
+ );
+ writeFileSync(file, text);
+ const plan = cli(cwd, ["plan", "--no-diff"]);
+ expect(plan.code).toBe(0);
+ expect(plan.output).toContain("warning DEDUPLICATE_MODE");
+ expect(plan.output).toContain('Deduplicate is redundant for "orders_clean"');
+ });
+});
diff --git a/test/examples.test.ts b/test/examples.test.ts
index d0dc90f..db4054a 100644
--- a/test/examples.test.ts
+++ b/test/examples.test.ts
@@ -44,7 +44,7 @@ describe("examples", () => {
}, 30_000);
it("lists every sample with a title, summary and the facts found by resolving it", () => {
const examples = listExamples();
- expect(examples).toHaveLength(26);
+ expect(examples).toHaveLength(27);
for (const example of examples) {
expect(example.title, example.name).not.toBe("");
expect(example.summary, example.name).toMatch(/\.$/);
@@ -52,6 +52,25 @@ describe("examples", () => {
}
});
+ it("checks the rows of the guides that keep tables on DuckDB and Synapse, by their layers' duties", () => {
+ for (const name of ["duckdb-local", "data-products", "synapse-views"]) {
+ expect(readExampleFile(name, "standard.yaml"), name).toContain("does: raw");
+ const cwd = mkdtempSync(join(tmpdir(), `docs-duties-${name}-`));
+ cpSync(exampleFolder(name), cwd, { recursive: true });
+ let output = "";
+ expect(run(["apply", "--no-diff"], { cwd, write: (text) => (output += text) }), name).toBe(0);
+ expect(output, name).not.toContain("NOT_APPLIED");
+ }
+ // The passport says what checks each table, and no check goes unenforced.
+ const passport = readExampleFile("data-products", "build/products/dev/customers.md");
+ const quality = passport.slice(passport.indexOf("## Quality"), passport.indexOf("## Fresh"));
+ expect(quality).toContain("Counted only by DuckDB");
+ expect(quality).not.toContain("Not enforced");
+ // The role DuckDB cannot grant is accepted, and the passport still says so.
+ expect(readExampleFile("data-products", "platform.yaml")).toContain("feature: grants");
+ expect(passport).toContain("Not enforced on DuckDB: DuckDB has no access control.");
+ }, 30_000);
+
it("keeps an empty folder the example needs, such as DuckDB's published folder", () => {
const files = exampleFiles("duckdb-local");
expect(files).toContain("published/.gitkeep");
diff --git a/test/extracts.test.ts b/test/extracts.test.ts
index 174f7da..f4c32b7 100644
--- a/test/extracts.test.ts
+++ b/test/extracts.test.ts
@@ -232,7 +232,7 @@ describe("the platform request", () => {
it("shows the named sections, each with its heading, in the order asked for", () => {
const shown = requestSections("synapse-views", "dev", ["Report users", "Schedules"]);
- expect(shown).toMatch(/^## Report users\n\n- In `sales_dev`/);
+ expect(shown).toMatch(/^## Report users\n\n- `sales-data-engineers-dev`: in `sales_dev`/);
expect(shown).toContain("\n\n## Schedules\n\n- Data Factory trigger `tr_sales_erp`");
expect(shown).not.toContain("## ADF prerequisites");
const request = block(Request({ name: "duckdb-local", sections: ["DuckDB job"] }));
diff --git a/test/grant-access.test.ts b/test/grant-access.test.ts
new file mode 100644
index 0000000..5d51ccb
--- /dev/null
+++ b/test/grant-access.test.ts
@@ -0,0 +1,90 @@
+// "Grant access, mask columns and filter rows", run as the page shows it: each
+// output it quotes comes from planning a copy of the example it names, changed
+// as the page says, and the examples it shows plan with nothing to warn about.
+import { readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { readExampleFile } from "../lib/examples.js";
+import { cli, copy, fences, read, section } from "./pages.js";
+
+const page = read("how-to/grant-access.mdx");
+
+/** Changes one file of a copy of an example, failing when the text to change is not there. */
+function change(cwd: string, file: string, from: string, to: string): void {
+ const path = join(cwd, file);
+ const text = readFileSync(path, "utf8");
+ expect(text, `${file} holds ${from}`).toContain(from);
+ writeFileSync(path, text.replace(from, to));
+}
+
+describe("how-to/grant-access", () => {
+ it("shows examples that grant, mask and filter with nothing to warn about", () => {
+ for (const name of ["governed-databricks", "synapse-views"]) {
+ const { code, output } = cli(copy(name), ["plan", "--no-diff"]);
+ expect(code, output).toBe(0);
+ expect(output, name).not.toContain("CAPABILITY_OFF");
+ expect(output, name).not.toContain("NOT_APPLIED");
+ }
+ const task = readExampleFile(
+ "governed-databricks",
+ "build/databricks/tasks/shop/apply_contract.py",
+ );
+ for (const policy of [
+ "ingestron_mask_confidential",
+ "ingestron_mask_restricted",
+ "ingestron_filter_region_rows",
+ ])
+ expect(task).toContain(`"name": "${policy}"`);
+ });
+
+ it("quotes the warning a role gives while grants are off", () => {
+ const cwd = copy("governed-databricks");
+ change(cwd, "platform.yaml", " grants: true", " grants: false");
+ const [warning] = fences(section(page, "## Turn grants on"), "text");
+ const { code, output } = cli(cwd, ["plan", "--no-diff"]);
+ expect(code).toBe(0);
+ expect(output).toContain(warning!);
+ });
+
+ it("quotes the error a text mask gives on a column that is not text", () => {
+ const cwd = copy("governed-databricks");
+ change(
+ cwd,
+ "contracts/customers.yaml",
+ "primaryKey: true, classification: internal",
+ "primaryKey: true, classification: confidential",
+ );
+ const [error] = fences(section(page, "## Mask sensitive columns"), "text");
+ const { code, output } = cli(cwd, ["plan", "--no-diff"]);
+ expect(code).toBe(1);
+ expect(output).toContain(error!);
+ });
+
+ it("quotes the warning a mask gives on Synapse", () => {
+ const cwd = copy("synapse-views");
+ change(
+ cwd,
+ "contracts/customers.yaml",
+ "{ name: name, logicalType: string }",
+ "{ name: name, logicalType: string, classification: confidential }",
+ );
+ writeFileSync(
+ join(cwd, "standard.yaml"),
+ `${readFileSync(join(cwd, "standard.yaml"), "utf8")}governance:\n masks: { confidential: hash }\n`,
+ );
+ const [warning] = fences(section(page, "## On Synapse"), "text");
+ const { code, output } = cli(cwd, ["plan", "--no-diff"]);
+ expect(code).toBe(0);
+ expect(output).toContain(warning!);
+ });
+
+ it("accepts the role DuckDB cannot grant, and the passport still says so", () => {
+ const { code, output } = cli(copy("data-products"), ["plan", "--no-diff"]);
+ expect(code).toBe(0);
+ expect(output).toContain("1 accepted warning (platform.yaml)");
+ expect(output).not.toContain("NOT_APPLIED");
+ expect(readExampleFile("data-products", "build/products/dev/customers.md")).toContain(
+ "Not enforced on DuckDB: DuckDB has no access control.",
+ );
+ });
+});
diff --git a/test/layers.test.ts b/test/layers.test.ts
index d5715dc..94f048d 100644
--- a/test/layers.test.ts
+++ b/test/layers.test.ts
@@ -95,9 +95,10 @@ describe("concepts/layers", () => {
by: "standard-rule",
when: { layer: "gold" },
});
- expect(table("region_sales").store?.settings.cluster_by).toBe("none");
+ const retention = { "delta.logRetentionDuration": "interval 90 days" };
+ expect(table("region_sales").store?.settings.properties).toEqual(retention);
expect(table("store_sales").store?.chosenBy.by).not.toBe("standard-rule");
- expect(table("store_sales").store?.settings.cluster_by).not.toBe("none");
+ expect(table("store_sales").store?.settings.properties).not.toEqual(retention);
});
it("replaces the list with a standard's own layers, so the default ones no longer exist", () => {
diff --git a/test/not-applied.test.ts b/test/not-applied.test.ts
new file mode 100644
index 0000000..7f12387
--- /dev/null
+++ b/test/not-applied.test.ts
@@ -0,0 +1,30 @@
+// "Not applied here": what it shows is what the CLI prints for the first
+// project, and the layers it gives take the warning away.
+import { readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { cli, copy, fences, read, section } from "./pages.js";
+
+const page = read("concepts/not-applied.mdx");
+
+describe("concepts/not-applied", () => {
+ it("shows the warning the first project's plan prints", () => {
+ const shown = fences(section(page, "## Where it shows [#where-it-shows]"), "text")[0]!;
+ const plan = cli(copy("duckdb"), ["plan", "--no-diff"]);
+ expect(plan.code).toBe(0);
+ expect(plan.output.startsWith(shown)).toBe(true);
+ });
+
+ it("takes the warning away with the layers it gives, and then counts the rule", () => {
+ const layers = fences(section(page, "## A layer that does not check [#layers]"), "yaml")[0]!;
+ const cwd = copy("duckdb");
+ const standard = join(cwd, "standard.yaml");
+ writeFileSync(standard, readFileSync(standard, "utf8") + layers);
+ const plan = cli(cwd, ["apply", "--no-diff"]);
+ expect(plan.code).toBe(0);
+ expect(plan.output).not.toContain("warning");
+ expect(readFileSync(join(cwd, "build/duckdb/dev/run.sql"), "utf8")).toContain(
+ "('bronze.orders', 'order_id_not_null', 'warn',",
+ );
+ });
+});
diff --git a/test/page-urls.txt b/test/page-urls.txt
index d035ee6..d44380a 100644
--- a/test/page-urls.txt
+++ b/test/page-urls.txt
@@ -3,6 +3,7 @@
/docs/concepts/environments-and-platform
/docs/concepts/generated-files
/docs/concepts/layers
+/docs/concepts/not-applied
/docs/concepts/settings
/docs/concepts/standards-and-packs
/docs/concepts/yaml-references
@@ -84,20 +85,28 @@
/docs/connectors/vertica
/docs/connectors/web-table
/docs/connectors/xero
+/docs/how-to/accept-a-warning
/docs/how-to/add-a-source
/docs/how-to/add-an-environment
/docs/how-to/add-quality-checks
/docs/how-to/ask-the-platform-team
/docs/how-to/check-in-ci
+/docs/how-to/choose-a-published-standard
+/docs/how-to/cleanse-data
/docs/how-to/configure-a-standard
/docs/how-to/define-a-data-product
/docs/how-to/deploy-through-environments
+/docs/how-to/grant-access
/docs/how-to/keep-a-table-current
+/docs/how-to/kits/databricks-lakehouse-streaming
+/docs/how-to/kits/duckdb-local-medallion
/docs/how-to/name-things-your-way
/docs/how-to/plan-and-generate
/docs/how-to/share-a-template
+/docs/how-to/start-from-a-published-standard
/docs/how-to/start-from-a-template
/docs/how-to/upgrade
+/docs/how-to/upgrade-a-standard
/docs/how-to/write-a-contract
/docs/how-to/write-a-transform
/docs/platforms/adf
@@ -112,6 +121,7 @@
/docs/reference/cli
/docs/reference/documents
/docs/reference/errors
+/docs/reference/intent
/docs/reference/packs
/docs/reference/packs/adf
/docs/reference/packs/databricks
diff --git a/test/platform-defaults.test.ts b/test/platform-defaults.test.ts
index e8fe798..5bf2641 100644
--- a/test/platform-defaults.test.ts
+++ b/test/platform-defaults.test.ts
@@ -24,19 +24,23 @@ function platform(folder: string, environment: string) {
describe("platform defaults", () => {
it("lists each field the Databricks pack reads, with its default, linked to its reference", () => {
const rows = platformDefaults("databricks");
+ // A field of the pack's own section is shown there, where it is written now.
expect(rows.map((row) => row.field.path.join("."))).toEqual([
- "mode",
- "compute.serverless",
+ "databricks.mode",
+ "databricks.compute.serverless",
"schedule.timezone",
"schedule.paused",
- "manage.schemas",
- "manage.tags",
- "manage.comments",
- "manage.checks",
+ "databricks.manage.schemas",
+ "databricks.manage.tags",
+ "databricks.manage.comments",
+ "databricks.manage.checks",
+ "databricks.manage.grants",
+ "databricks.manage.masks",
+ "databricks.manage.filters",
]);
const html = renderToStaticMarkup(createElement(PlatformDefaults, { pack: "databricks" }));
expect(html).toContain(
- 'mode',
+ 'databricks.mode',
);
expect(html).toContain(
"development in an environment named dev, and production in every other, test and default included",
@@ -92,11 +96,11 @@ describe("platform defaults", () => {
platformDefaults("databricks").map((row) => [row.field.path.join("."), row.default]),
);
expect(stated).toMatchObject({
- "compute.serverless": "true",
+ "databricks.compute.serverless": "true",
"schedule.timezone": "UTC",
"schedule.paused":
"running in an environment named prod or production, and where the platform sets mode: production itself; paused everywhere else, test included, though its mode is production by default",
- "manage.tags": "false",
+ "databricks.manage.tags": "false",
});
});
});
diff --git a/test/platform-pages.test.ts b/test/platform-pages.test.ts
index 6c203fe..8784c60 100644
--- a/test/platform-pages.test.ts
+++ b/test/platform-pages.test.ts
@@ -291,6 +291,6 @@ describe("reference/versions", () => {
expect(tools).toContain(`\`@microsoft/azure-data-factory-utilities\` ${adf}`);
expect(tools).toContain(`Databricks CLI ${databricks}`);
expect(tools).toContain(`sqlcmd (Go) ${sqlcmd}`);
- expect(tools).toContain("`ingestron@2.2.0`");
+ expect(tools).toContain("`ingestron@2.3.0`");
});
});
diff --git a/test/reader-journeys.test.ts b/test/reader-journeys.test.ts
index b651d9c..8bfc8b0 100644
--- a/test/reader-journeys.test.ts
+++ b/test/reader-journeys.test.ts
@@ -10,6 +10,7 @@ import {
readdirSync,
readFileSync,
rmSync,
+ writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -160,8 +161,8 @@ describe("connector pages", () => {
});
describe("how-to/add-quality-checks", () => {
- const table = read("how-to/add-quality-checks.mdx").split("\n## ")[0]!.split("\n\n")[2]!;
- const row = (link: string) => table.split("\n").find((line) => line.includes(`](${link})`))!;
+ const page = read("how-to/add-quality-checks.mdx");
+ const prose = (text: string) => text.replace(/\s+/g, " ");
/** Plans a copy of a sample after giving one table an expectation, the way the guide adds one. */
function plan(folder: string, flow: string, table: string): string {
@@ -174,34 +175,70 @@ describe("how-to/add-quality-checks", () => {
return cli(cwd, ["plan", "--no-diff"]).output;
}
- it("says what each engine does with a check, as the CLI reports it", () => {
- expect(row("/docs/platforms/databricks")).toContain("Runs them");
- const databricks = plan(exampleFolder("quality"), "flows/readings.yaml", "readings");
- expect(databricks).not.toContain("error");
+ it("lists first what Synapse cannot apply in the quality example, as the page shows", () => {
+ const engines = section(page, "## What each engine does [#engines]");
+ const planned = cli(copy("quality"), ["plan", "--no-diff"]);
+ expect(planned.code).toBe(0);
+ expect(planned.output.startsWith(fences(engines, "text")[0]!)).toBe(true);
+ expect(engines).toContain("");
+ });
+
+ it("says where DuckDB and Synapse check rows, as the CLI reports it", () => {
+ const layers = prose(section(page, "## Say which layers check [#layers]"));
+ expect(layers).toContain(
+ "DuckDB and Synapse run checks only in a layer whose `does` lists `check` or is `raw`",
+ );
+ expect(layers).toContain("Databricks checks them as it always has");
+ // Databricks checks the quality example's expectation; it is in a layer that checks.
+ expect(plan(exampleFolder("quality"), "flows/readings.yaml", "readings")).not.toContain(
+ "error",
+ );
+
+ // The DuckDB starter's layers say nothing of checking.
const duckdb = plan(exampleFolder("duckdb"), "flows/orders.yaml", "orders");
expect(duckdb).toContain("error METHOD_CHECK");
- expect(duckdb).toContain("DuckDB tables check no rows yet");
- expect(row("/docs/platforms/duckdb")).toContain("`METHOD_CHECK`");
+ expect(duckdb).toContain("DuckDB checks rows only in a layer whose does lists check");
+ expect(duckdb).toContain(
+ "The notNull rule isn't applied on duckdb: it is applied only in a layer whose does lists check.",
+ );
+ expect(layers).toContain("`METHOD_CHECK`");
- const synapse = plan(exampleFolder("synapse-views"), "flows/erp.yaml", "customers");
+ // The Synapse example's layers check; without their duties, its rules are not applied.
+ const checked = plan(exampleFolder("synapse-views"), "flows/erp.yaml", "customers");
+ expect(checked).not.toContain("error");
+ expect(checked).not.toContain("NOT_APPLIED");
+ const folder = copy("synapse-views");
+ const standard = join(folder, "standard.yaml");
+ const text = readFileSync(standard, "utf8");
+ const duties = /^layers:\n(?: {2}- .*\n(?: {4}.*\n)*)+/m.exec(text)?.[0];
+ expect(duties, "the example's layers").toBeDefined();
+ writeFileSync(standard, text.replace(duties!, ""));
+ const synapse = plan(folder, "flows/erp.yaml", "customers");
expect(synapse).not.toContain("error");
- expect(synapse).not.toContain("has_rows");
- expect(synapse).toContain("warning CONTRACT_FIELD_UNUSED");
- expect(synapse).toContain("Synapse views check no rows.");
- expect(row("/docs/platforms/synapse")).toContain("`expectations` are not run");
- expect(row("/docs/platforms/synapse")).toContain("`CONTRACT_FIELD_UNUSED`");
+ expect(synapse).toContain(
+ "The notNull rule isn't applied on synapse: it is applied only in a layer whose does lists check.",
+ );
+ expect(synapse).toContain("Add `check` to the layer's `does` to apply it here.");
+ });
+ it("says nothing checks a table that ends at the landed files", () => {
const landing = plan(exampleFolder("landing-only"), "flows/erp.yaml", "customers");
expect(landing).not.toContain("error");
- expect(landing).toContain("warning QUALITY_RULE_NOT_CHECKED");
- expect(row("/docs/platforms/landing-only")).toContain("`QUALITY_RULE_NOT_CHECKED`");
+ expect(landing).toContain(
+ 'Expectation "has_rows" isn\'t applied on adf: the table ends at the landed files, and nothing checks them.',
+ );
+ expect(prose(read("platforms/landing-only.mdx"))).toContain(
+ "each contract quality rule and flow expectation on these tables is",
+ );
});
- it("is linked from the DuckDB page, which says checks are an error there", () => {
+ it("is linked from the DuckDB page, which says where checks run there", () => {
const duckdb = read("platforms/duckdb.mdx");
- expect(duckdb).toContain("DuckDB tables check no rows yet");
- expect(duckdb).toContain("(/docs/how-to/add-quality-checks)");
+ expect(prose(duckdb)).toContain(
+ "DuckDB checks rows only in a layer whose `does` lists `check`",
+ );
+ expect(duckdb).toContain("(/docs/how-to/add-quality-checks#duckdb)");
});
});
@@ -310,7 +347,7 @@ describe("platforms/databricks", () => {
// Deploying as one identity: a shared root_path, and binding what another identity deployed.
const identity = section(deployPage, "### Deploy as one identity");
expect(identity).toContain(
- '',
+ '',
);
expect(fences(identity, "sh")[0]).toContain(
"databricks bundle deployment bind -t prod",
@@ -318,7 +355,7 @@ describe("platforms/databricks", () => {
const platform = parse(
readFileSync(join(exampleFolder("environments"), "platform.yaml"), "utf8"),
);
- expect(platform.workspace.root_path).toMatch(/^\/Workspace\/Shared\//);
+ expect(platform.databricks.workspace.root_path).toMatch(/^\/Workspace\/Shared\//);
expect(section(page, "## Deploy and run")).toContain("(/docs/platforms/databricks-deploy)");
const ci = section(deployPage, "### From CI");
expect(ci).toContain('file="github-actions/deploy-environment.yml"');
diff --git a/test/screenshot.test.ts b/test/screenshot.test.ts
index 2923c2c..25adec5 100644
--- a/test/screenshot.test.ts
+++ b/test/screenshot.test.ts
@@ -9,8 +9,8 @@ import { checkedOn, Screenshot } from "../components/screenshot";
import { overflows } from "../lib/keyboard";
import { contentFolder } from "../lib/content-files.js";
-const light = "/images/vscode/editor-light.png";
-const dark = "/images/vscode/editor-dark.png";
+const light = "/images/vscode/problems-light.png";
+const dark = "/images/vscode/problems-dark.png";
const platform = "/images/platforms/adf/adf-11-copy-details.png";
const render = (props: Parameters[0]) =>
renderToStaticMarkup(createElement(Screenshot, props));
diff --git a/test/site-journey.test.ts b/test/site-journey.test.ts
index 699e6ec..2647704 100644
--- a/test/site-journey.test.ts
+++ b/test/site-journey.test.ts
@@ -46,7 +46,9 @@ it("opens with one learning path, then areas by the reader's job, with no index
"write-a-transform",
"keep-a-table-current",
"add-quality-checks",
+ "cleanse-data",
"define-a-data-product",
+ "grant-access",
"add-an-environment",
"plan-and-generate",
"check-in-ci",
@@ -54,8 +56,13 @@ it("opens with one learning path, then areas by the reader's job, with no index
]);
expect(JSON.parse(read("how-to/(standards)/meta.json")).pages).toEqual([
"configure-a-standard",
+ "choose-a-published-standard",
+ "start-from-a-published-standard",
+ "upgrade-a-standard",
+ "kits",
"name-things-your-way",
"ask-the-platform-team",
+ "accept-a-warning",
"share-a-template",
"upgrade",
]);
@@ -64,6 +71,7 @@ it("opens with one learning path, then areas by the reader's job, with no index
"standards-and-packs",
"settings",
"layers",
+ "not-applied",
"environments-and-platform",
"yaml-references",
"generated-files",
@@ -88,6 +96,7 @@ it("opens with one learning path, then areas by the reader's job, with no index
]);
expect(JSON.parse(read("reference/meta.json")).pages).toEqual([
"documents",
+ "intent",
"packs",
"errors",
"cli",
diff --git a/test/sql-watermarks.test.ts b/test/sql-watermarks.test.ts
index 0f3fb9d..257e97a 100644
--- a/test/sql-watermarks.test.ts
+++ b/test/sql-watermarks.test.ts
@@ -42,15 +42,21 @@ it("describes the watermark example as the sample has it, on every adf-copy page
expect(item.ingest?.settings.mode).toBe("incremental");
expect(item.ingest?.settings.every).toEqual({ frequency: "Hour", interval: 1 });
}
- // "using `modified_at`. Customers inherit that default.": the standard sets it, the flow does not.
+ // "the standard copies changed rows once an hour, and the flow reads them by `modified_at`".
const standard = parse(readExampleFile(watermarkExample, "standard.yaml"));
- expect(standard.ingest.settings["adf-copy"].watermark_column).toBe("modified_at");
- expect(words).toContain("using `modified_at`. Customers inherit that default.");
- expect(customers.ingest?.settings.watermark_column).toBe("modified_at");
- // "Invoices override the watermark with `updated_at` in the flow and set the store's ordering column to match."
+ expect(standard.load.default).toBe("incremental");
const flow = parse(readExampleFile(watermarkExample, "flows/erp.yaml"));
- expect(flow.tables.customers.ingest).toBeUndefined();
- expect(flow.tables.invoices.ingest.with.watermark_column).toBe("updated_at");
- expect(flow.tables.invoices.store.with.sequence_by).toBe("updated_at");
- expect(words).toContain("Invoices override the watermark with `updated_at` in the flow");
+ expect(flow.load.cursor).toBe("modified_at");
+ expect(words).toContain(
+ "the flow reads them by `modified_at`. Customers take the flow's cursor.",
+ );
+ expect(customers.ingest?.settings.watermark_column).toBe("modified_at");
+ expect(flow.tables.customers.load).toBeUndefined();
+ // "Invoices give their own, `updated_at`, and order their changes by it too."
+ expect(flow.tables.invoices.load.cursor).toBe("updated_at");
+ expect(flow.tables.invoices.sequence).toBe("updated_at");
+ expect(invoices.store?.settings.sequence_by).toBe("updated_at");
+ expect(words).toContain(
+ "Invoices give their own, `updated_at`, and order their changes by it too.",
+ );
});
diff --git a/test/standards-and-teams.test.ts b/test/standards-and-teams.test.ts
index c6d7f03..0b8cb5b 100644
--- a/test/standards-and-teams.test.ts
+++ b/test/standards-and-teams.test.ts
@@ -324,7 +324,7 @@ describe("how-to/upgrade", () => {
it("pins the exact bundle these pages describe, as the templates install it", () => {
// The ingestron bundle these pages describe: the installed cli's version.
- const bundle = "2.2.0";
+ const bundle = "2.3.0";
const choose = section(page, "## Choose the version you run");
const version = JSON.parse(
readFileSync(join(process.cwd(), "node_modules/@ingestron/cli/package.json"), "utf8"),
diff --git a/test/standards-kits.test.ts b/test/standards-kits.test.ts
new file mode 100644
index 0000000..b632781
--- /dev/null
+++ b/test/standards-kits.test.ts
@@ -0,0 +1,333 @@
+// Published standards (kits), intent and the field reference's help: the pages
+// show what the packages and the installed kits hold, and what the CLI prints.
+import { execFileSync } from "node:child_process";
+import { appendFileSync, mkdtempSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createElement } from "react";
+import { renderToStaticMarkup } from "react-dom/server";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { documentReference, intentPaths, intentUseOf } from "@ingestron/definitions";
+import { FieldTable, PlatformSection } from "../components/reference/document-fields";
+import { KitFacts, KitIntent, KitList } from "../components/reference/kits";
+import {
+ IntentAliases,
+ IntentOverview,
+ IntentSupport,
+ QualityEngines,
+ ZonesNotApplied,
+ IntentVocabulary,
+} from "../components/reference/pack-intent";
+import { kitBlanks, kitById, kitPage, kits, latestKits } from "../lib/kits";
+import {
+ fieldPath,
+ intentRows,
+ intentVocabulary,
+ packs,
+ platformSectionFields,
+} from "../lib/reference";
+import { cli, fences, read, section } from "./pages.js";
+
+const html = (element: Parameters[0]) => renderToStaticMarkup(element);
+
+// No test reaches the network: the catalogue is a repository that is not there.
+let saved: string | undefined;
+beforeAll(() => {
+ saved = process.env.INGESTRON_STANDARDS_REPOSITORY;
+ process.env.INGESTRON_STANDARDS_REPOSITORY = `file://${join(tmpdir(), "no-such-catalogue.git")}`;
+});
+afterAll(() => {
+ if (saved === undefined) delete process.env.INGESTRON_STANDARDS_REPOSITORY;
+ else process.env.INGESTRON_STANDARDS_REPOSITORY = saved;
+});
+
+describe("the field reference's help", () => {
+ it("shows where to find a value and a suggestion, where a field has them", () => {
+ const field = platformSectionFields("databricks").find(
+ (item) => fieldPath(item) === "databricks.run_as.service_principal_name",
+ )!;
+ expect(field.find).toBeTruthy();
+ expect(field.suggest).toBe("");
+ const shown = html(createElement(FieldTable, { label: "x", fields: [field] }));
+ expect(shown).toContain("Where to find it: Settings, then Identity and access");
+ expect(shown).toContain("Suggested: <application id>");
+ // A document's field too: an alert's address.
+ const to = documentReference("Platform").find((item) => fieldPath(item) === "alerts.*.to.*")!;
+ expect(html(createElement(FieldTable, { label: "x", fields: [to] }))).toContain(
+ "Suggested: <team>@<domain>",
+ );
+ });
+
+ it("gives each field of the Databricks section its anchor on the pack's page", () => {
+ const page = read("reference/packs/databricks.mdx");
+ expect(page).toContain('');
+ const shown = html(createElement(PlatformSection, { pack: "databricks" }));
+ for (const field of platformSectionFields("databricks")) {
+ const [path, anchor] = field.docs.split("#");
+ expect(path).toBe("/docs/reference/packs/databricks");
+ expect(shown).toContain(`id="${anchor}"`);
+ }
+ });
+});
+
+describe("intent", () => {
+ it("has a support matrix on every pack's page, from the pack's own coverage", () => {
+ for (const pack of packs) {
+ expect(read(`reference/packs/${pack.id}.mdx`)).toContain(
+ `## Intent [#intent]\n\nWhat this pack does with each value of [intent](/docs/reference/intent)`,
+ );
+ expect(read(`reference/packs/${pack.id}.mdx`)).toContain(
+ ``,
+ );
+ // Every value of every path, each covered once.
+ const rows = intentRows(pack.id);
+ expect(rows.flatMap((row) => row.values.map((value) => `${row.path}=${value}`))).toEqual(
+ Object.entries(intentPaths).flatMap(([path, values]) =>
+ values.map((value) => `${path}=${value}`),
+ ),
+ );
+ }
+ const databricks = html(createElement(IntentSupport, { pack: "databricks" }));
+ // A preview feature shows its release level, linked to the page, and the date checked.
+ const slow = intentUseOf(packs[0]!, "operations.alerts", "slow");
+ expect(slow?.use === "used" && slow.status?.level).toBe("preview");
+ expect(databricks).toContain(">Public Preview");
+ expect(databricks).toContain(">Generally available");
+ expect(databricks).toContain("
2026-11-09
");
+ // Where a method differs from its pack.
+ expect(databricks).toContain("Managed ingestion reads the source, not files.");
+ });
+
+ it("has a quality and cleansing section on every pack's page, apart from the rest of intent", () => {
+ for (const pack of packs) {
+ const page = read(`reference/packs/${pack.id}.mdx`);
+ expect(page).toContain("## Quality and cleansing [#quality-and-cleansing]");
+ expect(page).toContain(``);
+ const quality = html(createElement(IntentSupport, { pack: pack.id, part: "quality" }));
+ const rest = html(createElement(IntentSupport, { pack: pack.id }));
+ expect(quality).toContain("quality.rules");
+ expect(quality).toContain("cleanse.timezone");
+ expect(quality).not.toContain("history");
+ expect(rest).toContain("history");
+ expect(rest).not.toContain("quality.rules");
+ }
+ // Where a pack applies a value only in some layers, and the zones it cannot convert.
+ const duckdb = html(createElement(IntentSupport, { pack: "duckdb", part: "quality" }));
+ expect(duckdb).toContain("Only in a layer whose does lists check");
+ expect(read("reference/packs/synapse.mdx")).toContain('');
+ const zones = html(createElement(ZonesNotApplied, { pack: "synapse" }));
+ const use = intentUseOf(
+ packs.find((pack) => pack.id === "synapse")!,
+ "cleanse.timezone",
+ "true",
+ );
+ const listed = Object.keys((use?.use === "used" && use.zonesNotApplied) || {});
+ expect(listed).toContain("Antarctica/Troll");
+ for (const zone of listed) expect(zones).toContain(`${zone}`);
+ expect(() => html(createElement(ZonesNotApplied, { pack: "duckdb" }))).toThrow();
+ // The engine table: what each engine does, or why it does not.
+ const engines = html(createElement(QualityEngines));
+ expect(engines).toContain("Not applied: a view can't stop a load");
+ expect(engines).toContain("ON VIOLATION DROP ROW");
+ });
+
+ it("has an access section on every pack's page, apart from the rest of intent", () => {
+ for (const pack of packs) {
+ const page = read(`reference/packs/${pack.id}.mdx`);
+ expect(page).toContain("## Access [#access]");
+ expect(page).toContain(``);
+ const access = html(createElement(IntentSupport, { pack: pack.id, part: "access" }));
+ const rest = html(createElement(IntentSupport, { pack: pack.id }));
+ const quality = html(createElement(IntentSupport, { pack: pack.id, part: "quality" }));
+ for (const path of ["access.grants", "access.exact", "access.masks", "access.filters"]) {
+ expect(access).toContain(`${path}`);
+ expect(rest).not.toContain(`${path}`);
+ expect(quality).not.toContain(`${path}`);
+ }
+ expect(access).not.toContain("history");
+ }
+ // What each pack does, or why not, from its own coverage.
+ const databricks = html(createElement(IntentSupport, { pack: "databricks", part: "access" }));
+ expect(databricks).toContain("GRANT SELECT, MODIFY ON TABLE, after each refresh.");
+ const synapse = html(createElement(IntentSupport, { pack: "synapse", part: "access" }));
+ expect(synapse).toContain("Synapse serverless has no data masking");
+ const duckdb = html(createElement(IntentSupport, { pack: "duckdb", part: "access" }));
+ expect(duckdb).toContain("DuckDB has no access control.");
+ });
+
+ it("describes every intent path, and shows every pack's use of each value", () => {
+ const vocabulary = intentVocabulary();
+ expect(vocabulary.map((row) => row.path)).toEqual(Object.keys(intentPaths));
+ const history = vocabulary.find((row) => row.path === "history")!;
+ expect(history.places).toEqual(["layer", "flow", "flow table", "transform"]);
+ expect(vocabulary.find((row) => row.path === "load.cursor")!.places).toEqual([
+ "flow",
+ "flow table",
+ ]);
+ // The checks a contract gives, and the cleansing each table writes for its columns.
+ const placesOf = (path: string) => vocabulary.find((row) => row.path === path)!.places;
+ expect(placesOf("quality.rules")).toEqual(["contract"]);
+ expect(placesOf("quality.table-checks")).toEqual(["contract"]);
+ expect(placesOf("quality.actions")).toEqual(["standard", "layer", "flow table", "transform"]);
+ expect(placesOf("quality.quarantine")).toEqual([
+ "standard",
+ "layer",
+ "flow table",
+ "transform",
+ ]);
+ expect(placesOf("cleanse.trim")).toEqual(["standard", "flow table", "transform"]);
+ expect(placesOf("cleanse.map")).toEqual(["flow table", "transform"]);
+ expect(placesOf("deduplicate")).toEqual(["flow table", "transform"]);
+ const page = read("reference/intent.mdx");
+ for (const component of ["", "", ""])
+ expect(page).toContain(component);
+ const described = html(createElement(IntentVocabulary));
+ expect(described).toContain("Only the latest row per key.");
+ expect(described).toContain("Matches a regular expression");
+ const overview = html(createElement(IntentOverview));
+ for (const pack of packs) expect(overview).toContain(`/docs/reference/packs/${pack.id}#intent`);
+ expect(html(createElement(IntentAliases))).toContain(
+ "mode: upsert is history: latest",
+ );
+ });
+});
+
+describe("published standards", () => {
+ it("has a page for each installed kit, titled as the kit, with its current version", () => {
+ expect(latestKits().map((kit) => kit.id)).toEqual([
+ "databricks/lakehouse-streaming",
+ "duckdb/local-medallion",
+ ]);
+ const list = html(createElement(KitList));
+ for (const kit of latestKits()) {
+ const path = `${kitPage(kit.id).replace("/docs/", "")}.mdx`;
+ const page = read(path);
+ expect(page).toContain(`title: ${kit.title}\n`);
+ expect(page).toContain(``);
+ expect(fences(page, "sh")).toEqual([`ingestron init shop --kit ${kit.id}@${kit.version}\n`]);
+ expect(list).toContain(`href="${kitPage(kit.id)}"`);
+ expect(html(createElement(KitFacts, { id: kit.id }))).toContain(kit.summary);
+ }
+ expect(kits().every((kit) => kit.features === "ga")).toBe(true);
+ });
+
+ it("explains every REPLACE_ME of each kit with the field reference the hover uses", () => {
+ for (const kit of latestKits())
+ for (const file of ["platform.yaml", "project.yaml"]) {
+ for (const blank of kitBlanks(kit, file)) {
+ expect(blank.help, `${kit.id} ${file} ${blank.pointer}`).toBeDefined();
+ expect(blank.help!.find, `${kit.id} ${file} ${blank.pointer}`).toBeTruthy();
+ expect(blank.help!.suggest, `${kit.id} ${file} ${blank.pointer}`).toBeTruthy();
+ }
+ }
+ const kit = kitById("databricks/lakehouse-streaming");
+ expect(kitBlanks(kit, "platform.yaml").map((blank) => [blank.line, blank.help!.field])).toEqual(
+ [
+ [12, "to"],
+ [15, "principals"],
+ [21, "service_principal_name"],
+ [34, "functions"],
+ ],
+ );
+ // Each permission group is a project.yaml value of its own, explained by the field that reads it.
+ expect(
+ kitBlanks(kit, "project.yaml")
+ .filter((blank) => blank.help!.field === "group_name")
+ .map((blank) => blank.line),
+ ).toEqual([5, 7]);
+ // Each kit page shows the files that hold them.
+ const page = read(`${kitPage(kit.id).replace("/docs/", "")}.mdx`);
+ for (const file of ["platform.yaml", "project.yaml"])
+ expect(page).toContain(``);
+ });
+
+ it("shows where each table's intent came from", () => {
+ const shown = html(createElement(KitIntent, { id: "databricks/lakehouse-streaming" }));
+ expect(shown).toContain("silver.orders");
+ expect(shown).toContain("history: latest");
+ expect(shown).toContain("the table's layer in the standard (standard.yaml:");
+ expect(shown).toContain("this table (flows/orders.yaml:");
+ expect(shown).toContain("the pack's default");
+ });
+
+ it("prints what Start from a published standard shows", () => {
+ const page = read("how-to/start-from-a-published-standard.mdx");
+ const create = section(page, "## Create the project");
+ const [command] = fences(create, "sh");
+ expect(command).toBe("ingestron init shop --kit duckdb/local-medallion@2026-11-10\n");
+ const cwd = mkdtempSync(join(tmpdir(), "docs-kit-"));
+ const { code, output } = cli(cwd, command!.trim().split(" ").slice(1));
+ expect(code).toBe(0);
+ expect(output).toBe(fences(create, "text")[0]);
+ // The name alone, with the catalogue out of reach: the newest installed version, and a note.
+ const alone = cli(cwd, ["init", "alone", "--kit", "duckdb/local-medallion"]);
+ expect(alone.code).toBe(0);
+ const [note] = alone.output.split("\n");
+ expect(note).toBe(
+ "Could not reach the catalogue; using the newest installed version, duckdb/local-medallion@2026-11-10.",
+ );
+ expect(create).toContain(`\`${note}\``);
+ const connection = section(page, "## With a connection [#with-a-connection]");
+ const [warning] = fences(connection, "text");
+ const sales = cli(cwd, [
+ "init",
+ "sales",
+ "--kit",
+ "databricks/lakehouse-streaming@2026-11-10",
+ "--connector",
+ "sqlserver",
+ "--connectivity",
+ "public",
+ "--via",
+ "databricks",
+ "--yes",
+ ]);
+ expect(sales.code).toBe(0);
+ expect(sales.output).toContain(warning!.trimEnd());
+ });
+
+ it("prints what Choose a published standard and Move to a newer one show", () => {
+ const cwd = mkdtempSync(join(tmpdir(), "docs-kit-"));
+ expect(cli(cwd, ["init", "p", "--kit", "duckdb/local-medallion@2026-11-10"]).code).toBe(0);
+ const project = join(cwd, "p");
+ const git = (...args: string[]) =>
+ execFileSync("git", args, {
+ cwd: project,
+ env: {
+ ...process.env,
+ GIT_AUTHOR_NAME: "t",
+ GIT_AUTHOR_EMAIL: "t@example.com",
+ GIT_COMMITTER_NAME: "t",
+ GIT_COMMITTER_EMAIL: "t@example.com",
+ },
+ });
+ git("init", "-q");
+ git("add", "-A");
+ git("commit", "-qm", "init");
+
+ const choose = read("how-to/choose-a-published-standard.mdx");
+ const listed = cli(project, ["standard", "list"]);
+ expect(listed.code).toBe(0);
+ const reached = "Could not reach the catalogue; showing the kits installed with ingestron.\n";
+ expect(listed.output.startsWith(reached)).toBe(true);
+ expect(choose).toContain(`\`${reached.trim()}\``);
+ expect(listed.output.slice(reached.length)).toBe(
+ fences(section(choose, "## List them"), "text")[0],
+ );
+
+ const upgrade = read("how-to/upgrade-a-standard.mdx");
+ const [refused, printed] = fences(upgrade, "text");
+ const install = cli(project, ["standard", "install", "duckdb/local-medallion@2026-11-10"]);
+ expect(install.code).toBe(0);
+ expect(install.output).toBe(printed);
+ // The new file is what the project had: the same version.
+ expect(git("status", "--porcelain").toString()).toBe("");
+ // A standard with changes that are not committed is refused.
+ const standard = join(project, "standard.yaml");
+ appendFileSync(standard, "# a change\n");
+ const stopped = cli(project, ["standard", "install", "duckdb/local-medallion@2026-11-10"]);
+ expect(stopped.code).toBe(1);
+ expect(stopped.output).toBe(refused);
+ expect(readFileSync(standard, "utf8").endsWith("# a change\n")).toBe(true);
+ });
+});
diff --git a/test/troubleshooting.test.ts b/test/troubleshooting.test.ts
index c1c3dd9..0d158f7 100644
--- a/test/troubleshooting.test.ts
+++ b/test/troubleshooting.test.ts
@@ -28,6 +28,18 @@ const entry = (heading: string) => section(page, `### ${heading}`);
/** The trimmed lines of a command's output. */
const lines = (output: string) => output.split("\n").map((line) => line.trim());
+/**
+ * What every plan of the DuckDB starter prints first: its contract's required
+ * order_id is not checked, as the starter's layers say nothing of checking.
+ */
+const starterNotApplied = [
+ "Not applied on this platform",
+ "dev:",
+ "contracts/orders.yaml:11:59 warning NOT_APPLIED",
+ "The notNull rule isn't applied on duckdb: it is applied only in a layer whose does lists check.",
+ "Add `check` to the layer's `does` to apply it here.",
+];
+
/** Every line of the entry's `text` blocks is a line of the output, unless it is listed as a placeholder. */
function quotes(heading: string, output: string, placeholders: string[] = []) {
const shown = fences(entry(heading), "text").flatMap((block) =>
@@ -133,6 +145,7 @@ describe("generated files", () => {
rmSync(leftover);
const plan = cli(project, ["plan", "--no-diff"]);
expect(lines(plan.output)).toEqual([
+ ...starterNotApplied,
"duckdb → build/duckdb (dev)",
"update dev/tables/silver.customer_totals.sql",
"",
@@ -180,7 +193,10 @@ describe("generated files", () => {
const project = copy("private-sql-server-filled");
cli(project, ["apply"]);
appendFileSync(join(project, "build/databricks/databricks.yml"), "# my change\n");
- appendFileSync(join(project, "standard.yaml"), "layers: [raw, curated]\n");
+ appendFileSync(
+ join(project, "standard.yaml"),
+ "store: { settings: { streaming-table: { properties: { team: erp } } } }\n",
+ );
const { code, output } = cli(project, ["apply", "--no-diff"]);
expect(code).toBe(1);
expect(lines(output)).toContain("Applied: adf. Not applied: databricks.");
@@ -224,6 +240,12 @@ describe("generated files", () => {
});
describe("definitions", () => {
+ it("quotes the first project's warning under Not applied on this platform", () => {
+ const plan = cli(copy("duckdb"), ["plan", "--no-diff"]);
+ expect(plan.code).toBe(0);
+ quotes("Plan lists warnings under `Not applied on this platform` [#not-applied]", plan.output);
+ }, 30_000);
+
it("quotes the setting messages", () => {
const project = copy("duckdb");
edit(project, "flows/orders.yaml", (text) =>
@@ -389,13 +411,18 @@ describe("environment variables and .env", () => {
expect(cli(project, ["plan"]).output).toContain("ENVVAR_MISSING");
writeFileSync(join(project, ".env.dev"), "SAMPLES_DIR=./samples\n");
expect(cli(project, ["plan"]).code).toBe(0);
- expect(ProjectSnapshot.open(project).problems("dev")).toEqual([]);
+ // Only the starter's own warning: its required order_id is not checked.
+ const codes = () =>
+ ProjectSnapshot.open(project)
+ .problems("dev")
+ .map((found) => found.code);
+ expect(codes()).toEqual(["NOT_APPLIED"]);
// A variable in the editor's own environment is seen, one exported later is not.
rmSync(join(project, ".env.dev"));
rmSync(join(project, ".env"));
process.env.SAMPLES_DIR = "./samples";
try {
- expect(ProjectSnapshot.open(project).problems("dev")).toEqual([]);
+ expect(codes()).toEqual(["NOT_APPLIED"]);
} finally {
delete process.env.SAMPLES_DIR;
}
@@ -542,6 +569,26 @@ describe("Synapse serverless views", () => {
describe("DuckDB", () => {
const noise: RegExp[] = [];
+ it("quotes the messages a failing check stops the run with, as run.sql writes them", () => {
+ const project = copy("quality");
+ expect(cli(project, ["apply", "--no-diff"]).code).toBe(0);
+ const sql = readFileSync(join(project, "build/duckdb/duckdb/run.sql"), "utf8");
+ const text = entry("A check stops the run [#a-check-stops-the-run]");
+ const [row] = fences(text, "text");
+ // DuckDB names the error; the rest is the message run.sql formats.
+ const message = row!.trim().replace(/^Invalid Input Error: /, "");
+ expect(sql).toContain(
+ `format('${message.replace('""', '"{}"').replace("", "{}").replace("