From ff1c1e93a72267352771ea0ac86eed0a9f9aba44 Mon Sep 17 00:00:00 2001
From: The Architect
Date: Tue, 10 Nov 2026 00:14:02 +0000
Subject: [PATCH 01/17] Standards: kits, upgrading, the intent reference and
pack support
Signed-off-by: The Architect
---
CONTRIBUTING.md | 3 +-
app/blocks.css | 52 +++
app/content.css | 6 +
ci-templates/azure-devops/azure-pipelines.yml | 2 +-
ci-templates/azure-devops/ingestron-check.yml | 2 +-
ci-templates/github-actions/deploy.yml | 2 +-
.../github-actions/ingestron-check.yml | 2 +-
components/mdx.tsx | 22 +-
components/reference/connector-settings.tsx | 6 +
components/reference/document-fields.tsx | 13 +-
components/reference/field-rows.tsx | 24 +-
components/reference/kits.tsx | 283 ++++++++++++++++
components/reference/pack-intent.tsx | 302 ++++++++++++++++++
components/reference/pack-methods.tsx | 58 ++--
components/reference/versions.tsx | 4 +
components/reference/vscode-extension.json | 2 +-
components/replace-me.tsx | 135 ++++++++
content/docs/(reference)/reference/cli.mdx | 26 +-
.../docs/(reference)/reference/documents.mdx | 17 +-
content/docs/(reference)/reference/intent.mdx | 167 ++++++++++
content/docs/(reference)/reference/meta.json | 2 +-
.../docs/(reference)/reference/packs/adf.mdx | 10 +
.../reference/packs/databricks.mdx | 38 ++-
.../(reference)/reference/packs/duckdb.mdx | 10 +
.../(reference)/reference/packs/synapse.mdx | 10 +
.../docs/(reference)/reference/templates.mdx | 6 +-
.../docs/(reference)/reference/versions.mdx | 2 +-
content/docs/(reference)/tools/vscode.mdx | 27 +-
content/docs/(reference)/troubleshooting.mdx | 21 +-
.../concepts/environments-and-platform.mdx | 10 +-
content/docs/concepts/layers.mdx | 6 +-
content/docs/concepts/settings.mdx | 15 +
content/docs/concepts/standards-and-packs.mdx | 27 +-
.../how-to/(build)/add-an-environment.mdx | 9 +-
.../how-to/(build)/add-quality-checks.mdx | 5 +
content/docs/how-to/(build)/check-in-ci.mdx | 2 +-
.../(build)/deploy-through-environments.mdx | 2 +-
.../how-to/(build)/keep-a-table-current.mdx | 73 +++--
.../how-to/(build)/start-from-a-template.mdx | 13 +-
.../docs/how-to/(build)/write-a-contract.mdx | 4 +-
.../(standards)/ask-the-platform-team.mdx | 5 +-
.../choose-a-published-standard.mdx | 91 ++++++
.../(standards)/configure-a-standard.mdx | 16 +-
.../kits/databricks-lakehouse-streaming.mdx | 62 ++++
.../kits/duckdb-local-medallion.mdx | 49 +++
.../docs/how-to/(standards)/kits/meta.json | 4 +
content/docs/how-to/(standards)/meta.json | 6 +-
.../how-to/(standards)/share-a-template.mdx | 55 +++-
.../start-from-a-published-standard.mdx | 137 ++++++++
.../how-to/(standards)/upgrade-a-standard.mdx | 110 +++++++
content/docs/how-to/(standards)/upgrade.mdx | 69 +++-
content/docs/index.mdx | 10 +-
content/docs/platforms/adf-pipelines.mdx | 10 +
content/docs/platforms/adf.mdx | 14 +-
content/docs/platforms/databricks-deploy.mdx | 8 +-
content/docs/platforms/databricks.mdx | 18 +-
content/docs/start/how-it-works.mdx | 13 +-
examples/change-data/flows/crm.yaml | 18 +-
examples/environments/platform.yaml | 7 +-
examples/layers-and-transforms/README.md | 3 +-
examples/layers-and-transforms/standard.yaml | 4 +-
examples/northwind/platform.yaml | 5 +-
examples/platform-setup/platform.yaml | 33 +-
.../private-sql-server-filled/flows/erp.yaml | 10 +-
.../private-sql-server-filled/standard.yaml | 13 +-
examples/retail-filled/flows/shop.yaml | 5 +-
examples/retail-filled/platform.yaml | 23 +-
lib/areas.ts | 2 +-
lib/connector-page.ts | 18 +-
lib/kits.ts | 155 +++++++++
lib/reference.ts | 197 +++++++++++-
lib/search.ts | 42 ++-
package.json | 17 +-
scripts/ci-templates.test.mjs | 2 +-
test/areas.test.ts | 8 +-
test/build-a-project.test.ts | 48 ++-
test/layers.test.ts | 5 +-
test/page-urls.txt | 6 +
test/platform-defaults.test.ts | 19 +-
test/platform-pages.test.ts | 2 +-
test/reader-journeys.test.ts | 4 +-
test/site-journey.test.ts | 5 +
test/sql-watermarks.test.ts | 24 +-
test/standards-and-teams.test.ts | 2 +-
test/standards-kits.test.ts | 244 ++++++++++++++
test/troubleshooting.test.ts | 7 +-
86 files changed, 2734 insertions(+), 291 deletions(-)
create mode 100644 components/reference/kits.tsx
create mode 100644 components/reference/pack-intent.tsx
create mode 100644 components/replace-me.tsx
create mode 100644 content/docs/(reference)/reference/intent.mdx
create mode 100644 content/docs/how-to/(standards)/choose-a-published-standard.mdx
create mode 100644 content/docs/how-to/(standards)/kits/databricks-lakehouse-streaming.mdx
create mode 100644 content/docs/how-to/(standards)/kits/duckdb-local-medallion.mdx
create mode 100644 content/docs/how-to/(standards)/kits/meta.json
create mode 100644 content/docs/how-to/(standards)/start-from-a-published-standard.mdx
create mode 100644 content/docs/how-to/(standards)/upgrade-a-standard.mdx
create mode 100644 lib/kits.ts
create mode 100644 test/standards-kits.test.ts
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 3a8212e..f3cfb9a 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -15,7 +15,8 @@ example README, package README and user-facing command text.
`from` and `to`; never paste sample YAML into a page. `` shows
part of a file in `ci-templates/` the same way. What a sample uses comes from resolving it; never hand-write it.
- Facts the code knows (settings, defaults, connector facts, error codes,
- document fields, CLI commands) are rendered by components in
+ document fields, CLI commands, what each pack does with intent, and the
+ published standards' files) are rendered by components in
`components/reference/`. Change the code, not the page.
- Every page belongs to one of seven areas, each a folder whose `meta.json`
says `"root": true`. A folder in brackets, such as `how-to/(build)/`,
diff --git a/app/blocks.css b/app/blocks.css
index 315bcd1..8fca0a4 100644
--- a/app/blocks.css
+++ b/app/blocks.css
@@ -205,3 +205,55 @@ figure[data-kind="run"] .line.cmd::before {
color: var(--brand-signal);
user-select: none;
}
+
+/* A value still REPLACE_ME in a kit's file (components/replace-me.tsx): a
+ dashed underline and its note's number mark it, not colour alone, with a
+ light tint. Its tooltip sits above the page; touch screens, which cannot
+ hover, show the notes below the block instead. */
+.replace-me {
+ text-decoration: underline dashed 1px;
+ text-underline-offset: 3px;
+ background: color-mix(in srgb, var(--docs-link) 12%, transparent);
+ border-radius: 2px;
+ cursor: help;
+}
+.replace-me::after {
+ content: attr(data-note);
+ margin-left: 0.15em;
+ font-size: 0.7em;
+ vertical-align: super;
+ line-height: 0;
+}
+.replace-me:focus-visible {
+ outline: 2px solid var(--docs-link);
+ outline-offset: 2px;
+}
+.replace-me-tip {
+ position: fixed;
+ z-index: 60;
+ max-width: min(24rem, calc(100vw - 16px));
+ padding: 0.6rem 0.75rem;
+ border: 1px solid var(--color-fd-border);
+ border-radius: 0.5rem;
+ background: var(--color-fd-popover);
+ color: var(--color-fd-popover-foreground);
+ box-shadow: 0 6px 20px rgb(0 0 0 / 0.18);
+ font-size: 0.85rem;
+ line-height: 1.5;
+}
+.replace-me-line {
+ display: block;
+ margin-top: 0.3rem;
+}
+.prose .replace-me-notes {
+ display: none;
+ font-size: 0.9rem;
+}
+.replace-me-where {
+ color: var(--color-fd-muted-foreground);
+}
+@media (hover: none), print {
+ .prose .replace-me-notes {
+ display: block;
+ }
+}
diff --git a/app/content.css b/app/content.css
index c6d422f..81f7de8 100644
--- a/app/content.css
+++ b/app/content.css
@@ -219,3 +219,9 @@ pre code > .line {
height: 0;
scroll-margin-top: 7rem;
}
+
+/* Where to find a field's value, and a suggestion (components/reference/field-rows.tsx): each on a line of its own. */
+.field-help {
+ display: block;
+ margin-top: 0.3rem;
+}
diff --git a/ci-templates/azure-devops/azure-pipelines.yml b/ci-templates/azure-devops/azure-pipelines.yml
index 10eed32..358be6f 100644
--- a/ci-templates/azure-devops/azure-pipelines.yml
+++ b/ci-templates/azure-devops/azure-pipelines.yml
@@ -33,7 +33,7 @@ stages:
version: 22.x
# An exact version: change it when you upgrade, as
# https://docs.ingestron.io/docs/how-to/upgrade#choose-the-version-you-run says.
- - script: npm install -g ingestron@2.2.0
+ - script: npm install -g ingestron@2.3.0
displayName: Install Ingestron
- script: ingestron check
displayName: ingestron check
diff --git a/ci-templates/azure-devops/ingestron-check.yml b/ci-templates/azure-devops/ingestron-check.yml
index a32ac45..0df120e 100644
--- a/ci-templates/azure-devops/ingestron-check.yml
+++ b/ci-templates/azure-devops/ingestron-check.yml
@@ -15,7 +15,7 @@ steps:
version: 22.x
# An exact version: change it when you upgrade, as
# https://docs.ingestron.io/docs/how-to/upgrade#choose-the-version-you-run says.
- - script: npm install -g ingestron@2.2.0
+ - script: npm install -g ingestron@2.3.0
displayName: Install Ingestron
- script: ingestron check
displayName: ingestron check
diff --git a/ci-templates/github-actions/deploy.yml b/ci-templates/github-actions/deploy.yml
index fec6a14..922bf20 100644
--- a/ci-templates/github-actions/deploy.yml
+++ b/ci-templates/github-actions/deploy.yml
@@ -29,7 +29,7 @@ jobs:
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: 22
- - run: npm install -g ingestron@2.2.0
+ - run: npm install -g ingestron@2.3.0
- run: ingestron check
build:
diff --git a/ci-templates/github-actions/ingestron-check.yml b/ci-templates/github-actions/ingestron-check.yml
index d01ca94..ea7efe5 100644
--- a/ci-templates/github-actions/ingestron-check.yml
+++ b/ci-templates/github-actions/ingestron-check.yml
@@ -18,5 +18,5 @@ jobs:
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: 22
- - run: npm install -g ingestron@2.2.0
+ - run: npm install -g ingestron@2.3.0
- run: ingestron check
diff --git a/components/mdx.tsx b/components/mdx.tsx
index 57f66be..8a42d8c 100644
--- a/components/mdx.tsx
+++ b/components/mdx.tsx
@@ -23,7 +23,18 @@ import { ContractUseTable } from "@/components/reference/contract-use";
import { PackCapabilities } from "@/components/reference/pack-capabilities";
import { DefaultLayers, PackNaming } from "@/components/reference/pack-defaults";
import { SamplesCatalogue } from "@/components/reference/samples-catalogue";
-import { ContractFields, DocumentFields } from "@/components/reference/document-fields";
+import {
+ ContractFields,
+ DocumentFields,
+ PlatformSection,
+} from "@/components/reference/document-fields";
+import {
+ IntentAliases,
+ IntentOverview,
+ IntentSupport,
+ IntentVocabulary,
+} from "@/components/reference/pack-intent";
+import { KitFacts, KitFile, KitIntent, KitList } from "@/components/reference/kits";
import { ErrorCodes, LoadingCodes } from "@/components/reference/error-codes";
import { ExtensionCommands } from "@/components/reference/extension-commands";
import { ExamplesList } from "@/components/reference/examples-list";
@@ -82,6 +93,15 @@ export function getMDXComponents(components?: MDXComponents) {
LoadingCodes,
DocumentFields,
ContractFields,
+ PlatformSection,
+ IntentSupport,
+ IntentOverview,
+ IntentVocabulary,
+ IntentAliases,
+ KitList,
+ KitFacts,
+ KitFile,
+ KitIntent,
CliCommands,
BuiltInTemplates,
ProjectSources,
diff --git a/components/reference/connector-settings.tsx b/components/reference/connector-settings.tsx
index 5910fed..21923d6 100644
--- a/components/reference/connector-settings.tsx
+++ b/components/reference/connector-settings.tsx
@@ -3,6 +3,7 @@ import type { Connector } from "@ingestron/definitions";
import { parse } from "yaml";
import { readExampleFile } from "@/lib/examples";
import { connectorEntries, shown } from "@/lib/reference";
+import { FindAndSuggest } from "./field-rows";
import { Inline } from "./inline";
interface Props {
@@ -84,6 +85,11 @@ function ConnectionField({
+ {(setting.find || setting.suggest) && (
+
+
+
+ )}
{example !== undefined && (
In the example: {shown(example)}.
diff --git a/components/reference/document-fields.tsx b/components/reference/document-fields.tsx
index 7967615..49582b7 100644
--- a/components/reference/document-fields.tsx
+++ b/components/reference/document-fields.tsx
@@ -1,6 +1,6 @@
// A document kind's fields at every depth, from its JSON Schema.
import { contractReads, type FieldInfo } from "@ingestron/definitions";
-import { documentFields } from "@/lib/reference";
+import { documentFields, packName, platformSectionFields } from "@/lib/reference";
import { Meaning, Name } from "./field-rows";
import { TableFrame } from "@/components/table-frame";
@@ -29,7 +29,16 @@ export function ContractFields() {
);
}
-function FieldTable({ label, fields }: { label: string; fields: FieldInfo[] }) {
+/** The fields of a pack's own section of the Platform, written under the pack's id.
+ * @example
+ */
+export function PlatformSection({ pack }: { pack: string }) {
+ const fields = platformSectionFields(pack);
+ if (fields.length === 0) throw new Error(`The ${pack} pack's Platform section has no fields.`);
+ return ;
+}
+
+export function FieldTable({ label, fields }: { label: string; fields: FieldInfo[] }) {
return (
diff --git a/components/reference/field-rows.tsx b/components/reference/field-rows.tsx
index dd82da3..51b7aab 100644
--- a/components/reference/field-rows.tsx
+++ b/components/reference/field-rows.tsx
@@ -1,10 +1,11 @@
// Rows of described fields: what each is, its values with their meanings, its
-// default and an example. Each row has the anchor the editor links to.
+// default, an example, and where to find a value with a suggestion, as the
+// editor's hover says them. Each row has the anchor the editor links to.
import type { FieldInfo } from "@ingestron/definitions";
import { anchorId, fieldPath, shown } from "@/lib/reference";
import { Inline } from "./inline";
-/** The meaning column: description, allowed values with meanings, example. */
+/** The meaning column: description, allowed values with meanings, example, where to find it and a suggestion. */
export function Meaning({ field }: { field: FieldInfo }) {
const described = field.values ?? [];
return (
@@ -36,6 +37,25 @@ export function Meaning({ field }: { field: FieldInfo }) {
For example: {shown(field.example)}.
>
)}
+
+ >
+ );
+}
+
+/** Where the owner finds a value, and what it could look like, each on a line of its own. */
+export function FindAndSuggest({ find, suggest }: { find?: string; suggest?: string }) {
+ return (
+ <>
+ {find && (
+
+ Where to find it:
+
+ )}
+ {suggest && (
+
+ Suggested: {suggest}
+
+ )}
>
);
}
diff --git a/components/reference/kits.tsx b/components/reference/kits.tsx
new file mode 100644
index 0000000..22c4745
--- /dev/null
+++ b/components/reference/kits.tsx
@@ -0,0 +1,283 @@
+// Published standards (kits), from the installed @ingestron/standards
+// package: the list to choose from, one kit's facts, a file of it with each
+// REPLACE_ME explained, and how each of its tables resolves.
+import type { ReactNode } from "react";
+import { ServerCodeBlock, type ServerCodeBlockProps } from "fumadocs-ui/components/codeblock.rsc";
+import { KindBlock } from "@/components/code-block";
+import { ReplaceMe, TipText } from "@/components/replace-me";
+import { TableFrame } from "@/components/table-frame";
+import { codeThemes } from "@/lib/code-themes";
+import {
+ kitBlanks,
+ kitById,
+ kitFile,
+ kitIntent,
+ kitPage,
+ latestKits,
+ type Blank,
+} from "@/lib/kits";
+import { packName } from "@/lib/reference";
+import { Inline } from "./inline";
+
+const levels = { ga: "Generally available features only", preview: "Uses preview features" };
+
+/** The pack ids of a kit's `packs`, such as `databricks` for `databricks@2`. */
+const packIds = (packs: string[]) => packs.map((pack) => pack.split("@")[0]!);
+
+/**
+ * Every published standard installed with Ingestron, newest version of each,
+ * linked to its page.
+ * @example
+ */
+export function KitList() {
+ return (
+
+
+
+
+ | Standard |
+ Platform |
+ What it builds |
+ Start with |
+
+
+
+ {latestKits().map((kit) => (
+
+ |
+ {kit.title}
+ |
+ {packIds(kit.packs).map(packName).join(", ")} |
+ {kit.summary} |
+
+
+ --kit {kit.id}@{kit.version}
+
+ |
+
+ ))}
+
+
+
+ );
+}
+
+/**
+ * One kit's facts, from its kit.yaml: what it builds, its version, packs,
+ * release level and the Ingestron it needs.
+ * @example
+ */
+export function KitFacts({ id }: { id: string }) {
+ const kit = kitById(id);
+ return (
+
+ - Builds
+ - {kit.summary}
+ - Name
+ -
+
{kit.id}
+
+ - Version
+ -
+
{kit.version}
+ {kit.notes && (
+ <>
+ :
+ >
+ )}
+
+ - Packs
+ -
+ {kit.packs.map((pack, index) => (
+
+ {index > 0 && ", "}
+
+
{pack}
+
+
+ ))}
+
+ - Features
+ -
+ {levels[kit.features]} (
features: {kit.features})
+
+ - Needs
+ -
+
ingestron {kit.requires.ingestron}
+
+
+ );
+}
+
+/** The note id of a kit file's blank, unique on the page. */
+const noteId = (id: string, file: string, index: number) =>
+ `${id}-${file}-${index + 1}`.replace(/[^a-z0-9-]+/gi, "-");
+
+type Transformer = NonNullable[number];
+type Content = Parameters>[0]["children"][number];
+
+/** Marks each REPLACE_ME token as a `replace-me` element that names its blank. */
+function markBlanks(blanks: Blank[]): Transformer {
+ return {
+ name: "replace-me",
+ span(node, _line, _column, _lineElement, token) {
+ const index = blanks.findIndex(
+ (blank) =>
+ blank.offset >= token.offset && blank.offset < token.offset + token.content.length,
+ );
+ if (index < 0) return;
+ const at = blanks[index]!.offset - token.offset;
+ const parts: Content[] = [
+ { type: "text", value: token.content.slice(0, at) },
+ {
+ type: "element",
+ tagName: "replace-me",
+ properties: { dataBlank: String(index) },
+ children: [{ type: "text", value: "REPLACE_ME" }],
+ },
+ { type: "text", value: token.content.slice(at + "REPLACE_ME".length) },
+ ];
+ node.children = parts.filter((part) => part.type !== "text" || part.value !== "");
+ },
+ };
+}
+
+/**
+ * A file of a kit as the kit has it. Each value still `REPLACE_ME` is marked,
+ * focusable, with a tooltip giving the field, where to find the value and a
+ * suggestion, from the same field reference as the editor's hover; the notes
+ * below the block say the same, and show on touch screens.
+ * @example
+ */
+export function KitFile({ id, file }: { id: string; file: string }) {
+ const kit = kitById(id);
+ const blanks = file.endsWith(".yaml") ? kitBlanks(kit, file) : [];
+ const unexplained = blanks.filter((blank) => !blank.help);
+ if (unexplained.length > 0)
+ throw new Error(
+ `${kit.id}@${kit.version} ${file}: no field explains the REPLACE_ME at ${unexplained.map((blank) => blank.pointer).join(", ")}.`,
+ );
+ const title = `${file}`;
+ const extension = file.split(".").pop() ?? "text";
+ return (
+ <>
+ (
+
+ {children}
+
+ ),
+ // Not an HTML element: the mark markBlanks writes in place of each REPLACE_ME token.
+ ...({
+ "replace-me": ({
+ "data-blank": blank,
+ children,
+ }: {
+ "data-blank": string;
+ children?: ReactNode;
+ }) => {
+ const index = Number(blank);
+ const help = blanks[index]!.help!;
+ return (
+
+ {children}
+
+ );
+ },
+ } as ServerCodeBlockProps["components"]),
+ }}
+ />
+ {blanks.length > 0 && (
+
+ {blanks.map((blank, index) => (
+ -
+ Line {blank.line}:{" "}
+ {" "}
+
+ Reference for
{blank.help!.field}
+
+
+ ))}
+
+ )}
+ >
+ );
+}
+
+/** Where an intent value came from, in the words the editor's hover uses. */
+function from(
+ origin: { layer: string; file?: string; line?: number; alias?: string },
+ kind: string,
+) {
+ const layer = {
+ pack: "the pack's default",
+ standard: "the standard",
+ platform: "the platform",
+ layer: "the table's layer in the standard",
+ rule: "a standard rule",
+ definition: kind === "Flow" ? "the flow" : "the transform",
+ table: "this table",
+ fixed: "the standard, fixed",
+ }[origin.layer];
+ const where = origin.file ? ` (${origin.file}:${origin.line})` : "";
+ return origin.alias ? `the setting "${origin.alias}" in ${layer}${where}` : `${layer}${where}`;
+}
+
+/**
+ * Each of the kit's tables, its intent and where each value came from, as
+ * `plan` resolves the kit and the editor's hover shows it.
+ * @example
+ */
+export function KitIntent({ id }: { id: string }) {
+ const kit = kitById(id);
+ return (
+
+
+
+
+ | Table |
+ Intent |
+ From |
+
+
+
+ {kitIntent(kit).map((row) => (
+
+
+
+ {row.layer}.{row.table}
+
+ |
+
+
+ {row.path}: {row.value}
+
+ |
+ {from(row.origin, row.kind)} |
+
+ ))}
+
+
+
+ );
+}
diff --git a/components/reference/pack-intent.tsx b/components/reference/pack-intent.tsx
new file mode 100644
index 0000000..d50bf64
--- /dev/null
+++ b/components/reference/pack-intent.tsx
@@ -0,0 +1,302 @@
+// What each pack does with each intent path and value, from the pack's own
+// `intent` declarations: the support matrix on each pack's page, the methods
+// that differ from their pack, and an overview of every pack for the intent
+// reference. Nothing here is written by hand.
+import type { FeatureStatus, IntentUse } from "@ingestron/definitions";
+import { intentPaths } from "@ingestron/definitions";
+import {
+ aliasTexts,
+ intentRows,
+ intentVocabulary,
+ methodIntentRows,
+ packName,
+ packOf,
+ packs,
+} from "@/lib/reference";
+import { Inline } from "./inline";
+import { TableFrame } from "@/components/table-frame";
+
+const uses = {
+ used: "Used",
+ "not-yet": "Not yet",
+ elsewhere: "Elsewhere",
+ "not-applicable": "Not applicable",
+} as const;
+
+/** What the entry says: what the pack does, or why it does not. */
+function said(use: IntentUse): string {
+ return use.use === "used" ? use.does : use.reason;
+}
+
+/** A value as the vocabulary writes it; `*` is any text, such as a column's name. */
+function Values({ values }: { values: string[] }) {
+ if (values.length === 1 && values[0] === "*") return <>any>;
+ return (
+ <>
+ {values.map((value, index) => (
+
+ {index > 0 && ", "}
+ {value}
+
+ ))}
+ >
+ );
+}
+
+/** The platform feature's release level, linked to the page that says so. */
+function Release({ status }: { status?: FeatureStatus }) {
+ if (!status) return null;
+ const words = status.level === "ga" ? "Generally available" : (status.label ?? "In preview");
+ return {words};
+}
+
+/**
+ * The pack's support matrix: each intent path and value, whether the pack
+ * uses it, what it does or why not, and, where it is used, the platform
+ * feature's release level and the date it was checked. Then each method whose
+ * own entries differ from the pack's.
+ * @example
+ */
+export function IntentSupport({ pack }: { pack: string }) {
+ const name = packName(pack);
+ const methods = methodIntentRows(pack);
+ return (
+ <>
+
+
+
+
+ | Intent |
+ Value |
+ Use |
+ What the pack does, or why not |
+ Release |
+ Checked |
+
+
+
+ {intentRows(pack).map(({ path, values, use }) => (
+
+
+ {path}
+ |
+
+
+ |
+ {uses[use.use]} |
+
+
+ |
+ {use.use === "used" && } |
+ {use.use === "used" && use.status?.checked} |
+
+ ))}
+
+
+
+ {methods.length > 0 && (
+ <>
+ Where a method differs from the pack, its own entry wins for the tables it builds:
+
+
+
+
+ | Method |
+ Intent |
+ Value |
+ Use |
+ What the method does, or why not |
+ Release |
+
+
+
+ {methods.map(({ method, path, value, use }) => (
+
+
+
+ {method}
+
+ |
+
+ {path}
+ |
+ {value === undefined ? "every value" : } |
+ {uses[use.use]} |
+
+
+ |
+ {use.use === "used" && } |
+
+ ))}
+
+
+
+ >
+ )}
+ >
+ );
+}
+
+/** One word per pack for one intent value: used (and in preview), or not. */
+function Cell({ use }: { use: IntentUse }) {
+ if (use.use !== "used") return <>{uses[use.use]}>;
+ return (
+ <>
+ Used
+ {use.status?.level === "preview" && <> ({use.status.label ?? "in preview"})>}
+ >
+ );
+}
+
+/**
+ * Every intent path and value, and what each pack does with it, in one word;
+ * each pack's name links to its full matrix. `only` keeps some, each written
+ * `path=value`, or `path` for every value of it; `packs` keeps some packs.
+ * @example
+ * @example
+ */
+export function IntentOverview({ only, packs: shown }: { only?: string[]; packs?: string[] }) {
+ const columns = shown ? shown.map(packOf) : packs;
+ const rows = Object.fromEntries(columns.map((pack) => [pack.id, intentRows(pack.id)]));
+ const useOf = (pack: string, path: string, value: string) =>
+ rows[pack]!.find((row) => row.path === path && row.values.includes(value))!.use;
+ const kept = Object.entries(intentPaths).flatMap(([path, values]) =>
+ values
+ .filter((value) => !only || only.includes(path) || only.includes(`${path}=${value}`))
+ .map((value) => ({ path, value })),
+ );
+ if (only && kept.length === 0) throw new Error(`No intent matches ${only.join(", ")}.`);
+ return (
+
+
+
+
+ | Intent |
+ Value |
+ {columns.map((pack) => (
+
+ {packName(pack.id)}
+ |
+ ))}
+
+
+
+ {kept.map(({ path, value }) => (
+
+
+ {path}
+ |
+
+
+ |
+ {columns.map((pack) => (
+
+ |
+ |
+ ))}
+
+ ))}
+
+
+
+ );
+}
+
+/**
+ * The intent vocabulary: each path, what it means, its values with their
+ * meanings, and where it can be written, from the documents' own fields.
+ * @example
+ */
+export function IntentVocabulary() {
+ return (
+
+
+
+
+ | Intent |
+ Meaning and values |
+ Written in |
+
+
+
+ {intentVocabulary().map(({ path, field, values, places }) => (
+
+
+
+ {path}
+
+ |
+
+ {field.description && }
+ {values.length === 1 && values[0]!.value === "*" ? (
+ " Any value."
+ ) : (
+
+ {values.map(({ value, description }) => (
+ -
+
{value}
+ {description && (
+ <>
+ :
+ >
+ )}
+
+ ))}
+
+ )}
+ |
+ {places.join(", ")} |
+
+ ))}
+
+
+
+ );
+}
+
+/**
+ * Every method setting that is another name for intent, by pack and method,
+ * linked to the method's reference.
+ * @example
+ */
+export function IntentAliases() {
+ const rows = packs.flatMap((pack) =>
+ Object.keys(pack.methods).flatMap((method) => {
+ const texts = aliasTexts(pack.id, method);
+ return texts.length > 0 ? [{ pack: pack.id, method, texts }] : [];
+ }),
+ );
+ return (
+
+
+
+
+ | Method |
+ Setting and the intent it means |
+
+
+
+ {rows.map(({ pack, method, texts }) => (
+
+
+
+ {method}
+ {" "}
+ ({packName(pack)})
+ |
+
+
+ {texts.map((text) => (
+ -
+
+
+ ))}
+
+ |
+
+ ))}
+
+
+
+ );
+}
diff --git a/components/reference/pack-methods.tsx b/components/reference/pack-methods.tsx
index d339cd5..9c47507 100644
--- a/components/reference/pack-methods.tsx
+++ b/components/reference/pack-methods.tsx
@@ -1,28 +1,48 @@
-// Every method of a pack: what it does, what it reads and hands over, its settings.
-import { packs } from "@/lib/reference";
+// Every method of a pack: what it does, what it reads and hands over, the
+// intent it gives and the settings that are other names for intent, and its
+// settings.
+import { aliasTexts, intentPairs, packOf } from "@/lib/reference";
+import { Inline } from "./inline";
import { SettingsTable } from "./settings-table";
/** One section per method, with an id the connector pages link to. */
export function PackMethods({ pack }: { pack: string }) {
- const found = packs.find((candidate) => candidate.id === pack);
- if (!found) throw new Error(`No pack "${pack}".`);
+ const found = packOf(pack);
return (
<>
- {Object.entries(found.methods).map(([id, method]) => (
-
-
- {id} ({method.action})
-
- {method.description}
-
- {method.connectors && - Reads: {method.connectors.join(", ")}
}
- {method.connectivity && - Connectivity: {method.connectivity.join(", ")}
}
- {method.accepts && - Accepts: {method.accepts.join(", ")}
}
- - Hands over: {method.produces}
-
-
-
- ))}
+ {Object.entries(found.methods).map(([id, method]) => {
+ const defaults = method.intentDefaults ? intentPairs(method.intentDefaults) : [];
+ const aliases = aliasTexts(pack, id);
+ return (
+
+
+ {id} ({method.action})
+
+ {method.description}
+
+ {method.connectors && - Reads: {method.connectors.join(", ")}
}
+ {method.connectivity && - Connectivity: {method.connectivity.join(", ")}
}
+ {method.accepts && - Accepts: {method.accepts.join(", ")}
}
+ - Hands over: {method.produces}
+ {defaults.length > 0 && (
+ -
+ `\`${pair}\``).join(", ")}`}
+ />
+
+ )}
+ {aliases.length > 0 && (
+ -
+
+
+ )}
+
+
+
+ );
+ })}
>
);
}
diff --git a/components/reference/versions.tsx b/components/reference/versions.tsx
index 40096f0..e082d91 100644
--- a/components/reference/versions.tsx
+++ b/components/reference/versions.tsx
@@ -33,6 +33,10 @@ export function PackageVersions() {
["@ingestron/core", "the API the command is built on"],
["@ingestron/definitions", "loading and resolving definitions"],
["@ingestron/authoring", "editor features"],
+ [
+ "@ingestron/standards",
+ "the [published standards](/docs/how-to/choose-a-published-standard) installed with the command",
+ ],
...packs.map((pack): [string, string] => [
`@ingestron/pack-${pack.id}`,
`${packName(pack.id)} pack, \`${pack.id}@${pack.version}\``,
diff --git a/components/reference/vscode-extension.json b/components/reference/vscode-extension.json
index 3c04cdb..bf38d5b 100644
--- a/components/reference/vscode-extension.json
+++ b/components/reference/vscode-extension.json
@@ -1,3 +1,3 @@
{
- "version": "2.1.4"
+ "version": "2.2.0"
}
diff --git a/components/replace-me.tsx b/components/replace-me.tsx
new file mode 100644
index 0000000..ba6b148
--- /dev/null
+++ b/components/replace-me.tsx
@@ -0,0 +1,135 @@
+"use client";
+// A value still REPLACE_ME in a kit's file (components/reference/kits.tsx):
+// marked by an outline and its note's number, not by colour alone, and
+// focusable. On hover or focus it shows the field, where to find the value and
+// a suggestion, in a tooltip drawn above the page, so the code block's
+// scrolling frame does not clip it and Copy does not copy it. Escape closes
+// it, and the pointer can move onto it. The same text is the note the mark's
+// aria-describedby names, which touch screens show below the block.
+import { useEffect, useLayoutEffect, useRef, useState, type ReactNode } from "react";
+import { createPortal } from "react-dom";
+import { Inline } from "./reference/inline";
+
+/** The help one mark gives, as the editor's hover gives it. */
+export interface ReplaceMeTip {
+ field: string;
+ description?: string;
+ find?: string;
+ suggest?: string;
+}
+
+/** One REPLACE_ME, with the id of its note and the note's number. */
+export function ReplaceMe({
+ note,
+ number,
+ tip,
+ children,
+}: {
+ note: string;
+ number: number;
+ tip?: ReplaceMeTip;
+ children: ReactNode;
+}) {
+ const mark = useRef(null);
+ const box = useRef(null);
+ const closing = useRef(undefined);
+ const [open, setOpen] = useState(false);
+ const [place, setPlace] = useState<{ top: number; left: number } | undefined>();
+
+ const show = () => {
+ window.clearTimeout(closing.current);
+ setOpen(true);
+ };
+ // A moment's grace, so the pointer can move from the mark onto the tooltip.
+ const hide = () => {
+ window.clearTimeout(closing.current);
+ closing.current = window.setTimeout(() => setOpen(false), 150);
+ };
+
+ // Below the mark, or above it when there is no room, and inside the window.
+ useLayoutEffect(() => {
+ if (!open || !mark.current || !box.current) return;
+ const at = mark.current.getBoundingClientRect();
+ const size = box.current.getBoundingClientRect();
+ const below = at.bottom + 6;
+ const top = below + size.height > window.innerHeight ? at.top - 6 - size.height : below;
+ const left = Math.max(8, Math.min(at.left, window.innerWidth - size.width - 8));
+ setPlace({ top: Math.max(8, top), left });
+ }, [open]);
+
+ useEffect(() => {
+ if (!open) return;
+ const close = () => setOpen(false);
+ const escape = (event: KeyboardEvent) => {
+ if (event.key === "Escape") close();
+ };
+ window.addEventListener("scroll", close, true);
+ window.addEventListener("resize", close);
+ window.addEventListener("keydown", escape);
+ return () => {
+ window.removeEventListener("scroll", close, true);
+ window.removeEventListener("resize", close);
+ window.removeEventListener("keydown", escape);
+ };
+ }, [open]);
+
+ useEffect(() => () => window.clearTimeout(closing.current), []);
+
+ return (
+ setOpen(false)}
+ >
+ {children}
+ {open &&
+ tip &&
+ createPortal(
+
+
+
,
+ document.body,
+ )}
+
+ );
+}
+
+/** The help as text: the field, then what it is, where to find it and a suggestion. */
+export function TipText({ tip }: { tip: ReplaceMeTip }) {
+ return (
+ <>
+
+ {tip.field}
+ {" "}
+ still needs a value.
+ {tip.description && (
+
+
+
+ )}
+ {tip.find && (
+
+ Where to find it:
+
+ )}
+ {tip.suggest && (
+
+ Suggested: {tip.suggest}
+
+ )}
+ >
+ );
+}
diff --git a/content/docs/(reference)/reference/cli.mdx b/content/docs/(reference)/reference/cli.mdx
index 3f1b7a9..c83006a 100644
--- a/content/docs/(reference)/reference/cli.mdx
+++ b/content/docs/(reference)/reference/cli.mdx
@@ -5,7 +5,8 @@ description: The ingestron command's commands, options, output and exit codes.
The `ingestron` command reads a project folder and writes each platform's
files, for every environment. [Install it](/docs/start/install) first. `ingestron init` creates a
-project; see [Create a project](/docs/how-to/start-from-a-template). Then
+project; see [Create a project](/docs/how-to/start-from-a-template), or
+[Published standards](#standard) below. Then
`ingestron plan` and `ingestron apply` are the everyday loop; see
[Plan, apply and review](/docs/how-to/plan-and-generate). In CI, run
`ingestron check`; see [Check generated files in CI](/docs/how-to/check-in-ci).
@@ -36,6 +37,29 @@ ingestron check --definitions shop --out shop/build
With `--definitions shop` alone, `apply` would write `build` in the folder you
ran it from, not in `shop`.
+## Published standards [#standard]
+
+`ingestron init --kit @` creates a project from a
+[published standard](/docs/how-to/choose-a-published-standard). `--kit` takes
+the name and version, such as `duckdb/local-medallion@2026-11-10`; the name
+alone, for the newest version that is not a preview; or
+`#@`, for one kept in a Git repository. It is not
+combined with `--template` or `--standard`. See
+[Start from a published standard](/docs/how-to/start-from-a-published-standard).
+
+`ingestron standard` takes one of two commands:
+
+- `ingestron standard install ` replaces the project's standard with the
+ published standard's, keeping its path and `id`, and lists the published
+ standard's other files that changed; it refuses a standard file with
+ changes that are not committed. It exits `0` once the file is written, and
+ `1` when it writes nothing. See
+ [Move to a newer published standard](/docs/how-to/upgrade-a-standard).
+- `ingestron standard list` lists the versions installed with the command,
+ then those only the public catalogue has. `--platform ` keeps one
+ platform's. It always exits `0`. See
+ [List them](/docs/how-to/choose-a-published-standard#list-them).
+
## Output
`plan` and `apply` print one block for each output folder under `--out`, such
diff --git a/content/docs/(reference)/reference/documents.mdx b/content/docs/(reference)/reference/documents.mdx
index 117c2cf..5dd042b 100644
--- a/content/docs/(reference)/reference/documents.mdx
+++ b/content/docs/(reference)/reference/documents.mdx
@@ -4,7 +4,9 @@ description: Every field of project.yaml and the Standard, Platform, Connection,
---
Each document starts with `apiVersion: ingestron.io/v1`, its `kind` and an
-`id`. The fields below come from the documents' schemas. For how the
+`id`. The fields below come from the documents' schemas, with where to find a
+value and a suggestion for each field that has them, as the editor's hover
+gives them. For how the
documents work together, see [Definitions](/docs/concepts/definitions).
Your editor can check these as you type: run `ingestron schemas` and follow
@@ -24,10 +26,21 @@ documents read with `${values.…}`, and the files to load. It has no
See [Standards and packs](/docs/concepts/standards-and-packs),
-[Which setting wins](/docs/concepts/settings) and [Name tables and resources](/docs/how-to/name-things-your-way).
+[Which setting wins](/docs/concepts/settings), [Intent](/docs/reference/intent)
+for `layers`, `load`, `audit`, `operations` and the other intent, and
+[Name tables and resources](/docs/how-to/name-things-your-way).
## Platform
+Each pack can have a section of its own, under its id, such as `databricks:`;
+its fields are on the pack's page, as for
+[the `databricks:` section](/docs/reference/packs/databricks#platform). The
+Databricks fields below, `workspace`, `mode`, `run_as`, `compute`,
+`permissions`, `manage` and `grants`, are read at the top of the platform as
+before, with a
+[`PLATFORM_FIELD_MOVED`](/docs/reference/errors#PLATFORM_FIELD_MOVED) warning,
+and `notifications` with one that moves it to `alerts`.
+
See [Environments and platform](/docs/concepts/environments-and-platform).
diff --git a/content/docs/(reference)/reference/intent.mdx b/content/docs/(reference)/reference/intent.mdx
new file mode 100644
index 0000000..5d093bb
--- /dev/null
+++ b/content/docs/(reference)/reference/intent.mdx
@@ -0,0 +1,167 @@
+---
+title: Intent
+description: The words a standard, flow or transform uses for how data is loaded, kept and run, which value wins, where each came from, and what each pack does with it.
+---
+
+Intent says how a table's data is loaded, kept and run, in words that belong
+to no platform: `load: files`, `history: latest`, `audit: [loaded_at]`. Each
+pack turns it into its platform's terms, and says what it cannot do. Every
+part is optional: a project that writes none builds as it did before.
+
+This is the standard of the
+[local medallion](/docs/how-to/kits/duckdb-local-medallion) published
+standard. Its `layers`, `load` and `features` are intent; `ingest` is a method
+setting:
+
+
+
+## Paths and values [#paths]
+
+Each path, what it means, its values, and the places that can write it.
+Follow a path's link for every field that writes it in
+[Documents](/docs/reference/documents).
+
+
+
+A duration is a whole number and a unit, `s`, `m`, `h` or `d`, such as `30d`.
+An audit value is a column after the contract's columns, named `_`,
+such as `_loaded_at`; `{ value: loaded_at, as: _ingested_at }` names it
+yourself. Two audit columns with one name, or one with a contract column's
+name, is [`BAD_SETTING`](/docs/reference/errors#BAD_SETTING).
+
+## Where to write it [#where]
+
+- **The standard**, for every table: `load` (`default`, `deletes` and `when`
+ rules), `audit`, `rescued`, `retention`, `storage`, `quality.results` and
+ `operations`. `load.when` rules match a table's `connector`, `connectivity`
+ and `layer`; the first that matches chooses its load mode.
+- **A layer** of the standard, for every table in it: write the layer as an
+ object, `{ name: silver, history: latest }`, with `history`, `audit`,
+ `rescued` and `retention`. A layer written as a name gives nothing. Each
+ layer is listed once; a second is
+ [`DUPLICATE_LAYER`](/docs/reference/errors#DUPLICATE_LAYER).
+- **A flow**, for its tables, and **a flow table**, for itself: `load`
+ (`mode`, `cursor` and `deletes`), `history`, `sequence` and `audit`.
+- **A transform**, for its table: `history`, `sequence` and `audit`. A
+ transform's table reads no source, so it has no `load`, and the audit
+ values only a source gives, such as `source_file`, are
+ [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) there.
+- **The standard's `fixed`**: any of these that no layer, flow or transform
+ may change, such as `fixed: { history: latest }`. A definition that writes
+ another value is [`FIXED_SETTING`](/docs/reference/errors#FIXED_SETTING).
+
+This flow of the
+[streaming lakehouse](/docs/how-to/kits/databricks-lakehouse-streaming)
+published standard says which change to an order is newest, and which change
+is a delete:
+
+
+
+## Which value wins [#precedence]
+
+Each table's intent is resolved before its methods are chosen, from these
+places, a later one winning:
+
+1. the standard: `load.default`, `load.deletes`, `audit`, `rescued`,
+ `retention` and `storage`;
+2. the table's layer in the standard;
+3. the first `load.when` rule of the standard that matches the table;
+4. the flow or the transform;
+5. the flow table;
+6. the standard's `fixed`.
+
+Mappings merge key by key, so a flow's `load: { cursor: updated_at }` keeps
+the load mode the standard gave. A list, such as `audit`, is replaced whole.
+
+Then the methods are chosen, and what is still unset comes from the chosen
+method's own default: `streaming-table` keeps every row as it arrived,
+`history: none`, unless something says otherwise. Each method's defaults are
+on its pack's page, under the method. A `${envvar:…}` value cannot decide
+what is generated, so in `load.mode`, `load.deletes.mode`, `history` or
+`storage.features` it sets nothing, and is
+[`DEPLOY_VALUE`](/docs/reference/errors#DEPLOY_VALUE) when deploying.
+
+## Where a value came from [#origins]
+
+Each value of a table's intent remembers where it was written. In VS Code,
+hover a flow table or a transform: the hover lists the table's intent, each
+value with where it came from, in these words:
+
+- **the standard**, **the table's layer in the standard**, **a standard rule**
+ or **the standard, fixed**, with the file and line;
+- **the flow** or **the transform**, and **this table**, with the file and
+ line;
+- **the setting "…"** in one of those places, for a
+ [setting that is intent](#settings-that-are-intent);
+- **the platform**, for such a setting in the platform's `settings`;
+- **the pack's default**, with no file.
+
+This is how each table of the streaming lakehouse resolves:
+
+
+
+## Settings that are intent [#settings-that-are-intent]
+
+Some method settings mean the same as intent. They still work, as intent
+written at the place the setting is: a flow's `store.with.mode: append`
+wins over a layer's `history: latest`, as the flow's own `history: none`
+would.
+
+
+
+`plan` reports each such setting as
+[`SETTING_MOVED`](/docs/reference/errors#SETTING_MOVED). In a flow's or a
+table's `with`, its fix, **Write as intent**, writes the intent there and
+removes the setting. Anywhere else, such as the standard's settings or a
+rule's `with`, write the intent by hand where it should apply: written for a
+whole standard or layer, intent also reaches tables and methods the setting
+did not. A setting in the standard's `fixed.` is not reported: it
+fixes that intent for the method's tables.
+
+A setting and intent at the same place that disagree, such as
+`mode: append` beside `history: latest` on one table, is
+[`INTENT_CONFLICT`](/docs/reference/errors#INTENT_CONFLICT). A setting a
+pack's own rule writes is kept as the rule wrote it, and intent that it
+overrides is `NOT_APPLIED`.
+
+## What each pack does [#support]
+
+Every pack says what it does with each value: **Used**, with how; **Not yet**;
+**Elsewhere**, done by another step or pack, such as the pack that lands the
+files; or **Not applicable** on its platform. A value its pack does not apply
+is a [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) warning where it is
+written, with the pack's reason; **Elsewhere** never warns. Each pack's name
+links to its full table, with what it does and the platform feature's release
+level.
+
+
+
+## Release level of features [#features]
+
+Some values use platform features in preview, such as Databricks' run
+duration alert. The standard's `features` says which a project may use:
+
+- `features: ga`: only generally available features. A value that needs one
+ in preview is [`PREVIEW_FEATURE`](/docs/reference/errors#PREVIEW_FEATURE),
+ a warning that names the feature's release level and the date it was
+ checked, with a fix that allows preview features.
+- `features: preview`: preview features as well; they may change before they
+ are generally available.
+
+Without `features`, nothing is reported. Published standards say
+`features: ga` unless their version ends `-preview`.
+
+## Operations [#operations]
+
+`operations` is the policy for running what is generated: one schedule for
+the project (`schedule: { per: project }`), what starts a run (`trigger`),
+`retries`, a `timeout`, `alerts` and `concurrency`. Without it, the project
+runs on one schedule, and the platform's alerts are sent when a run fails. Values that differ per
+environment belong in the platform: the cron and time zone in `schedule`, who
+is told in `alerts`, and the tags in `tags`. `require_tags` names tags the
+platform must give; one missing is
+[`BAD_SETTING`](/docs/reference/errors#BAD_SETTING) at the platform's `tags`.
+
+The first pack in the standard's list that runs jobs applies `operations`:
+the Databricks refresh job, or the Data Factory triggers and copies. When no
+pack of the standard runs jobs, each value is `NOT_APPLIED`.
diff --git a/content/docs/(reference)/reference/meta.json b/content/docs/(reference)/reference/meta.json
index ac1125e..a887259 100644
--- a/content/docs/(reference)/reference/meta.json
+++ b/content/docs/(reference)/reference/meta.json
@@ -1,4 +1,4 @@
{
"title": "Reference",
- "pages": ["documents", "packs", "errors", "cli", "templates", "versions"]
+ "pages": ["documents", "intent", "packs", "errors", "cli", "templates", "versions"]
}
diff --git a/content/docs/(reference)/reference/packs/adf.mdx b/content/docs/(reference)/reference/packs/adf.mdx
index 6bb235c..8482bda 100644
--- a/content/docs/(reference)/reference/packs/adf.mdx
+++ b/content/docs/(reference)/reference/packs/adf.mdx
@@ -52,6 +52,16 @@ or step and never warn.
+## Intent [#intent]
+
+What this pack does with each value of [intent](/docs/reference/intent), from
+the pack itself. ADF lands files, so what a table keeps is the store pack's. A value the pack does not apply, **Not yet** or
+**Not applicable**, is a [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED)
+warning where it is written, with the reason below. **Elsewhere** values are
+applied by another pack or step and never warn.
+
+
+
## Native configuration sources
diff --git a/content/docs/(reference)/reference/packs/databricks.mdx b/content/docs/(reference)/reference/packs/databricks.mdx
index 8688cc9..a25c5a8 100644
--- a/content/docs/(reference)/reference/packs/databricks.mdx
+++ b/content/docs/(reference)/reference/packs/databricks.mdx
@@ -19,7 +19,8 @@ then [deploy the generated bundle](/docs/platforms/databricks-deploy#deploy).
Without a platform document, or where it leaves a setting out, the Databricks
pack uses these defaults. Each target of the bundle gets its environment's
-`mode`, workspace, `run_as` and `permissions`. Without `run_as`, the pipelines and jobs run as the
+`mode`, workspace, `run_as` and `permissions`, from the platform's
+`databricks:` section. Without `run_as`, the pipelines and jobs run as the
identity that deploys the bundle, and without `schedule`, nothing runs on a
schedule.
@@ -27,13 +28,29 @@ schedule.
-Only enable management of shared resources when this project owns them. The
-following capability reference describes each switch and its prerequisites.
+`schedule`, `tags` and `alerts` stay at the top of the platform, where every
+pack that runs jobs reads them. Only enable management of shared resources
+when this project owns them. The following capability reference describes
+each switch and its prerequisites.
+
+### The `databricks:` section [#platform]
+
+What only Databricks reads goes in the platform's `databricks:` section. Each
+field says where to find its value, and what it could look like; the editor's
+hover says the same.
+
+
+
+Written at the top of the platform, as before 2.3, `workspace`, `mode`,
+`run_as`, `compute`, `permissions`, `manage` and `grants` still work, with a
+[`PLATFORM_FIELD_MOVED`](/docs/reference/errors#PLATFORM_FIELD_MOVED) warning
+whose fix moves each into the section. `notifications.on_failure` becomes an
+email alert in `alerts`.
## Platform prerequisites
Some features change shared Unity Catalog objects, so the platform turns each
-on with a `manage` switch. While a switch is off, a definition or contract that
+on with a `databricks.manage` switch. While a switch is off, a definition or contract that
uses the feature gets a `CAPABILITY_OFF` warning with a quick fix that turns it
on. When a switch is on, the bundle has a `prerequisites/.sql` for
each environment, which lists the grants to run once there as an
@@ -54,3 +71,16 @@ field with the reason below; **Elsewhere** fields are handled by another pack
or step and never warn.
+
+## Intent [#intent]
+
+What this pack does with each value of [intent](/docs/reference/intent), from
+the pack itself. A value the pack does not apply, **Not yet** or **Not
+applicable**, is a [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) warning
+where it is written, with the reason below. **Elsewhere** values are applied by
+another pack or step and never warn. **Release** is the Databricks feature's
+own release level, as the linked page said on the date checked; with
+`features: ga`, a feature in preview is a
+[`PREVIEW_FEATURE`](/docs/reference/errors#PREVIEW_FEATURE) warning.
+
+
diff --git a/content/docs/(reference)/reference/packs/duckdb.mdx b/content/docs/(reference)/reference/packs/duckdb.mdx
index be4c0cc..8cf1932 100644
--- a/content/docs/(reference)/reference/packs/duckdb.mdx
+++ b/content/docs/(reference)/reference/packs/duckdb.mdx
@@ -25,3 +25,13 @@ field with the reason below; **Elsewhere** fields are handled by another pack
or step and never warn.
+
+## Intent [#intent]
+
+What this pack does with each value of [intent](/docs/reference/intent), from
+the pack itself. A value the pack does not apply, **Not yet** or
+**Not applicable**, is a [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED)
+warning where it is written, with the reason below. **Elsewhere** values are
+applied by another pack or step and never warn.
+
+
diff --git a/content/docs/(reference)/reference/packs/synapse.mdx b/content/docs/(reference)/reference/packs/synapse.mdx
index 676f092..dc62c2a 100644
--- a/content/docs/(reference)/reference/packs/synapse.mdx
+++ b/content/docs/(reference)/reference/packs/synapse.mdx
@@ -26,3 +26,13 @@ field with the reason below; **Elsewhere** fields are handled by another pack
or step and never warn.
+
+## Intent [#intent]
+
+What this pack does with each value of [intent](/docs/reference/intent), from
+the pack itself. A view keeps nothing, so most intent is not applicable. A value the pack does not apply, **Not yet** or
+**Not applicable**, is a [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED)
+warning where it is written, with the reason below. **Elsewhere** values are
+applied by another pack or step and never warn.
+
+
diff --git a/content/docs/(reference)/reference/templates.mdx b/content/docs/(reference)/reference/templates.mdx
index a855f3f..32ddde8 100644
--- a/content/docs/(reference)/reference/templates.mdx
+++ b/content/docs/(reference)/reference/templates.mdx
@@ -7,7 +7,11 @@ A template is a project folder with a `template.yaml`. `ingestron init` copies
the folder into a new project, writes its `README.md`, and asks nothing. A
starter is a template that ships with Ingestron: a complete project for one
common situation, with `REPLACE_ME` where you write your own values. See [Share templates and standards](/docs/how-to/share-a-template)
-to make a template of your own.
+to make a template of your own. A
+[published standard](/docs/how-to/choose-a-published-standard), which
+`--kit` names, is a dated standard for one platform, with a project that shows
+it; its `kit.yaml` is described in
+[Share a published standard](/docs/how-to/share-a-template#share-a-published-standard).
## Create a project from a starter [#create-a-project-from-a-starter]
diff --git a/content/docs/(reference)/reference/versions.mdx b/content/docs/(reference)/reference/versions.mdx
index 2f64233..f65460b 100644
--- a/content/docs/(reference)/reference/versions.mdx
+++ b/content/docs/(reference)/reference/versions.mdx
@@ -29,5 +29,5 @@ Connector maturity, such as Public Preview or Beta, is shown on each
The [deployment templates](/docs/how-to/deploy-through-environments) pin the
tools their stages run: `@microsoft/azure-data-factory-utilities` 1.0.3, the
Databricks CLI v1.23.0, and sqlcmd (Go) v1.10.0. Their check stage installs
-`ingestron@2.2.0`, and each GitHub action is pinned to a commit, with its
+`ingestron@2.3.0`, and each GitHub action is pinned to a commit, with its
version beside it.
diff --git a/content/docs/(reference)/tools/vscode.mdx b/content/docs/(reference)/tools/vscode.mdx
index f971a0a..603f523 100644
--- a/content/docs/(reference)/tools/vscode.mdx
+++ b/content/docs/(reference)/tools/vscode.mdx
@@ -112,7 +112,8 @@ CLI prints the first fix under the problem, as `Fix: …`.
When the same problem has a fix in several places, the menu also offers
**Fix all N … problems**, which applies each one's first fix in one change; an
-edit several of them share, such as turning on `manage.checks`, is made once.
+edit several of them share, such as turning on `databricks.manage.checks`, is
+made once.
A method's setting written outside the file its method says it belongs in,
such as a table's `properties` in a flow or `integration_runtime` in a
@@ -222,7 +223,15 @@ Hover a flow table or a transform to see its full name, the method for each
step, and every setting with the layer that set it: the pack's default, the
standard, the platform, a rule, the flow or transform, the table itself, or a
setting the standard fixes, with the file and line. Hover a setting under
-`with` for that one value.
+`with` for that one value. The same hover lists the table's
+[intent](/docs/reference/intent), each value with where it came from, such as
+`history`: `latest`, from the table's layer in the standard
+(`standard.yaml:18`).
+
+Hover a value that is still `REPLACE_ME` to see the field it fills, what the
+field is, where to find the value, and a suggestion of what it looks like,
+such as ``. A value in `project.yaml`'s `values` is explained
+by the field that reads it.
Hover any field or setting to see what it is, what each allowed value means,
its default, an example and where it is usually set; hover a value for that
@@ -402,17 +411,23 @@ workspace.
## Start a project
-**Ingestron: New project** starts from a connection or from a starter.
+**Ingestron: New project** starts from a connection, a starter or a published
+standard.
- **From a connection** asks for a source category, connection, network
reachability and supported route. You can search across all sources or
within a category.
- **From a starter** offers the ready-made projects, such as **DuckDB on a
laptop**, which [runs locally with DuckDB](/docs/platforms/duckdb).
+- **From a published standard** offers the dated standards installed with the
+ extension, such as **Local medallion on DuckDB**, and makes the same project
+ as `ingestron init --kit`; see
+ [Start from a published standard](/docs/how-to/start-from-a-published-standard).
Nothing else is asked. The destination folder comes last, and its name is the
-project's id; the environments are `dev` and `prod`, or the starter's, and the
-standard is the route's or the starter's own. The new `README.md` opens in a
+project's id; the environments are `dev` and `prod`, or the starter's or the
+published standard's, and the standard is the route's, the starter's or the
+published one. The new `README.md` opens in a
preview, and a message says how many values are still `REPLACE_ME`: the
README lists each with where to find it. When the folder is outside the
workspace, the message offers to open it or add it to the workspace. See
@@ -422,7 +437,7 @@ workspace, the message offers to open it or add it to the workspace. See
light="/images/vscode/new-project-light.png"
dark="/images/vscode/new-project-dark.png"
alt="The New project picker asks How do you want to start? and offers From a connection, to choose a source, how it is reached and a supported route, and From a starter, a ready-made project to try, such as DuckDB on a laptop."
- caption="New project's first question: from a connection or from a starter. The folder comes last."
+ caption="New project's first question: how to start. The folder comes last."
/>
## Check files with the YAML extension [#check-files-with-the-yaml-extension]
diff --git a/content/docs/(reference)/troubleshooting.mdx b/content/docs/(reference)/troubleshooting.mdx
index 7c94195..8d65137 100644
--- a/content/docs/(reference)/troubleshooting.mdx
+++ b/content/docs/(reference)/troubleshooting.mdx
@@ -544,8 +544,9 @@ flows/erp.yaml:6:13 error METHOD_CHECK
under the same name every load would never be read again. For the same reason,
a file you re-export under its old name is skipped.
-**Fix:** use `replace` or `upsert`, or give each load its own file, such as
-with `increments`. To reload everything, delete `ingestron.duckdb` and run
+**Fix:** keep the table with `history: none` (the method's `replace`) or
+`history: latest` (`upsert`) instead of `mode: append`, or give each load its
+own file, such as with `increments`. To reload everything, delete `ingestron.duckdb` and run
again. See [Keep a table current](/docs/how-to/keep-a-table-current#on-duckdb).
### Upsert needs a primary key
@@ -557,8 +558,9 @@ flows/erp.yaml:6:13 error METHOD_CHECK
"duckdb-table" for table "invoices": upsert needs the contract's primaryKey.
```
-**Cause:** `upsert` replaces rows with the same key, and the contract names no
-key. `sequence_by` has to be a column of the contract too.
+**Cause:** `history: latest`, the method's `upsert`, replaces rows with the
+same key, and the contract names no key. `sequence` has to be a column of the
+contract too.
**Fix:** mark the key column `primaryKey: true` in the contract. See
[Keep a table current](/docs/how-to/keep-a-table-current).
@@ -649,12 +651,13 @@ flows/crm.yaml:9:5 error METHOD_CHECK
or `mode upsert needs primary key columns in the contract`.
-**Cause:** every `mode` but `append` keeps a table current by key, in the order
+**Cause:** `history: latest` or `full`, which the method writes as
+`mode: upsert` or `mode: history`, keeps a table current by key, in the order
of a column.
-**Fix:** set `sequence_by` to the column that orders a key's changes, and mark
-the key columns `primaryKey: true` in the contract. See
-[Keep a table current](/docs/how-to/keep-a-table-current).
+**Fix:** set `sequence` on the flow or the table to the column that orders a
+key's changes, and mark the key columns `primaryKey: true` in the contract.
+See [Keep a table current](/docs/how-to/keep-a-table-current).
### Databricks rejects the bundle
@@ -667,7 +670,7 @@ them, or see
A Lakeflow Connect pipeline refers to a Unity Catalog connection, which
Ingestron does not create.
-**Fix:** when the platform sets `manage.schemas`, an administrator runs the
+**Fix:** when the platform sets `databricks.manage.schemas`, an administrator runs the
generated `prerequisites/.sql` once before the first deploy: it grants
`USE CATALOG` and `CREATE SCHEMA` on each catalog to the identity the pipelines
run as, and the same must be given to the identity that deploys when it
diff --git a/content/docs/concepts/environments-and-platform.mdx b/content/docs/concepts/environments-and-platform.mdx
index 100a715..567f66f 100644
--- a/content/docs/concepts/environments-and-platform.mdx
+++ b/content/docs/concepts/environments-and-platform.mdx
@@ -38,9 +38,13 @@ an ADF method. A project still has one Platform document when it uses several
packs. The method reference says which fields it consumes and what must already
exist before deployment.
-Workspace, compute, identity, scheduling and `manage` fields mean something
-specific to each platform, and not every platform uses each one. See
-[Databricks platform configuration](/docs/reference/packs/databricks#platform-configuration)
+`schedule`, `tags` and `alerts` sit at the top of the platform, for every
+pack that runs jobs: when they run, how they are tagged, and who is told when
+a run fails. What only one platform reads goes in that pack's own section,
+under its id: the Databricks workspace, identity, compute, permissions and
+`manage` switches are under `databricks:`. A section for a pack the standard
+does not name is [`UNKNOWN_PACK`](/docs/reference/errors#UNKNOWN_PACK). See
+[the `databricks:` section](/docs/reference/packs/databricks#platform)
and [ADF runtime and Key Vault](/docs/platforms/adf#runtime-and-key-vault) for
their effects.
diff --git a/content/docs/concepts/layers.mdx b/content/docs/concepts/layers.mdx
index 17ae5be..0c4a803 100644
--- a/content/docs/concepts/layers.mdx
+++ b/content/docs/concepts/layers.mdx
@@ -52,12 +52,12 @@ use `{{ layer }}` anywhere; see
**Rules.** A rule in the standard can match `layer`, to choose a method or
settings for the tables in that layer. This standard makes every `gold`
-table a materialised view, with no clustering and a longer log retention:
+table a materialised view, with a longer log retention:
-`region_sales` is the only `gold` table, so only its view has clustering
-off and the longer retention. `store_sales` keeps the pack's settings.
+`region_sales` is the only `gold` table, so only its view has the longer
+retention. `store_sales` keeps the pack's settings.
## Name your own layers
diff --git a/content/docs/concepts/settings.mdx b/content/docs/concepts/settings.mdx
index 0174ca9..c847e53 100644
--- a/content/docs/concepts/settings.mdx
+++ b/content/docs/concepts/settings.mdx
@@ -37,6 +37,21 @@ missing setting is reported with the file and line where it was written
([`BAD_SETTING`](/docs/reference/errors#BAD_SETTING)). Each method's
settings and defaults are in [Reference: packs and methods](/docs/reference/packs).
+## Intent
+
+[Intent](/docs/reference/intent), such as `history` or `audit`, has an order
+of its own: the standard, then the table's layer, a `load.when` rule, the flow
+or transform, and the table, with the standard's `fixed` intent last. What is
+still unset comes from the chosen method's own default. See
+[Which value wins](/docs/reference/intent#precedence).
+
+Some settings are other names for intent, such as `streaming-table`'s
+`mode: upsert`, which is `history: latest`. Written as a setting, one still
+works, in the place it is written, and `plan` says
+[`SETTING_MOVED`](/docs/reference/errors#SETTING_MOVED), with a fix that
+writes the intent instead; see
+[Settings that are intent](/docs/reference/intent#settings-that-are-intent).
+
## Names, layers, types and quality
These are not method settings, and `fixed` does not apply to them. For each,
diff --git a/content/docs/concepts/standards-and-packs.mdx b/content/docs/concepts/standards-and-packs.mdx
index 854f3b4..f5a5384 100644
--- a/content/docs/concepts/standards-and-packs.mdx
+++ b/content/docs/concepts/standards-and-packs.mdx
@@ -31,9 +31,30 @@ This example uses a pack for copying and another for downstream storage:
-[Choose where your data goes](/docs/start/choose-a-route) lists the routes and
-the packs each one uses; [Packs and methods](/docs/reference/packs) lists each
-pack's methods and capabilities.
+Its `layers` and `load` are intent, below. [Choose where your data
+goes](/docs/start/choose-a-route) lists the routes and the packs each one
+uses; [Packs and methods](/docs/reference/packs) lists each pack's methods and
+capabilities.
+
+## Intent: what to keep, not how [#intent]
+
+Besides choosing methods, a standard can say how data is loaded, kept and run
+in words that belong to no platform: load only new files, keep the latest row
+per key, add a column saying when a row was loaded, try a failed run twice
+more. That is **intent**. Each pack turns it into its own platform's terms:
+`history: latest` is AUTO CDC on Databricks and `INSERT OR REPLACE` on DuckDB.
+
+Methods are chosen first, as below; intent changes what a chosen method
+writes, never which method is chosen, unless a `when` rule matches on it. A
+value the chosen pack does not apply is a
+[`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) warning, with the pack's
+reason, so a standard never promises what a project does not build. Each pack's
+reference page lists what it does with every value, and the
+[intent reference](/docs/reference/intent) lists the words, where each can be
+written and which value wins.
+
+A [published standard](/docs/how-to/choose-a-published-standard) is a
+standard of this kind for one platform, with a small project that builds.
## How a method is chosen
diff --git a/content/docs/how-to/(build)/add-an-environment.mdx b/content/docs/how-to/(build)/add-an-environment.mdx
index 21a4a7a..8b9f245 100644
--- a/content/docs/how-to/(build)/add-an-environment.mdx
+++ b/content/docs/how-to/(build)/add-an-environment.mdx
@@ -42,8 +42,8 @@ Other platforms add their own file or folder for it; see
Only an environment named `dev` is in development mode; every other one,
`test` included, is in production mode, and schedules run only in `prod` or
-`production`, so `test`'s refresh job is paused. A platform can set `mode` and
-`schedule.paused` itself; see the
+`production`, so `test`'s refresh job is paused. A platform can set
+`databricks.mode` and `schedule.paused` itself; see the
[platform defaults](/docs/reference/packs/databricks#platform-configuration).
Environments may differ only in values, such as names, paths and hosts: a table
built in one environment only is refused as
@@ -72,6 +72,7 @@ For the example above, it prints:
- `mode`: `development` → `production`
- `workspace.root_path`: `/Workspace/Shared/.bundle/finance/dev` → `/Workspace/Shared/.bundle/finance/test`
- `tags.environment`: `dev` → `test`
+- `sections.databricks.workspace.root_path`: `/Workspace/Shared/.bundle/finance/dev` → `/Workspace/Shared/.bundle/finance/test`
## Table journals
@@ -83,7 +84,9 @@ For the example above, it prints:
place, and lists only what differs:
- **Platform:** the platform document's values, and the defaults that follow
- from the environment, such as `mode`. Compared with `prod`, it also shows
+ from the environment, such as `mode`. The values of a pack's own section,
+ such as `databricks:`, show twice: as the pack reads them, such as
+ `workspace.root_path`, and as written, under `sections.databricks`. Compared with `prod`, it also shows
`schedule.paused`, because only `prod` runs its schedule.
- **Names:** the pipeline and job names, which are not per table.
- **Table:** one section for each table that differs: its full name, each
diff --git a/content/docs/how-to/(build)/add-quality-checks.mdx b/content/docs/how-to/(build)/add-quality-checks.mdx
index 626e834..8d609ef 100644
--- a/content/docs/how-to/(build)/add-quality-checks.mdx
+++ b/content/docs/how-to/(build)/add-quality-checks.mdx
@@ -34,6 +34,11 @@ required columns and the default for your own checks:
+`quality: { results: true }` in the standard also keeps the results of the
+checks as a table you can query: on Databricks, the pipeline's event log. It is
+[intent](/docs/reference/intent), so a pack that cannot keep them says so with
+[`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED).
+
diff --git a/content/docs/how-to/(build)/check-in-ci.mdx b/content/docs/how-to/(build)/check-in-ci.mdx
index 8b449e5..3b3a371 100644
--- a/content/docs/how-to/(build)/check-in-ci.mdx
+++ b/content/docs/how-to/(build)/check-in-ci.mdx
@@ -8,7 +8,7 @@ from the definitions project folder on every pull request: it fails when
someone changed the definitions without generating, so the files you deploy
are always what the definitions say. It needs only Node.js 22.13 or later:
each template installs an exact version of the command from npm,
-`npm install -g ingestron@2.2.0`, then runs it. Change the version when you
+`npm install -g ingestron@2.3.0`, then runs it. Change the version when you
upgrade, as
[Upgrade Ingestron and its packs](/docs/how-to/upgrade#choose-the-version-you-run)
shows, so a new release never fails a pull request that changed nothing.
diff --git a/content/docs/how-to/(build)/deploy-through-environments.mdx b/content/docs/how-to/(build)/deploy-through-environments.mdx
index 0952e03..4f34c5f 100644
--- a/content/docs/how-to/(build)/deploy-through-environments.mdx
+++ b/content/docs/how-to/(build)/deploy-through-environments.mdx
@@ -43,7 +43,7 @@ The tools are pinned: Microsoft's `@microsoft/azure-data-factory-utilities`
1.0.3, the Databricks CLI v1.23.0 and sqlcmd (Go) v1.10.0, each download
checked by its SHA-256 where the template downloads it. Each GitHub action is
pinned to a full commit, with its version beside it, and the check installs an
-exact Ingestron, `ingestron@2.2.0`: change them when you upgrade. The ADF
+exact Ingestron, `ingestron@2.3.0`: change them when you upgrade. The ADF
build fetches its build tool from `adf.azure.com`, so the build stage needs
network access there.
diff --git a/content/docs/how-to/(build)/keep-a-table-current.mdx b/content/docs/how-to/(build)/keep-a-table-current.mdx
index c417eb4..34a5bd2 100644
--- a/content/docs/how-to/(build)/keep-a-table-current.mdx
+++ b/content/docs/how-to/(build)/keep-a-table-current.mdx
@@ -4,25 +4,33 @@ description: Keep the current row for each key, or every version, from loads of
---
Use this when each load brings rows that change rows the table already has:
-a key, a column that orders the changes, and sometimes a column that says
-which rows are deletions. The store method's `mode` decides what the table
-keeps. Set it in the flow's `store.with` for every table of the flow, or in a
-table's own `store.with` for that table alone. These modes need the contract to
-mark the key with `primaryKey: true`; see
+a key, a column that orders the changes, and sometimes a condition that says
+which rows are deletions. Three words of [intent](/docs/reference/intent) say
+what the table keeps:
+
+- `history: latest` keeps the current row per key, and `history: full`
+ every version;
+- `sequence` names the column that orders a key's changes;
+- `load: { deletes: { when } }` is a SQL condition that is true for a row
+ deleted at the source.
+
+Write them on a flow for every table of the flow, on a table for that table
+alone, or `history` on a layer of the standard for every table in it. They
+need the contract to mark the key with `primaryKey: true`; see
[Describe a table with a contract](/docs/how-to/write-a-contract#keys-and-required-columns).
+What each engine does with them:
-| Engine | Current row per key | Every version | Deleted keys |
-| ---------- | ------------------- | ------------- | ------------- |
-| DuckDB | `upsert` | No | No |
-| Databricks | `upsert` | `history` | `delete_when` |
-| Synapse | No | No | No |
+
## On DuckDB
`duckdb-table` replaces the table with the files' rows each run, unless the
-table sets another `mode`. To keep one row per order, the latest by
-`ordered_at`, give the orders table in the
-[DuckDB starter](/docs/start/first-project)'s `flows/orders.yaml` this `store`:
+table keeps history. To keep one row per order, the latest by `ordered_at`,
+give the orders table in the [DuckDB starter](/docs/start/first-project)'s
+`flows/orders.yaml` this intent:
```yaml
# flows/orders.yaml
@@ -30,19 +38,19 @@ tables:
orders:
object: orders.csv
contract: shop.orders
- store: { with: { mode: upsert, sequence_by: ordered_at } }
+ history: latest
+ sequence: ordered_at
```
-
-
-Each run then inserts each key's latest row, by `sequence_by` and then by file
-name, and replaces the row the table had for that key. Set `sequence_by` when
-one file can hold a key twice. Running again is safe in every mode. A changed
-primary key or column type needs the table built again: delete
-`ingestron.duckdb` and run again, which reloads everything from the files.
-Every setting is listed under
+Each run then inserts each key's latest row, by `sequence` and then by file
+name, and replaces the row the table had for that key: the method's
+`mode: upsert`. Set `sequence` when one file can hold a key twice. Running
+again is safe in every mode. A changed primary key or column type needs the
+table built again: delete `ingestron.duckdb` and run again, which reloads
+everything from the files. Every setting is listed under
[`duckdb-table`](/docs/reference/packs/duckdb#duckdb-table).
+The method also has a `mode` that no intent stands for, `append`.
`append` adds each file's rows once: it reads each file name once, so a file
re-exported under the same name is skipped and its new rows are not loaded.
Give a re-export a new name, or delete `ingestron.duckdb` and run again to
@@ -54,19 +62,22 @@ cannot be appended; see
## On Databricks
`streaming-table` keeps the current row per key, or every version, with
-`AUTO CDC`. Set its `mode` on the flow or on each table:
+`AUTO CDC`. This flow orders both tables' changes by `changed_at`, keeps the
+current account and every version of each contact, and removes an account
+whose change says `DELETE`:
-
+`history: latest` keeps the current row per key (SCD type 1), and
+`history: full` every version (SCD type 2), with `__START_AT` and `__END_AT`
+columns. `except_columns` is a setting of the method, not intent: the columns
+left out of the kept table, such as the one that says what the change was.
+
+
-`upsert` keeps the current row per key (SCD type 1). `history` keeps every
-version (SCD type 2), with `__START_AT` and `__END_AT` columns. Every setting is
-listed under [`streaming-table`](/docs/reference/packs/databricks#streaming-table).
+Every setting is listed under
+[`streaming-table`](/docs/reference/packs/databricks#streaming-table), with
+the settings that mean the same as this intent.
diff --git a/content/docs/how-to/(build)/start-from-a-template.mdx b/content/docs/how-to/(build)/start-from-a-template.mdx
index c9912fc..c81f06f 100644
--- a/content/docs/how-to/(build)/start-from-a-template.mdx
+++ b/content/docs/how-to/(build)/start-from-a-template.mdx
@@ -4,7 +4,7 @@ description: Start from a connection and a supported route, or from a starter or
---
A project is a folder of YAML definitions with a `project.yaml` at its root.
-Create one in either of two ways:
+Create one in any of three ways:
- **From a connection:** choose the source, how it can be reached, and the
route its data takes. The project holds only that connection, its example
@@ -12,8 +12,12 @@ Create one in either of two ways:
- **From a starter:** copy a complete project for a common situation, such as
[the DuckDB starter](/docs/start/first-project), or one your organisation
shares as a template.
+- **From a published standard:** copy a dated standard for your platform,
+ which says how every table is loaded, kept and run, with a small project
+ that builds; see
+ [Start from a published standard](/docs/how-to/start-from-a-published-standard).
-Either way, the result is ordinary definitions you edit, then plan and apply.
+Each way, the result is ordinary definitions you edit, then plan and apply.
@@ -80,7 +85,7 @@ panel.
light="/images/vscode/new-project-light.png"
dark="/images/vscode/new-project-dark.png"
alt="The New project picker asks How do you want to start? and offers From a connection, to choose a source, how it is reached and a supported route, and From a starter, a ready-made project to try, such as DuckDB on a laptop."
- caption="New project's first question: from a connection or from a starter. The folder comes last."
+ caption="New project's first question: how to start. The folder comes last."
/>
The project holds the connection, its example flows and contracts, and the
diff --git a/content/docs/how-to/(build)/write-a-contract.mdx b/content/docs/how-to/(build)/write-a-contract.mdx
index c430213..cd3df9d 100644
--- a/content/docs/how-to/(build)/write-a-contract.mdx
+++ b/content/docs/how-to/(build)/write-a-contract.mdx
@@ -105,7 +105,7 @@ standard's `quality.default` action:
Other rules compare a count over all rows, such as a row count or a share of
nulls, which a check of one row cannot. So does `unique: true`, which says no
value repeats. Where the pack checks rows, these run after each refresh when
-the platform turns on `manage.checks`. Otherwise, and always for a `custom`
+the platform turns on `databricks.manage.checks`. Otherwise, and always for a `custom`
rule, each `quality` rule is a
[warning](/docs/reference/errors#QUALITY_RULE_NOT_CHECKED) and stays as
documentation.
@@ -114,7 +114,7 @@ documentation.
A table's `tags`, a column's `tags` and its `classification` are kept for
packs that tag tables. Databricks sets them as Unity Catalog tags after each
-refresh when the platform turns on `manage.tags`; see
+refresh when the platform turns on `databricks.manage.tags`; see
[Databricks contract fields](/docs/reference/packs/databricks#contract-fields).
## Draft from a sample [#draft-from-a-sample]
diff --git a/content/docs/how-to/(standards)/ask-the-platform-team.mdx b/content/docs/how-to/(standards)/ask-the-platform-team.mdx
index 1f37692..216be6e 100644
--- a/content/docs/how-to/(standards)/ask-the-platform-team.mdx
+++ b/content/docs/how-to/(standards)/ask-the-platform-team.mdx
@@ -27,7 +27,7 @@ Each section appears only when it has something to say, in this order:
where it is used, whether it is set, and the name the deploying pipeline
supplies it as; never its value. See [Values to provide](#values-to-provide).
- The identity pipelines and jobs run as, and the workspace.
-- **Switched on:** each `manage` switch the platform turns on, what it does
+- **Switched on:** each `databricks.manage` switch the platform turns on, what it does
and what it needs on the platform, from the pack's
[platform prerequisites](/docs/reference/packs/databricks#platform-prerequisites).
- **Used by the definitions, but switched off:** features the definitions use
@@ -67,7 +67,8 @@ Its request for `dev` is:
-A platform document that names `run_as` and turns on `manage` switches adds
+A platform document whose `databricks:` section names `run_as` and turns on
+`manage` switches adds
what each switch needs and the SQL to run once; see
[Before you deploy](/docs/platforms/databricks-deploy#prerequisites) in Deploy the Databricks bundle.
diff --git a/content/docs/how-to/(standards)/choose-a-published-standard.mdx b/content/docs/how-to/(standards)/choose-a-published-standard.mdx
new file mode 100644
index 0000000..4b11674
--- /dev/null
+++ b/content/docs/how-to/(standards)/choose-a-published-standard.mdx
@@ -0,0 +1,91 @@
+---
+title: Choose a published standard
+description: Pick a dated standard for your platform, see what it decides, and list the ones there are.
+---
+
+A published standard is a [standard](/docs/how-to/configure-a-standard) that
+someone has checked against the platform's documentation and dated, with a
+small project that builds: a connection, flows, contracts and a transform that
+show each of its choices. Start a project from one, and you start from
+choices that are written down and checked, instead of a blank `standard.yaml`.
+The command calls one a kit, as in `--kit`.
+
+These come with the `ingestron` command and the VS Code extension:
+
+
+
+## What one decides
+
+Each published standard is a folder of ordinary definitions:
+
+- `standard.yaml` says how every table is loaded, kept and run, in
+ [intent](/docs/reference/intent): which layers there are and what each
+ keeps, how files are read, the columns that say when and where each row came
+ from, and how a failed run is tried again and who is told.
+- `platform.yaml` and `project.yaml` hold what belongs to your workspace,
+ such as its address and who the jobs run as. What only you can know is
+ `REPLACE_ME`, for you to write.
+- `STANDARD.md`, once it is in your project, explains each choice and the
+ platform page it was checked against.
+
+Each published standard has a page that shows these files, what each
+`REPLACE_ME` stands for, and how each table of its project resolves, such as
+[Streaming lakehouse on Databricks](/docs/how-to/kits/databricks-lakehouse-streaming).
+
+## Versions
+
+A published standard is named `/`, and each version by the
+date it was checked: `duckdb/local-medallion@2026-11-10`. A version never
+changes once it is published; a newer one has a later date and says in its
+notes what changed. A version whose date ends `-preview`, such as
+`2026-12-01-preview`, uses platform features still in preview, and is used
+only when you name it. The others use only generally available features, and
+their standard says `features: ga`.
+
+Each version names the `ingestron` it needs, such as `>=2.3.0`. `init`
+refuses a version that needs a newer one than you have, and names both,
+before it writes anything.
+
+## Choose one
+
+- **Your platform.** A published standard is for the packs it names, such as
+ `databricks@2`. To keep your source's route, its packs must be among them;
+ see [With a connection](/docs/how-to/start-from-a-published-standard#with-a-connection).
+- **Try it first.** [Local medallion on DuckDB](/docs/how-to/kits/duckdb-local-medallion)
+ runs on a laptop with nothing to replace, so you can see how intent becomes
+ SQL before you choose one for a workspace.
+- **What it keeps.** Read its page: each table's intent, and what the pack
+ does with each value on its [reference page](/docs/reference/packs). A value
+ that does not fit your tables can be changed once the project is yours.
+
+## List them
+
+```sh
+ingestron standard list
+```
+
+```text
+databricks/lakehouse-streaming@2026-11-10 Streaming lakehouse on Databricks (ga)
+duckdb/local-medallion@2026-11-10 Local medallion on DuckDB (ga)
+```
+
+Each line is a version, with its title and `(ga)` or `(preview)`. The versions
+installed with `ingestron` come first, and work offline. Then come the
+versions only the public catalogue has, the
+[`ingestron/standards`](https://github.com/ingestron/standards) repository,
+each marked `(online)`: `ingestron` reads its tags with your own `git`. When it
+cannot reach the catalogue, the list starts with
+`Could not reach the catalogue; showing the kits installed with ingestron.`
+`--platform duckdb` keeps one platform's.
+
+Set `INGESTRON_STANDARDS_REPOSITORY` to read another catalogue, such as a
+mirror inside your network. A team's own published standard is read from its
+Git repository instead; see
+[Share a published standard](/docs/how-to/share-a-template#share-a-published-standard).
+
+## Next
+
+- [Start from a published standard](/docs/how-to/start-from-a-published-standard):
+ create the project and write what it leaves to you.
+- [Move to a newer published standard](/docs/how-to/upgrade-a-standard):
+ replace a project's standard with a later version.
diff --git a/content/docs/how-to/(standards)/configure-a-standard.mdx b/content/docs/how-to/(standards)/configure-a-standard.mdx
index 0130898..d18c496 100644
--- a/content/docs/how-to/(standards)/configure-a-standard.mdx
+++ b/content/docs/how-to/(standards)/configure-a-standard.mdx
@@ -19,6 +19,10 @@ To make every project follow your rules:
4. Answer each project's [platform request](/docs/how-to/ask-the-platform-team),
and [upgrade](/docs/how-to/upgrade) on purpose.
+To start from a standard that is already written and checked for your
+platform, [choose a published standard](/docs/how-to/choose-a-published-standard):
+once a project has it, it is your copy, and this page applies to it.
+
## What a standard holds [#what-a-standard-holds]
This is Acme's standard. Its naming, types and storage choices belong to
@@ -51,6 +55,14 @@ set what happens to rows that fail a check (`quality`), and require
[Which setting wins](/docs/concepts/settings) explains what happens when a project
sets one too.
+It can also say how tables are loaded, kept and run, in
+[intent](/docs/reference/intent) that every pack reads: what each layer keeps
+(`history`), columns that record when and where each row came from (`audit`),
+how files and rows are loaded (`load`), how runs are retried and who is told
+(`operations`), and whether platform features in preview may be used
+(`features`). The [published standards](/docs/how-to/choose-a-published-standard)
+are complete examples.
+
## One standard for several packs
A project has exactly one standard, even when it uses several packs. Each
@@ -161,8 +173,8 @@ To take it from a Git repository instead, see
-**Ingestron: New project** keeps the starter's or route's own standard. To
-start from yours, run `ingestron init` with `--standard` in a terminal, or
+**Ingestron: New project** keeps the starter's or route's own standard, or
+takes a published one with **From a published standard**. To start from yours, run `ingestron init` with `--standard` in a terminal, or
replace the new project's `standard.yaml` with your standard and plan.
diff --git a/content/docs/how-to/(standards)/kits/databricks-lakehouse-streaming.mdx b/content/docs/how-to/(standards)/kits/databricks-lakehouse-streaming.mdx
new file mode 100644
index 0000000..6e00497
--- /dev/null
+++ b/content/docs/how-to/(standards)/kits/databricks-lakehouse-streaming.mdx
@@ -0,0 +1,62 @@
+---
+title: Streaming lakehouse on Databricks
+description: A published standard for Databricks. Streaming tables that read each new file once, silver kept to the latest row per key, gold materialised views, and a refresh job retried and alerted on failure.
+---
+
+
+
+A web shop's files become three layers in Databricks: `bronze.customers`, a row
+each time a customer is exported, with the values that do not fit kept aside;
+`silver.orders`, the latest row per order, from change records; and
+`gold.daily_sales`, totals per customer and day. One Lakeflow pipeline builds
+them, and one job refreshes it each day, tries again when it fails, and mails
+your team when it still fails.
+
+## Start from it
+
+From the folder that will hold the project:
+
+```sh
+ingestron init shop --kit databricks/lakehouse-streaming@2026-11-10
+```
+
+Then write each value that is still `REPLACE_ME`, below, and plan; see
+[Start from a published standard](/docs/how-to/start-from-a-published-standard)
+and [Deploy the Databricks bundle](/docs/platforms/databricks-deploy).
+
+## What to write [#replace-me]
+
+What belongs to your workspace is `REPLACE_ME`. Point at each one (or tap it
+on a touch screen) to see the field it fills, where to find the value, and a
+suggestion; in VS Code, hover it. The platform holds who is told of failures
+and the `databricks:` section:
+
+
+
+`project.yaml` holds what differs between `dev` and `prod`: each workspace's
+address, and the volume the shop's files land in:
+
+
+
+## The standard
+
+
+
+Each table starts from its layer's intent. `customers` lands in `bronze`, the
+first layer; the orders flow says `layer: silver`, and gives the column that
+orders its changes and the condition for a delete. Each value, and where it
+came from:
+
+
+
+What the Databricks pack does with each value, with each feature's release
+level and the date it was checked, is in
+[the Databricks pack's intent](/docs/reference/packs/databricks#intent).
+`features: ga` keeps out features in preview, so the job alerts on failure
+but not on slow runs: Databricks' run duration alert is in Public Preview.
+
+## Why each choice
+
+In your project, `STANDARD.md` explains each section of the standard on
+Databricks, and links the Databricks pages each was checked against, with the
+date.
diff --git a/content/docs/how-to/(standards)/kits/duckdb-local-medallion.mdx b/content/docs/how-to/(standards)/kits/duckdb-local-medallion.mdx
new file mode 100644
index 0000000..b0062f6
--- /dev/null
+++ b/content/docs/how-to/(standards)/kits/duckdb-local-medallion.mdx
@@ -0,0 +1,49 @@
+---
+title: Local medallion on DuckDB
+description: A published standard for DuckDB on a laptop. Bronze loaded in full from CSV files, silver kept to the latest row per key, and nothing to replace.
+---
+
+
+
+Sample orders become two layers in one DuckDB file: `bronze.orders`, the rows
+as they arrive in the file, and `silver.customer_totals`, one row per customer.
+Both are published as Parquet. Nothing is `REPLACE_ME`: the project runs as
+written, so it is the published standard to try first.
+
+## Start from it
+
+From the folder that will hold the project:
+
+```sh
+ingestron init shop --kit duckdb/local-medallion@2026-11-10
+```
+
+Then run it as the new `README.md` says; see
+[Start from a published standard](/docs/how-to/start-from-a-published-standard)
+and [DuckDB](/docs/platforms/duckdb).
+
+## The standard
+
+
+
+Each table starts from its layer's intent. The flow's table lands in `bronze`,
+the first layer, and the transform's in `silver`. Neither flow nor transform
+writes intent of its own, so each value comes from the standard, or from the
+DuckDB pack's default:
+
+
+
+What the DuckDB pack does with each value, and with every other, is in
+[the DuckDB pack's intent](/docs/reference/packs/duckdb#intent).
+
+## The platform
+
+Nothing in it is yours to replace. `published` is the folder each table is
+written to as Parquet:
+
+
+
+## Why each choice
+
+In your project, `STANDARD.md` explains each section of the standard, and
+links the DuckDB pages each was checked against, with the date.
diff --git a/content/docs/how-to/(standards)/kits/meta.json b/content/docs/how-to/(standards)/kits/meta.json
new file mode 100644
index 0000000..cfdca50
--- /dev/null
+++ b/content/docs/how-to/(standards)/kits/meta.json
@@ -0,0 +1,4 @@
+{
+ "title": "Published standards",
+ "pages": ["duckdb-local-medallion", "databricks-lakehouse-streaming"]
+}
diff --git a/content/docs/how-to/(standards)/meta.json b/content/docs/how-to/(standards)/meta.json
index d141e65..6b09591 100644
--- a/content/docs/how-to/(standards)/meta.json
+++ b/content/docs/how-to/(standards)/meta.json
@@ -1,9 +1,13 @@
{
"title": "Standards and teams",
- "description": "One standard for every project, shared from Git.",
+ "description": "One standard for every project: your own, or a published one, shared from Git.",
"root": true,
"pages": [
"configure-a-standard",
+ "choose-a-published-standard",
+ "start-from-a-published-standard",
+ "upgrade-a-standard",
+ "kits",
"name-things-your-way",
"ask-the-platform-team",
"share-a-template",
diff --git a/content/docs/how-to/(standards)/share-a-template.mdx b/content/docs/how-to/(standards)/share-a-template.mdx
index 12dffb8..1503f12 100644
--- a/content/docs/how-to/(standards)/share-a-template.mdx
+++ b/content/docs/how-to/(standards)/share-a-template.mdx
@@ -1,6 +1,6 @@
---
title: Share templates and standards
-description: Keep your team's project templates and your organisation's standard in Git, and start projects from them.
+description: Keep your team's project templates, your organisation's standard and your own published standards in Git, and start projects from them.
---
A template is a project folder your team starts from, and a standard is the
@@ -61,16 +61,59 @@ says how to give a project a newer version, and how to keep projects to it.
**Ingestron: New project** in VS Code keeps the template's own standard; use
the command for `--standard`.
+## Share a published standard [#share-a-published-standard]
+
+To give teams a dated standard with an example that builds, as Ingestron's own
+[published standards](/docs/how-to/choose-a-published-standard) are, make a
+folder `///` that holds a project, and a `kit.yaml`
+beside its files:
+
+- `standard.yaml`, with `origin: { standard: /, version: }`
+ and the same `packs` as `kit.yaml`;
+- `platform.yaml` and `project.yaml`, with `REPLACE_ME` and a comment for
+ each value that is the team's;
+- `README.md`, which explains each choice, and becomes the project's
+ `STANDARD.md`;
+- the connections, flows, contracts and transforms of a small example.
+
+This is the `kit.yaml` of the local medallion:
+
+
+
+`id` is `/`, its platform the first pack's id; `version` is
+the date it was checked, `YYYY-MM-DD`, or `YYYY-MM-DD-preview` with
+`features: preview`. `requires` names the `ingestron` versions it needs, as
+`>=`, `^` or `~` and a version; `replaces` names the version before it, and
+`notes` says what changed, which `standard install` prints. Any other key is
+refused, so a mistake shows at once.
+
+Commit the folder, and tag the commit `/@`. Teams
+start from it with the repository's URL, the folder after `#`, and the tag
+after `@`:
+
+```sh
+ingestron init orders \
+ --kit https://github.com/acme/data-platform.git#kits/databricks/acme-lakehouse/2026-11-10@databricks/acme-lakehouse@2026-11-10
+```
+
+`ingestron standard install` takes the same form. A version never changes once
+tagged: publish a change as a new version, in a new folder. A published
+standard from a Git URL is never listed by `standard list`.
+
## What Ingestron does with a Git URL
-Fetching a template or standard is the only time Ingestron uses the network.
+Fetching a template, a standard or a published standard is the only time
+Ingestron uses the network, besides `ingestron standard list` reading the
+catalogue's tags.
- Cloning uses your own `git`, so your Git credentials and settings apply. It
- takes only the latest commit of the default branch, and runs none of the repository's hooks or
+ takes only the latest commit of the default branch, or of the tag for a
+ published standard, and runs none of the repository's hooks or
submodules; filters from your own git settings, such as Git LFS, still
- apply.
+ apply. For a published standard, Git never asks for a password: it uses
+ the credentials Git already has.
- The part after `#` must stay inside the repository, and must be a folder
- for a template or a regular file for a standard.
-- It never runs anything from a template or standard.
+ for a template or a published standard, or a regular file for a standard.
+- It never runs anything from a template, a standard or a published standard.
- A user name or token in a URL is never printed or written into the
project.
diff --git a/content/docs/how-to/(standards)/start-from-a-published-standard.mdx b/content/docs/how-to/(standards)/start-from-a-published-standard.mdx
new file mode 100644
index 0000000..1a6d997
--- /dev/null
+++ b/content/docs/how-to/(standards)/start-from-a-published-standard.mdx
@@ -0,0 +1,137 @@
+---
+title: Start from a published standard
+description: Create a project from a published standard, write the values it leaves to you, and plan.
+---
+
+A [published standard](/docs/how-to/choose-a-published-standard) gives a new
+project its standard, its platform skeleton and a small example that builds.
+This page creates one on your laptop with
+[Local medallion on DuckDB](/docs/how-to/kits/duckdb-local-medallion), then
+shows what changes for a workspace, with
+[Streaming lakehouse on Databricks](/docs/how-to/kits/databricks-lakehouse-streaming).
+You need the `ingestron` command or the VS Code extension; see
+[Install](/docs/start/install).
+
+## Create the project
+
+
+
+
+From the folder that will hold the project, name the published standard and
+its version with `--kit`:
+
+```sh
+ingestron init shop --kit duckdb/local-medallion@2026-11-10
+```
+
+Without `@`, `--kit duckdb/local-medallion` takes the newest version
+that is not a preview. `--kit` replaces `--template` and `--standard`: give
+one of the three.
+
+
+
+
+Run **Ingestron: New project** from the Command Palette (⇧⌘P on macOS,
+Ctrl+Shift+P elsewhere), choose **From a published standard**, then
+**Local medallion on DuckDB**, then a folder named `shop`. The new
+`README.md` opens in a preview, and the plan is in the Ingestron output panel.
+
+
+
+
+Ingestron copies the published standard's files into `shop`, checks every
+environment, and plans:
+
+```text
+Created shop from the kit duckdb/local-medallion@2026-11-10 (13 files).
+
+duckdb → shop/build/duckdb (dev)
+create dev/README.md
+create dev/run.sql
+create dev/tables/bronze.orders.sql
+create dev/tables/silver.customer_totals.sql
+
+4 files: 4 create. Ready to apply.
+Nothing to replace: the project runs as written.
+```
+
+The project holds the standard's files as they are, except two:
+
+- `STANDARD.md` is the published standard's own README: what each section of
+ `standard.yaml` does on DuckDB, and the DuckDB pages it was checked
+ against.
+- `README.md` is the project's, as for every new project: how to run it, and
+ the values still to write.
+
+`standard.yaml` names its `origin`, the published standard and version it
+came from, so a later version can be compared with it; see
+[Move to a newer published standard](/docs/how-to/upgrade-a-standard).
+
+Run it as its `README.md` says: `ingestron apply`, then the DuckDB command
+line; see [DuckDB](/docs/platforms/duckdb).
+
+## Write the values it leaves to you [#write-the-values]
+
+A published standard for a workspace cannot know your workspace. Each value
+that is yours is `REPLACE_ME`, with a comment above it. These are the
+platform's, in the streaming lakehouse; point at a `REPLACE_ME` (or tap it on
+a touch screen) to see what to write:
+
+
+
+In VS Code, hover a `REPLACE_ME` to see the same: the field it fills, where to
+find the value, and a suggestion. The project's `README.md` lists every value
+still to write, with its file and line. `ingestron plan` warns about each one,
+and `ingestron check` fails until each is written; see
+[`REPLACE_ME`](/docs/reference/errors#REPLACE_ME).
+
+To write some as you create the project, give each with `--set =`,
+or `--set .=` for one environment:
+
+```sh
+ingestron init shop --kit databricks/lakehouse-streaming@2026-11-10 \
+ --set dev.host=https://adb-1111111111111111.1.azuredatabricks.net \
+ --set service_principal_name=7f3c2a10-0000-4000-8000-000000000001
+```
+
+Each published standard's page lists all of its values, in every file.
+
+## With a connection [#with-a-connection]
+
+To start from your own source with a published standard, add the connection
+options, as [Create a project](/docs/how-to/start-from-a-template#start-from-a-connection)
+describes:
+
+```sh
+ingestron init sales --kit databricks/lakehouse-streaming@2026-11-10 \
+ --connector sqlserver --connectivity public --via databricks --yes
+```
+
+The published standard gives `standard.yaml`, `project.yaml`, `platform.yaml`
+and `STANDARD.md`; the route gives the connection, its flows, contracts and
+transforms. Where both write a key of `project.yaml` or `platform.yaml`, the
+published standard's value stays. Every pack the route needs must be one the
+published standard names: `--via adf-databricks` needs `adf@2` too, so this
+one refuses it.
+
+Then read the plan. A value of the standard that the route's methods do not
+apply is a [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) warning with
+the reason. The streaming lakehouse reads files, and Lakeflow Connect reads
+SQL Server itself, so the plan begins:
+
+```text
+dev, prod: warnings
+standard.yaml:22:12 warning NOT_APPLIED
+ load.mode: files isn't applied on databricks: Managed ingestion reads the source, not files.
+```
+
+Keep the warnings you accept, or change the standard where it does not fit
+your source: the project is yours.
+
+## Make it yours
+
+Once created, the project is ordinary definitions: change the contracts and
+flows to your source's, and the standard where your team decides otherwise.
+Run `ingestron plan` after each change. To take a later version of the
+published standard, see
+[Move to a newer published standard](/docs/how-to/upgrade-a-standard).
diff --git a/content/docs/how-to/(standards)/upgrade-a-standard.mdx b/content/docs/how-to/(standards)/upgrade-a-standard.mdx
new file mode 100644
index 0000000..328bc99
--- /dev/null
+++ b/content/docs/how-to/(standards)/upgrade-a-standard.mdx
@@ -0,0 +1,110 @@
+---
+title: Move to a newer published standard
+description: Replace a project's standard with a later version of its published standard, see what changes with plan and git diff, and keep your own edits.
+---
+
+A published standard never changes once it is published: a newer version has
+a later date. To move a project to it, replace the project's standard in a
+pull request, read what the plan and `git diff` show, and put back the edits
+your team made. Only `standard.yaml` is replaced; the rest of the project is
+yours and stays as it is.
+
+Upgrading `ingestron` itself is a separate step; see
+[Upgrade Ingestron and its packs](/docs/how-to/upgrade).
+
+## Find the version
+
+`ingestron standard list` lists the versions there are; see
+[List them](/docs/how-to/choose-a-published-standard#list-them). The
+project's `standard.yaml` names the one it has, in its `origin`:
+
+
+
+`ingestron check` never says a newer version exists, and makes no network
+call: moving is your choice.
+
+## Install it
+
+1. On a new branch, commit any change to the standard first. `install` refuses
+ to replace a file with changes that are not committed, so that `git diff`
+ shows only the new version:
+
+ ```text
+ standard.yaml has changes that are not committed; commit them first so git diff shows only the new standard.
+ ```
+
+2. From the project's folder, name the published standard and the version to
+ move to, such as `duckdb/local-medallion@2026-11-10`, or the name alone for
+ the newest version that is not a preview:
+
+ ```sh
+ ingestron standard install duckdb/local-medallion@2026-11-10
+ ```
+
+ It writes the new version's `standard.yaml` in place of the project's,
+ with the project's own `id` and at the same path, even when the standard
+ is in another file, such as `standards/acme.yaml`. Its comments come with
+ it. In a project created from that same version, it prints:
+
+ ```text
+ Replaced standard.yaml with duckdb/local-medallion@2026-11-10 (was 2026-11-10).
+
+ First version.
+
+ Not applied: no other kit file changed since 2026-11-10.
+
+ Next: run ingestron plan, then review git diff.
+ ```
+
+ After `Replaced`, the new version's notes say what changed. **Not
+ applied** lists the published standard's other files that changed since
+ the project's version, such as `platform.yaml` or a contract: `install`
+ does not copy them, because they are yours now. Compare each with yours.
+ When the project's `origin` names no version it can find, it says the
+ earlier version is not available, and you compare by hand. If the project
+ no longer builds with the new standard, the problems follow, as `plan`
+ prints them; the new file stays written.
+
+## Review the change
+
+3. Plan, to see every generated file the new standard changes:
+
+ ```sh
+ ingestron plan
+ ```
+
+ Review each `update`, `create` and `delete` as you would a change to the
+ definitions. A value the new standard sets that a pack does not apply is a
+ [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED) warning, with the
+ reason.
+
+4. Read the standard's own change:
+
+ ```sh
+ git diff standard.yaml
+ ```
+
+ Lines the new version adds, and lines your team had changed, both show
+ here. The new file has the published version's text, so an edit your team
+ made to `standard.yaml`, such as another catalog name, shows as a removed
+ line.
+
+5. Put back each edit you want to keep, in `standard.yaml`, and plan again.
+ For each file **Not applied** listed, take what you want from the new
+ version into yours.
+
+6. Apply, and commit the standard with the generated files:
+
+ ```sh
+ ingestron apply
+ ```
+
+ `ingestron check` in CI then passes on the pull request, because the
+ committed files are what the new standard generates.
+
+## Next
+
+- [Set your organisation's standard](/docs/how-to/configure-a-standard): keep
+ the project to the standard in CI and in review.
+- [Intent](/docs/reference/intent): what each value of a standard means, and
+ what each pack does with it.
diff --git a/content/docs/how-to/(standards)/upgrade.mdx b/content/docs/how-to/(standards)/upgrade.mdx
index 4a14349..82f40d0 100644
--- a/content/docs/how-to/(standards)/upgrade.mdx
+++ b/content/docs/how-to/(standards)/upgrade.mdx
@@ -1,6 +1,6 @@
---
title: Upgrade Ingestron and its packs
-description: Choose the version every project runs, see what an upgrade changes with plan, move CI and your team together, and upgrade from 1.x to 2.0.
+description: Choose the version every project runs, see what an upgrade changes with plan, move CI and your team together, and what changed from 1.x to 2.3.
---
The packs come inside the `ingestron` command: each release of the command
@@ -46,7 +46,7 @@ choose to upgrade. The [CI templates](/docs/how-to/check-in-ci) and the
[deployment templates](/docs/how-to/deploy-through-environments) do:
```sh
-npm install -g ingestron@2.2.0
+npm install -g ingestron@2.3.0
```
`npm ls -g ingestron` shows the version a computer has installed.
@@ -95,7 +95,7 @@ next; see [Deploy through environments](/docs/how-to/deploy-through-environments
### The standard and the folders
-1. Install `ingestron@2.2.0` and plan. Every pack is version 2, so a standard that
+1. Install `ingestron@2.3.0` and plan. Every pack is version 2, so a standard that
names `databricks@1`, `adf@1`, `synapse@1` or `duckdb@1` stops with
`PACK_VERSION`, once for all the environments that have it:
@@ -135,7 +135,7 @@ next; see [Deploy through environments](/docs/how-to/deploy-through-environments
[`DEPLOY_VALUE`](/docs/reference/errors#DEPLOY_VALUE): write it in
`project.yaml` for each environment. See
[Supplied when deploying](/docs/concepts/yaml-references#supplied-when-deploying).
-5. Change CI: the check templates install `ingestron@2.2.0`, and the deploy
+5. Change CI: the check templates install `ingestron@2.3.0`, and the deploy
pipelines deploy `build//` by target, parameter file or script, as
[Deploy through environments](/docs/how-to/deploy-through-environments)
shows.
@@ -151,28 +151,29 @@ given `--env`, they exit with `2`. `compare` and `request` still take `--env`.
resources in place, if the same identity deploys it. Run
`databricks bundle validate` and `databricks bundle summary` first.
- **Deploy as the identity that deployed 1.x.** Without
- `workspace.root_path` in the platform, a target's deployment state is in the
+ `databricks.workspace.root_path` in the platform, a target's deployment state is in the
deploying identity's own folder, `~/.bundle/…` in the workspace.
Another identity, such as the service principal of the new CI templates,
finds no state there and creates a second copy of every pipeline and job; in
`prod`, both copies run on schedule. `databricks bundle summary` shows the
folder a target uses. So run the first 2.0 deploy, and the schema steps
below, as the identity that deployed 1.x. Before deploys move to CI, set a
- shared `workspace.root_path` in `platform.yaml`, such as
+ shared `databricks.workspace.root_path` in `platform.yaml`, such as
`/Workspace/Shared/.bundle//${environment.name}`, and bind each
existing resource to it as the CI identity, as
[Deploy as one identity](/docs/platforms/databricks-deploy#one-identity)
shows.
- Only an environment named `dev` is in development mode; `test`, `uat`,
`staging` and every other environment are in production mode, unless the
- platform sets `mode`. On such an environment's first 2.0 deploy, jobs and
+ platform sets `databricks.mode`. On such an environment's first 2.0 deploy, jobs and
pipelines lose the `[dev ]` prefix from their names, pipelines
leave development, the deployment lock comes on, and production mode checks
the deploying identity's `run_as`, `permissions` and `root_path`. To keep
- development mode there, write `mode: ${values.mode}` in the platform,
+ development mode there, write `mode: ${values.mode}` in the platform's
+ `databricks:` section,
`mode: production` in `project.yaml`'s `values`, and `mode: development` in
that environment's `values`.
-- With `manage.schemas`, managed schemas are keyed by layer, such as `bronze`,
+- With `databricks.manage.schemas`, managed schemas are keyed by layer, such as `bronze`,
or by schema name when a layer has several, not `_`. Each
environment's deployment state still holds the old key, so before the first
2.0 deploy, for each environment and managed schema, as the identity that
@@ -351,6 +352,56 @@ extension works as before. Only code that calls the packages changes:
[`@ingestron/authoring` README](https://github.com/ingestron/ingestron/tree/main/packages/authoring)
shows an example.
+## Upgrade to 2.3 [#upgrade-to-2-3]
+
+2.3 generates the same files as 2.2 for a project that changes nothing. It
+adds [intent](/docs/reference/intent) and
+[published standards](/docs/how-to/choose-a-published-standard), and asks for
+some settings to be written in their new places:
+
+- **Settings that are intent.** `streaming-table`'s `mode`, `sequence_by`,
+ `delete_when` and `cluster_by`, `duckdb-table`'s `mode` and `sequence_by`,
+ and `adf-copy`'s `mode` and `watermark_column` mean the same as
+ [intent](/docs/reference/intent#settings-that-are-intent), such as
+ `history: latest` for `mode: upsert`. They still work, and `plan` reports
+ each as [`SETTING_MOVED`](/docs/reference/errors#SETTING_MOVED), a warning.
+ In a flow or a table, its fix writes the intent and removes the setting;
+ in the standard, write the intent by hand.
+- **The `databricks:` section.** The platform's `workspace`, `mode`, `run_as`,
+ `compute`, `permissions`, `manage` and `grants` now go under `databricks:`,
+ and `notifications.on_failure` becomes an email alert in `alerts`. At the
+ top they still work, with a
+ [`PLATFORM_FIELD_MOVED`](/docs/reference/errors#PLATFORM_FIELD_MOVED)
+ warning whose fix moves each; one written in both places with different
+ values is [`BAD_SETTING`](/docs/reference/errors#BAD_SETTING). See
+ [the `databricks:` section](/docs/reference/packs/databricks#platform).
+- **What each pack applies.** A value of intent the table's pack does not
+ apply is [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED), and with the
+ standard's `features: ga`, one that needs a feature in preview is
+ [`PREVIEW_FEATURE`](/docs/reference/errors#PREVIEW_FEATURE). Projects
+ written before 2.3 declare no intent, so they get neither.
+- **Published standards.** `ingestron init --kit` starts a project from one,
+ `ingestron standard install` moves a project to another version, and
+ `ingestron standard list` lists them; in VS Code, **New project** offers
+ **From a published standard**. Hover a `REPLACE_ME` to see where to find its
+ value.
+
+To move a project to the new places, plan with 2.3.0, apply each fix (in
+VS Code, **Fix all** applies one kind at once), and plan again: the plan should
+list no change to the generated files.
+
+For code that calls the packages:
+
+- `@ingestron/definitions` 1.13 adds intent: `intentPaths`, `intentUseOf` and
+ `intentCoverageGaps`; a pack's `intent`, and a method's `intent`,
+ `intentDefaults` and `aliases`; each table's `intent` and `intentOrigins`;
+ a pack's `platformSection` and `platformSectionReference`; and `find` and
+ `suggest` on each `FieldInfo`.
+- `@ingestron/core` 1.1 adds published standards: `installedKits`,
+ `readKit`, `parseKitSpec` and `kitSchema`.
+- `@ingestron/authoring` 2.3 adds `ProjectSnapshot.replaceMeHelp`, what a
+ `REPLACE_ME` stands for, and lists a table's intent in its hover.
+
## What changed in a release
Recent releases are commits whose message starts with the version, such as
diff --git a/content/docs/index.mdx b/content/docs/index.mdx
index d76814e..12d20ed 100644
--- a/content/docs/index.mdx
+++ b/content/docs/index.mdx
@@ -118,7 +118,9 @@ The platform team writes one standard: the packs, names, layers, types,
default settings, settings no project may change, and what every data product
must declare. VS Code marks a break as you type, `ingestron plan` reports
every one, and `ingestron check` fails the CI build. See
-[Set your organisation's standard](/docs/how-to/configure-a-standard).
+[Set your organisation's standard](/docs/how-to/configure-a-standard). Or
+start from a [published standard](/docs/how-to/choose-a-published-standard):
+one written for your platform, checked against its documentation and dated.
### Reuse your team's work
@@ -141,8 +143,10 @@ each one and what runs after each load.
## What Ingestron does not do
-- It doesn't connect to your sources or your cloud. Its only network call is
- to fetch a template or standard from a Git URL you give it.
+- It doesn't connect to your sources or your cloud. Its only network calls
+ use your own `git`: to fetch a template, a standard or a published standard
+ from a Git URL you give it, or a published standard that is not installed,
+ and to list the published standards' versions.
- It doesn't deploy anything. Your platform's tools do, from your CI/CD
pipeline.
- It doesn't read secret values. A `${secret:…}` reference carries only the
diff --git a/content/docs/platforms/adf-pipelines.mdx b/content/docs/platforms/adf-pipelines.mdx
index efc92f9..cd75202 100644
--- a/content/docs/platforms/adf-pipelines.mdx
+++ b/content/docs/platforms/adf-pipelines.mdx
@@ -85,6 +85,11 @@ platform's `schedule.paused`; see the
+The standard's [`operations`](/docs/reference/intent#operations) gives every
+ADF table that sets neither its `retries`, from `retries.count`, and its Copy
+activity's timeout, from `timeout`; see what the
+[ADF pack does with each intent value](/docs/reference/packs/adf#intent).
+
To stop only the copies, for example while the source database is down for
maintenance, set `paused: true` under `settings.adf-copy` in the platform,
beside the method's other settings, or under the method your flows use.
@@ -171,6 +176,11 @@ run values ADF fills in when it runs, or text:
+The `audit` [intent](/docs/reference/intent) adds the same kind of column in
+words every pack reads, such as `audit: [loaded_at, run_id]` on a flow or a
+layer: ADF adds those its copy can fill, each named `_`, and leaves the
+file values, such as `source_file`, to the pack that reads the files.
+
-The platform names the shared infrastructure the copies use, and the flow
-names the tables to copy from the `erp` connection, which reaches the ERP
-privately with its password in Key Vault:
+The platform names the shared infrastructure the copies use. The flow names
+the tables to copy from the `erp` connection, which reaches the ERP privately
+with its password in Key Vault, and the column each copy reads changed rows
+by, its `load: { cursor }`. The standard copies changed rows for every table,
+`load: { default: incremental }`, and its `bronze` layer keeps the latest row
+of each key, `history: latest`, in the order of the flow's `sequence`:
@@ -227,9 +230,10 @@ source its own landing folder.
Native table copies write a full selection each window. On the
`adf-databricks` route, repeated snapshots append repeated rows with the
-default store; choose [upsert or history](/docs/how-to/keep-a-table-current)
+default store; choose [`history: latest` or `full`](/docs/how-to/keep-a-table-current)
for current rows or change history. SQL-family copies, as in this starter,
-can copy only the rows changed in each window (`mode: incremental`). An
+can copy only the rows changed in each window: `load: incremental`, by the
+flow's `load: { cursor }`. An
unknown setting stops generation with a diagnostic, and the standard's fixed
settings still win.
diff --git a/content/docs/platforms/databricks-deploy.mdx b/content/docs/platforms/databricks-deploy.mdx
index 240bc8e..3a04739 100644
--- a/content/docs/platforms/databricks-deploy.mdx
+++ b/content/docs/platforms/databricks-deploy.mdx
@@ -16,7 +16,7 @@ v0.264.0 or later, which applies preset tags to pipelines.
-With `manage` switches on, the request also says what each needs and gives the
+With `databricks.manage` switches on, the request also says what each needs and gives the
SQL to run once. See [Ask the platform team](/docs/how-to/ask-the-platform-team)
for the whole request.
@@ -39,8 +39,8 @@ not start the ADF copies.
### Deploy as one identity [#one-identity]
A target's deployment state, which records the pipelines and jobs it created,
-is kept in the target's `root_path`. Without `workspace.root_path` in the
-platform, that is the deploying identity's own folder,
+is kept in the target's `root_path`. Without `databricks.workspace.root_path`
+in the platform, that is the deploying identity's own folder,
`~/.bundle//` in the workspace. Another identity
deploying the same target finds no state there and creates a second copy of
every pipeline and job; in `prod`, both copies then run on their schedule.
@@ -49,7 +49,7 @@ So deploy each environment as one identity, such as the service principal your
pipeline uses, and give every target a shared `root_path` in `platform.yaml`
before a second identity deploys it, as the `environments` example does:
-
+
A target that is already deployed under another `root_path` has its state
there. Before the first deployment with the new path, bind each of its
diff --git a/content/docs/platforms/databricks.mdx b/content/docs/platforms/databricks.mdx
index 125f4c9..cda6c38 100644
--- a/content/docs/platforms/databricks.mdx
+++ b/content/docs/platforms/databricks.mdx
@@ -177,9 +177,9 @@ instead, each one names its file, line and fix; see
## Settings that shape it [#settings]
-The platform document decides the targets: `workspace`, `run_as`,
-`permissions` and `mode` go into each target, and `tags` that differ become
-the target's preset tags. The defaults:
+The platform document decides the targets: the `workspace`, `run_as`,
+`permissions` and `mode` of its `databricks:` section go into each target, and
+`tags` that differ become the target's preset tags. The defaults:
@@ -206,7 +206,7 @@ before the first deploy. They are not deployed:
-Only set `manage.schemas` when no other project uses the schemas: deploying
+Only set `databricks.manage.schemas` when no other project uses the schemas: deploying
the bundle can change them, and `databricks bundle destroy` deletes them.
### Refresh on a schedule [#schedule]
@@ -232,6 +232,13 @@ platform's `schedule` stops both the job and the Data Factory triggers; to
stop only the copies, see
[Retry and pause](/docs/platforms/adf-pipelines#retry-and-pause).
+The standard's [`operations`](/docs/reference/intent#operations) says how the
+job runs: how often a failed refresh is tried again, how long a run may take,
+and whether a run that starts while another is running waits. Who is told when
+a run fails is the platform's `alerts`, as in `platform-setup` above. The
+[streaming lakehouse](/docs/how-to/kits/databricks-lakehouse-streaming)
+published standard sets each of them.
+
The tables themselves are shaped by the flows and the standard:
[Keep a table current](/docs/how-to/keep-a-table-current),
[Write a transform](/docs/how-to/write-a-transform) and
@@ -271,4 +278,5 @@ as one identity, and runs the refresh.
- [Add a source and its tables](/docs/how-to/add-a-source): another table, or
another source, in the same project.
- [Databricks pack](/docs/reference/packs/databricks): every setting, the
- platform defaults and the `manage` switches.
+ platform's `databricks:` section and its defaults, the `manage` switches,
+ and what the pack does with each intent value.
diff --git a/content/docs/start/how-it-works.mdx b/content/docs/start/how-it-works.mdx
index c6f9780..c0e01d6 100644
--- a/content/docs/start/how-it-works.mdx
+++ b/content/docs/start/how-it-works.mdx
@@ -108,7 +108,12 @@ from then on; see
- **What it holds:** the packs, and naming patterns, layers and column types.
`settings` gives defaults, `allowed` limits the methods a project may use,
`fixed` holds settings no project may change, and `governance` says what
- every data product must declare.
+ every data product must declare. [Intent](/docs/reference/intent) says how
+ tables are loaded, kept and run, in words every pack reads.
+- **Or a published one.** `ingestron init --kit @`
+ starts a project from a standard written for one platform, checked and
+ dated, with a small project that builds; see
+ [Choose a published standard](/docs/how-to/choose-a-published-standard).
- **The project overrides only what the standard leaves open.** A flow that
changes a fixed setting reports `FIXED_SETTING`, and nothing is generated.
[Which setting wins](/docs/concepts/settings) explains the order.
@@ -125,8 +130,10 @@ from then on; see
## What it never does
-- It doesn't connect to your sources or your cloud. Its only network call is
- to fetch a template or standard from a Git URL you give it.
+- It doesn't connect to your sources or your cloud. Its only network calls
+ use your own `git`: to fetch a template, a standard or a published standard
+ from a Git URL you give it, or a published standard that is not installed,
+ and to list the published standards' versions.
- It doesn't deploy anything. Your platform's tools do.
- It doesn't read secret values. A `${secret:…}` reference carries only the
secret's name; see [YAML references](/docs/concepts/yaml-references#reference-a-secret).
diff --git a/examples/change-data/flows/crm.yaml b/examples/change-data/flows/crm.yaml
index 17aad4b..ead942b 100644
--- a/examples/change-data/flows/crm.yaml
+++ b/examples/change-data/flows/crm.yaml
@@ -4,22 +4,20 @@ id: crm
source: crm
ingest:
with: { format: json }
+sequence: changed_at # the newest change to a key wins
tables:
accounts:
object: accounts
contract: crm.accounts
+ # Only the current version of each account.
+ history: latest
+ load: { deletes: { when: "operation = 'DELETE'" } }
store:
- # Only the current version of each account.
- with:
- {
- mode: upsert,
- sequence_by: changed_at,
- delete_when: "operation = 'DELETE'",
- except_columns: [operation],
- }
+ with: { except_columns: [operation] }
contacts:
object: contacts
contract: crm.contacts
+ # Every version, with __START_AT and __END_AT.
+ history: full
store:
- # Every version, with __START_AT and __END_AT.
- with: { mode: history, sequence_by: changed_at, except_columns: [operation] }
+ with: { except_columns: [operation] }
diff --git a/examples/environments/platform.yaml b/examples/environments/platform.yaml
index 3a21751..c9ff292 100644
--- a/examples/environments/platform.yaml
+++ b/examples/environments/platform.yaml
@@ -3,11 +3,12 @@
apiVersion: ingestron.io/v1
kind: Platform
id: lakehouse
-workspace:
- host: ${values.host}
- root_path: /Workspace/Shared/.bundle/finance/${environment.name}
schedule:
cron: 0 30 6 * * ?
timezone: Europe/London
tags:
environment: ${environment.name}
+databricks:
+ workspace:
+ host: ${values.host}
+ root_path: /Workspace/Shared/.bundle/finance/${environment.name}
diff --git a/examples/layers-and-transforms/README.md b/examples/layers-and-transforms/README.md
index ccf0e1b..110163f 100644
--- a/examples/layers-and-transforms/README.md
+++ b/examples/layers-and-transforms/README.md
@@ -6,8 +6,7 @@
`transforms/store_sales.sql`, where `{{ tables.sales }}` became
`` `bronze`.`sales` ``, in the pipeline's catalog.
- `region_sales` reads `store_sales` and says `layer: gold`.
- The standard's rule for `layer: gold` turns clustering off and adds a
- retention property: compare [gold](build/databricks/src/sales/gold/region_sales.sql)
+ The standard's rule for `layer: gold` adds a retention property: compare [gold](build/databricks/src/sales/gold/region_sales.sql)
with silver.
Transforms are ordered by their inputs, not by file name, and one pipeline
diff --git a/examples/layers-and-transforms/standard.yaml b/examples/layers-and-transforms/standard.yaml
index 969fa7e..5cd445d 100644
--- a/examples/layers-and-transforms/standard.yaml
+++ b/examples/layers-and-transforms/standard.yaml
@@ -4,7 +4,7 @@ id: sales
packs: [databricks@2]
store:
when:
- # Gold tables are small and read whole: no clustering, longer retention.
+ # Gold tables are small and read whole: kept longer, as materialised views.
- layer: gold
use: materialised-view
- with: { cluster_by: none, properties: { delta.logRetentionDuration: "interval 90 days" } }
+ with: { properties: { delta.logRetentionDuration: "interval 90 days" } }
diff --git a/examples/northwind/platform.yaml b/examples/northwind/platform.yaml
index a8042ac..07f6b29 100644
--- a/examples/northwind/platform.yaml
+++ b/examples/northwind/platform.yaml
@@ -1,11 +1,12 @@
apiVersion: ingestron.io/v1
kind: Platform
id: azure
-workspace:
- host: ${values.host}
settings:
adf-copy: # shared infrastructure, referenced by name
landing: ${values.landing}/northwind
key_vault: ${values.key_vault}
integration_runtime: ${values.runtime}
runtime_kind: azure
+databricks:
+ workspace:
+ host: ${values.host}
diff --git a/examples/platform-setup/platform.yaml b/examples/platform-setup/platform.yaml
index 62183d5..06fd545 100644
--- a/examples/platform-setup/platform.yaml
+++ b/examples/platform-setup/platform.yaml
@@ -1,24 +1,25 @@
apiVersion: ingestron.io/v1
kind: Platform
id: hr-lakehouse
-workspace:
- host: https://adb-3000000000000003.3.azuredatabricks.net
-run_as:
- service_principal_name: 0b6c3d2e-0000-4000-8000-000000000007
-compute:
- budget_policy_id: 4e1d0c5b-hr-budget
tags:
cost_centre: "4410"
data_classification: confidential
-permissions:
- - { group_name: hr-data-engineers, level: CAN_MANAGE }
- - { group_name: hr-analysts, level: CAN_VIEW }
schedule:
cron: 0 0 2 * * ?
-notifications:
- on_failure: [hr-data@example.com]
-manage:
- schemas: true # this project owns its schemas
-grants:
- bronze:
- - { principal: hr-data-engineers, privileges: [USE_SCHEMA, SELECT, MODIFY] }
+alerts:
+ - { kind: email, to: [hr-data@example.com], on: [failure] }
+databricks:
+ workspace:
+ host: https://adb-3000000000000003.3.azuredatabricks.net
+ run_as:
+ service_principal_name: 0b6c3d2e-0000-4000-8000-000000000007
+ compute:
+ budget_policy_id: 4e1d0c5b-hr-budget
+ permissions:
+ - { group_name: hr-data-engineers, level: CAN_MANAGE }
+ - { group_name: hr-analysts, level: CAN_VIEW }
+ manage:
+ schemas: true # this project owns its schemas
+ grants:
+ bronze:
+ - { principal: hr-data-engineers, privileges: [USE_SCHEMA, SELECT, MODIFY] }
diff --git a/examples/private-sql-server-filled/flows/erp.yaml b/examples/private-sql-server-filled/flows/erp.yaml
index bafddf9..cce4eb7 100644
--- a/examples/private-sql-server-filled/flows/erp.yaml
+++ b/examples/private-sql-server-filled/flows/erp.yaml
@@ -2,6 +2,10 @@ apiVersion: ingestron.io/v1
kind: Flow
id: erp
source: erp
+# When a row last changed: each copy reads the rows past it,
+# and the tables keep the latest row of each key by it
+load: { cursor: modified_at }
+sequence: modified_at
tables:
customers:
# The source table: its database and schema, database.schema.Customers
@@ -11,7 +15,5 @@ tables:
# The source table: its database and schema, database.schema.Invoices
object: erp.dbo.Invoices
contract: erp.invoices
- ingest:
- with: { watermark_column: updated_at }
- store:
- with: { sequence_by: updated_at }
+ load: { cursor: updated_at }
+ sequence: updated_at
diff --git a/examples/private-sql-server-filled/standard.yaml b/examples/private-sql-server-filled/standard.yaml
index f7236f3..f6c0b6e 100644
--- a/examples/private-sql-server-filled/standard.yaml
+++ b/examples/private-sql-server-filled/standard.yaml
@@ -5,16 +5,17 @@ packs: [databricks@2, adf@2] # Databricks first: the first pack names and types
naming:
# One catalog per environment, such as erp_dev
catalog: erp_{{ environment }}
+layers:
+ # Each window's file holds the rows changed in it; keep the current version.
+ - { name: bronze, history: latest }
+ - silver
+ - gold
+load:
+ default: incremental # copy the rows changed since the last copy, by each flow's cursor
ingest:
settings:
adf-copy:
# the schedule; where it lands is the platform's
- mode: incremental
- watermark_column: modified_at
every: { frequency: Hour, interval: 1 }
# Start of the first window (UTC): midnight on the day this project was created
start: 2026-11-06T00:00:00Z
-store:
- settings:
- # Each window's file holds the rows changed in it; keep the current version.
- streaming-table: { mode: upsert, sequence_by: modified_at }
diff --git a/examples/retail-filled/flows/shop.yaml b/examples/retail-filled/flows/shop.yaml
index 87cadb1..9dd49ba 100644
--- a/examples/retail-filled/flows/shop.yaml
+++ b/examples/retail-filled/flows/shop.yaml
@@ -7,8 +7,11 @@ tables:
orders:
object: orders
contract: shop.orders
+ history: latest
+ sequence: updated_at
+ load: { deletes: { when: "op = 'D'" } }
store:
- with: { mode: upsert, sequence_by: updated_at, delete_when: "op = 'D'", except_columns: [op] }
+ with: { except_columns: [op] }
expectations:
- { name: positive_total, check: total >= 0 }
customers:
diff --git a/examples/retail-filled/platform.yaml b/examples/retail-filled/platform.yaml
index e00553c..ab4bd38 100644
--- a/examples/retail-filled/platform.yaml
+++ b/examples/retail-filled/platform.yaml
@@ -2,20 +2,21 @@
apiVersion: ingestron.io/v1
kind: Platform
id: lakehouse
-workspace:
- host: ${values.host}
-run_as:
- service_principal_name: ${values.run_as}
tags:
team: sales
environment: ${environment.name}
-permissions:
- - group_name: ${values.engineers}
- level: CAN_MANAGE
- - group_name: ${values.analysts}
- level: CAN_VIEW
schedule:
cron: 0 0 5 * * ? # Quartz cron: the refresh job runs once a day
timezone: UTC # the time zone the cron is read in
-notifications:
- on_failure: ["${values.on_failure}"]
+alerts:
+ - { kind: email, to: ["${values.on_failure}"], on: [failure] }
+databricks:
+ workspace:
+ host: ${values.host}
+ run_as:
+ service_principal_name: ${values.run_as}
+ permissions:
+ - group_name: ${values.engineers}
+ level: CAN_MANAGE
+ - group_name: ${values.analysts}
+ level: CAN_VIEW
diff --git a/lib/areas.ts b/lib/areas.ts
index 9a084b8..0eef94d 100644
--- a/lib/areas.ts
+++ b/lib/areas.ts
@@ -47,8 +47,8 @@ export const areaCardSettings: { area: string; icon: string; links: string[] }[]
icon: "standard",
links: [
"/docs/how-to/configure-a-standard",
+ "/docs/how-to/choose-a-published-standard",
"/docs/how-to/ask-the-platform-team",
- "/docs/how-to/upgrade",
],
},
{
diff --git a/lib/connector-page.ts b/lib/connector-page.ts
index c394abf..447f095 100644
--- a/lib/connector-page.ts
+++ b/lib/connector-page.ts
@@ -194,13 +194,13 @@ function reading(id: string, entry: ConnectorEntry): Part {
...part,
title: "Full copies and incremental windows",
before: [
- `\`mode: full\` copies the whole table each ${windows}. \`mode: incremental\` copies rows whose \`watermark_column\` falls within that window. Choose a column that records when a row changes, such as \`modified_at\`; an integer identity is not a time-window watermark.`,
+ `\`load: full\` copies the whole table each ${windows}. \`load: incremental\` copies the rows whose cursor column falls within that window, the column a flow or a table names in \`load: { cursor }\`; the method writes them as its \`mode\` and \`watermark_column\` settings. Choose a column that records when a row changes, such as \`modified_at\`; an integer identity is not a time-window watermark.`,
],
after: [
- "The watermark is configured in the flow's `ingest.with`, or as a default in the standard's `ingest.settings.adf-copy`. It is not a contract property. Each table can override the default with its own `mode` and `watermark_column`; the contract describes the columns and keys of that table.",
+ "Write the load mode once in the standard, as `load: { default: incremental }`, and the cursor on each flow, as `load: { cursor: modified_at }`. Each table can give its own `load`. They are [intent](/docs/reference/intent), not contract properties; the contract describes the columns and keys of that table.",
"The filter is always the trigger window's inclusive start and exclusive end. Ingestron does not track the greatest value seen in an earlier run. A change outside the selected window is not copied by that window. Arbitrary SQL queries, extra `WHERE` criteria and per-table window schedules are not supported by `adf-copy`.",
- "Use the source column name for `watermark_column`. A contract's `physicalName` can rename a selected column, but it does not rewrite the watermark filter.",
- "All tables of one connection and source database share runtime, encryption, schedule, retry and paused settings. Those values must agree even when the tables use different watermarks.",
+ "Use the source column name for the cursor. A contract's `physicalName` can rename a selected column, but it does not rewrite the window filter.",
+ "All tables of one connection and source database share runtime, encryption, schedule, retry and paused settings. Those values must agree even when the tables use different cursors.",
],
};
case "adf-files":
@@ -253,9 +253,9 @@ function current(id: string, entry: ConnectorEntry): Part | undefined {
const databricks =
connector.method === "adf-files"
? "Auto Loader reads each file once by default. Choose how the table keeps the rows inside those files, separately from which files ADF copies; see [On Databricks](/docs/how-to/keep-a-table-current#on-databricks)."
- : "Databricks reads the landed files into a streaming table. With `mode: append`, repeated copies add repeated rows; `upsert` keeps the current row per key and `history` keeps every version, ordered by a column you choose. See [On Databricks](/docs/how-to/keep-a-table-current#on-databricks)." +
+ : "Databricks reads the landed files into a streaming table. With `history: none`, its default, repeated copies add repeated rows; `history: latest` keeps the current row per key and `history: full` keeps every version, ordered by the `sequence` column you choose. See [On Databricks](/docs/how-to/keep-a-table-current#on-databricks)." +
(connector.method === "adf-copy"
- ? " The store's `sequence_by` decides which version wins; it is separate from the extraction watermark. ADF and Databricks run on separate schedules."
+ ? " `sequence` decides which version wins; it is separate from the copy's cursor, though often the same column. ADF and Databricks run on separate schedules."
: "");
return {
kind: "current",
@@ -267,7 +267,7 @@ function current(id: string, entry: ConnectorEntry): Part | undefined {
"adf-synapse":
"A view reads the landed files each time it is queried and shows them as they are: it cannot keep a current row per key. See [On Synapse](/docs/how-to/keep-a-table-current#on-synapse).",
"adf-duckdb":
- "The DuckDB table's `mode` decides what it keeps; see [On DuckDB](/docs/how-to/keep-a-table-current#on-duckdb).",
+ "The DuckDB table's `history` decides what it keeps; see [On DuckDB](/docs/how-to/keep-a-table-current#on-duckdb).",
"adf-landing": "The landed files are the result: nothing merges them.",
}),
after: [],
@@ -332,9 +332,9 @@ function readSection(id: string, entry: ConnectorEntry): Section {
part: {
kind: "watermarks",
id: `${pack}-watermarks`,
- title: "Different watermarks per table",
+ title: "A different cursor per table",
before: [
- "In this example, the standard reads changed rows once an hour using `modified_at`. Customers inherit that default. Invoices override the watermark with `updated_at` in the flow and set the store's ordering column to match. The same hourly trigger serves both tables.",
+ "In this example, the standard copies changed rows once an hour, and the flow reads them by `modified_at`. Customers take the flow's cursor. Invoices give their own, `updated_at`, and order their changes by it too. The same hourly trigger serves both tables.",
...watermarkNote(id),
],
after: [],
diff --git a/lib/kits.ts b/lib/kits.ts
new file mode 100644
index 0000000..529070c
--- /dev/null
+++ b/lib/kits.ts
@@ -0,0 +1,155 @@
+// Published standards (kits), read at build time from the installed
+// @ingestron/standards package, so each kit's page shows the kit's own files.
+// What a REPLACE_ME stands for comes from the same field reference the
+// editor's hover uses (`ProjectSnapshot.replaceMeHelp`).
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { ProjectSnapshot, type ReplaceMeHelp } from "@ingestron/authoring";
+import { installedKits, packs as corePacks, type Kit } from "@ingestron/core";
+import { loadDefinitions, type Origin } from "@ingestron/definitions";
+import { isMap, isScalar, isSeq, parseDocument, type Node } from "yaml";
+
+/** The `kits` folder of the installed @ingestron/standards package. */
+export const kitsFolder = join(process.cwd(), "node_modules/@ingestron/standards/kits");
+
+let found: Kit[] | undefined;
+
+/** Every installed kit version, by id and then version, oldest first. */
+export function kits(): Kit[] {
+ found ??= installedKits(kitsFolder);
+ return found;
+}
+
+/** The newest version of each kit that is not a preview, in order of id. */
+export function latestKits(): Kit[] {
+ const ids = [...new Set(kits().map((kit) => kit.id))];
+ return ids.map(kitById);
+}
+
+/** A kit's newest version that is not a preview, as `ingestron init --kit ` takes it. */
+export function kitById(id: string): Kit {
+ const kit = kits()
+ .filter((item) => item.id === id && item.features === "ga")
+ .at(-1);
+ if (!kit) throw new Error(`No kit "${id}" is installed.`);
+ return kit;
+}
+
+/** A kit's page: `/docs/how-to/kits/duckdb-local-medallion` for `duckdb/local-medallion`. */
+export function kitPage(id: string): string {
+ return `/docs/how-to/kits/${id.replace("/", "-")}`;
+}
+
+/** A file of a kit, as it is in the package. */
+export function kitFile(kit: Kit, file: string): string {
+ return readFileSync(join(kit.folder, file), "utf8");
+}
+
+/** A value still `REPLACE_ME` in a kit's file, and what it stands for. */
+export interface Blank {
+ /** Where `REPLACE_ME` starts in the file's text. */
+ offset: number;
+ /** The file's line, from 1. */
+ line: number;
+ /** The value's JSON Pointer, such as `/databricks/run_as/service_principal_name`. */
+ pointer: string;
+ /** The field it fills and the help the field gives; undefined when no field describes it. */
+ help?: ReplaceMeHelp;
+}
+
+const snapshots = new Map();
+
+/** The kit's folder opened as a project, as the editor opens it, once. */
+function snapshotOf(kit: Kit): ProjectSnapshot {
+ let snapshot = snapshots.get(kit.folder);
+ if (!snapshot) {
+ snapshot = ProjectSnapshot.open(kit.folder);
+ snapshots.set(kit.folder, snapshot);
+ }
+ return snapshot;
+}
+
+/**
+ * Each `REPLACE_ME` value in a YAML file of a kit, in file order, with the
+ * help the editor's hover gives for it. A value under
+ * `environments.` of project.yaml is explained for that environment.
+ */
+export function kitBlanks(kit: Kit, file: string): Blank[] {
+ const text = kitFile(kit, file);
+ const snapshot = snapshotOf(kit);
+ const blanks: Blank[] = [];
+ const visit = (node: Node | null | undefined, path: string[]) => {
+ if (isMap(node))
+ for (const pair of node.items)
+ visit(pair.value as Node | null, [
+ ...path,
+ String(isScalar(pair.key) ? pair.key.value : pair.key),
+ ]);
+ else if (isSeq(node))
+ node.items.forEach((item, index) => visit(item as Node | null, [...path, String(index)]));
+ else if (isScalar(node) && node.value === "REPLACE_ME" && node.range) {
+ const pointer = `/${path.map((key) => key.replaceAll("~", "~0").replaceAll("/", "~1")).join("/")}`;
+ const environment =
+ file === "project.yaml" && path[0] === "environments" ? path[1] : undefined;
+ const help = snapshot.replaceMeHelp(file, pointer, environment);
+ blanks.push({
+ offset: node.range[0],
+ line: text.slice(0, node.range[0]).split("\n").length,
+ pointer,
+ ...(help && { help }),
+ });
+ }
+ };
+ visit(parseDocument(text).contents as Node | null, []);
+ return blanks;
+}
+
+/** One value of a table's intent in a kit, and where it came from. */
+export interface KitIntent {
+ table: string;
+ layer: string;
+ /** The intent path, such as `history`. */
+ path: string;
+ /** The value as the editor's hover shows it, such as `latest` or `[loaded_at, source_file]`. */
+ value: string;
+ origin: Origin;
+ /** Whether the table is a flow's or a transform's, for naming where a value came from. */
+ kind: "Flow" | "Transform";
+}
+
+/**
+ * Each table's intent, resolved for the kit's first environment as `plan`
+ * resolves it, in the order of the tables and of their intent paths. Audit
+ * columns are shown by their values.
+ */
+export function kitIntent(kit: Kit): KitIntent[] {
+ const environment = snapshotOf(kit).environments[0];
+ const result = loadDefinitions(kit.folder, { packs: corePacks, environment });
+ if (!result.model) throw new Error(`The kit ${kit.id}@${kit.version} does not resolve.`);
+ return result.model.tables.flatMap((table) =>
+ Object.entries(table.intentOrigins).map(([path, origin]) => ({
+ table: table.name,
+ layer: table.layer,
+ path,
+ value: written(valueAt(table.intent, path)),
+ origin,
+ kind: table.origin.kind === "Transform" ? "Transform" : "Flow",
+ })),
+ );
+}
+
+function valueAt(intent: object, path: string): unknown {
+ return path
+ .split(".")
+ .reduce(
+ (node, key) =>
+ node && typeof node === "object" ? (node as Record)[key] : undefined,
+ intent,
+ );
+}
+
+function written(value: unknown): string {
+ if (Array.isArray(value))
+ return `[${value.map((item) => (item && typeof item === "object" && "value" in item ? String(item.value) : written(item))).join(", ")}]`;
+ return String(value);
+}
diff --git a/lib/reference.ts b/lib/reference.ts
index 4f386c7..8120241 100644
--- a/lib/reference.ts
+++ b/lib/reference.ts
@@ -4,10 +4,14 @@ import { join } from "node:path";
import { projectRecipe, projectRoutes, projectSources, type ProjectRoute } from "@ingestron/core";
import {
documentReference,
+ intentPaths,
+ intentUseOf,
methodReference,
+ platformSectionReference,
referenceKinds,
type Connector,
type FieldInfo,
+ type IntentUse,
type Pack,
} from "@ingestron/definitions";
import { adf } from "@ingestron/pack-adf";
@@ -20,6 +24,13 @@ import { contentFile, contentFolder } from "./content-files";
/** The packs the CLI provides, in the order a standard usually lists them. */
export const packs: Pack[] = [databricks, adf, synapse, duckdb];
+/** A pack the CLI provides, by id; throws for any other. */
+export function packOf(id: string): Pack {
+ const found = packs.find((candidate) => candidate.id === id);
+ if (!found) throw new Error(`No pack "${id}".`);
+ return found;
+}
+
/** Each pack's reference page. */
export function packPages(): { pack: string; url: string }[] {
return packs.map((pack) => ({ pack: pack.id, url: `/docs/reference/packs/${pack.id}` }));
@@ -134,18 +145,194 @@ export function shown(value: unknown): string {
/**
* A pack's platform defaults: each Platform field the pack declares it reads
* (its `platformFields`), in the pack's order, with the default the field's
- * description states. Fields with no default, such as `run_as`, are left
- * out. Throws for a pack that reads no Platform field, or for a field the
- * Platform document does not describe.
+ * description states. A field the pack's own section of the Platform has
+ * (`databricks.mode` for `mode`) is shown there, where it is written now.
+ * Fields with no default, such as `run_as`, are left out. Throws for a pack
+ * that reads no Platform field, or for a field neither describes.
*/
export function platformDefaults(pack: string): { field: FieldInfo; default: string }[] {
- const read = packs.find((candidate) => candidate.id === pack)?.platformFields;
+ const found = packOf(pack);
+ const read = found.platformFields;
if (!read?.length) throw new Error(`The ${pack} pack reads no platform defaults.`);
const fields = documentReference("Platform");
+ const section = platformSectionFields(pack);
return read.flatMap((path) => {
- const field = fields.find((candidate) => fieldPath(candidate) === path);
+ const field =
+ section.find((candidate) => fieldPath(candidate) === `${pack}.${path}`) ??
+ section.find((candidate) => fieldPath(candidate) === path) ??
+ fields.find((candidate) => fieldPath(candidate) === path);
if (!field) throw new Error(`The Platform document has no field "${path}".`);
const stated = /Default: (.+?)\.?$/.exec(field.description ?? "")?.[1];
return stated ? [{ field, default: stated }] : [];
});
}
+
+/** The fields of a pack's own section of the Platform, such as `databricks.run_as`; none when it has none. */
+export function platformSectionFields(pack: string): FieldInfo[] {
+ return platformSectionReference(packOf(pack));
+}
+
+/** One row of a pack's intent support: a path, the values that share one entry, and the entry. */
+export interface IntentRow {
+ path: string;
+ /** The values the entry covers, in the vocabulary's order; `["*"]` for a path that takes any text. */
+ values: string[];
+ use: IntentUse;
+}
+
+/**
+ * What a pack does with each intent path and value, from the pack's own
+ * `intent` (each value takes its own entry, else its path's, else a parent
+ * path's, as resolving does). Values of one path that share an entry are one
+ * row. Throws when a value has no entry, so a pack's gap fails the build.
+ */
+export function intentRows(pack: string): IntentRow[] {
+ const found = packOf(pack);
+ const rows: IntentRow[] = [];
+ for (const [path, values] of Object.entries(intentPaths)) {
+ for (const value of values) {
+ const use = intentUseOf(found, path, value === "*" ? undefined : value);
+ if (!use) throw new Error(`The ${pack} pack says nothing about ${path}: ${value}.`);
+ const last = rows.at(-1);
+ if (last?.path === path && JSON.stringify(last.use) === JSON.stringify(use))
+ last.values.push(value);
+ else rows.push({ path, values: [value], use });
+ }
+ }
+ return rows;
+}
+
+/** One entry of a method's own `intent`: where the method differs from its pack. */
+export interface MethodIntentRow {
+ method: string;
+ path: string;
+ /** Undefined when the entry covers the whole path. */
+ value?: string;
+ use: IntentUse;
+}
+
+/** Each method's own intent entries, in the pack's method order, then the vocabulary's. */
+export function methodIntentRows(pack: string): MethodIntentRow[] {
+ const listed = Object.keys(intentPaths);
+ const rank = (path: string) => {
+ const at = listed.findIndex((item) => item === path || item.startsWith(`${path}.`));
+ return at < 0 ? listed.length : at;
+ };
+ return Object.entries(packOf(pack).methods).flatMap(([method, info]) =>
+ Object.entries(info.intent ?? {})
+ .map(([key, use]) => {
+ const [path, value] = key.split("=") as [string, string | undefined];
+ return { method, path, ...(value !== undefined && { value }), use };
+ })
+ .sort((a, b) => {
+ const values = intentPaths[a.path] ?? [];
+ return (
+ rank(a.path) - rank(b.path) ||
+ values.indexOf(a.value ?? "") - values.indexOf(b.value ?? "")
+ );
+ }),
+ );
+}
+
+/** Intent written as `path: value` pairs, such as `["load.mode: files"]`; a list is one value. */
+export function intentPairs(intent: object, prefix = ""): string[] {
+ return Object.entries(intent).flatMap(([key, value]) => {
+ const path = prefix ? `${prefix}.${key}` : key;
+ return value && typeof value === "object" && !Array.isArray(value)
+ ? intentPairs(value, path)
+ : [`${path}: ${shown(value)}`];
+ });
+}
+
+/**
+ * A method's settings that are other names for intent, each as a sentence
+ * part: `` `sequence_by` is `sequence` `` or `` `mode: upsert` is `history: latest` ``.
+ */
+export function aliasTexts(pack: string, method: string): string[] {
+ const aliases = packOf(pack).methods[method]?.aliases ?? {};
+ return Object.entries(aliases).flatMap(([setting, alias]) =>
+ "path" in alias
+ ? [`\`${setting}\` is \`${alias.path}\``]
+ : Object.entries(alias.values).map(
+ ([value, intent]) => `\`${setting}: ${value}\` is \`${intentPairs(intent).join(", ")}\``,
+ ),
+ );
+}
+
+/** Where each intent path's meaning is described: the field of a document that writes it. */
+const intentFields: Record = {
+ "load.mode": ["Standard", "load.default"],
+ "load.cursor": ["Flow", "load.cursor"],
+ "load.deletes.mode": ["Standard", "load.deletes"],
+ "load.deletes.when": ["Flow", "load.deletes.when"],
+ history: ["Standard", "layers.*.history"],
+ sequence: ["Flow", "sequence"],
+ audit: ["Standard", "audit.*.value"],
+ rescued: ["Standard", "rescued"],
+ "retention.time_travel": ["Standard", "retention.time_travel"],
+ "retention.log": ["Standard", "retention.log"],
+ "retention.landed_files": ["Standard", "retention.landed_files"],
+ "storage.clustering": ["Standard", "storage.clustering"],
+ "storage.features": ["Standard", "storage.features"],
+ "operations.schedule": ["Standard", "operations.schedule.per"],
+ "operations.trigger": ["Standard", "operations.trigger"],
+ "operations.retries": ["Standard", "operations.retries"],
+ "operations.timeout": ["Standard", "operations.timeout"],
+ "operations.alerts": ["Standard", "operations.alerts.on"],
+ "operations.concurrency": ["Standard", "operations.concurrency"],
+ "quality.results": ["Standard", "quality.results"],
+};
+
+/** One intent path: what it means, its values with theirs, and the places that can write it. */
+export interface IntentPathInfo {
+ path: string;
+ /** The field that describes it, in the documents reference. */
+ field: FieldInfo;
+ values: { value: string; description?: string }[];
+ /** The places that write it, in the order they win: `standard`, `layer`, `flow`, `flow table`, `transform`. */
+ places: string[];
+}
+
+/**
+ * Each intent path of the vocabulary, in its order, described by the
+ * documents' own fields. Throws for a path no field describes, so a new path
+ * fails the build until it is described.
+ */
+export function intentVocabulary(): IntentPathInfo[] {
+ const fields = {
+ Standard: documentReference("Standard"),
+ Flow: documentReference("Flow"),
+ Transform: documentReference("Transform"),
+ };
+ const has = (kind: keyof typeof fields, path: string) =>
+ fields[kind].some((field) => fieldPath(field) === path);
+ return Object.entries(intentPaths).map(([path, values]) => {
+ const [kind, at] = intentFields[path] ?? [];
+ const field = kind && fields[kind].find((candidate) => fieldPath(candidate) === at);
+ if (!field) throw new Error(`No field describes the intent path "${path}".`);
+ const meanings = new Map(
+ (field.values ?? []).map((item) => [String(item.value), item.description]),
+ );
+ // The standard writes the load mode as `load.default`, and deletes as
+ // `load.deletes`; a cursor or a delete condition only for one flow.
+ const inStandard = (
+ { "load.mode": "load.default", "load.deletes.mode": "load.deletes" } as Record
+ )[path];
+ const places = [
+ has("Standard", inStandard ?? path) && "standard",
+ has("Standard", `layers.*.${path}`) && "layer",
+ has("Flow", path) && "flow",
+ has("Flow", `tables.*.${path}`) && "flow table",
+ has("Transform", path) && "transform",
+ ].filter((place): place is string => typeof place === "string");
+ return {
+ path,
+ field,
+ values: values.map((value) => ({
+ value,
+ ...(meanings.get(value) && { description: meanings.get(value) }),
+ })),
+ places,
+ };
+ });
+}
diff --git a/lib/search.ts b/lib/search.ts
index c87bf43..70ec00e 100644
--- a/lib/search.ts
+++ b/lib/search.ts
@@ -6,7 +6,16 @@ import { codeHelp } from "@ingestron/definitions";
import type { StructuredData } from "fumadocs-core/mdx-plugins";
import { ingestionRows } from "./connector-settings";
import { categoryAnchor, connectorSections, sourceCategories } from "./connector-page";
-import { anchorId, connectorEntries, fieldPath, packPages, packs, settingRows } from "./reference";
+import {
+ anchorId,
+ connectorEntries,
+ fieldPath,
+ intentRows,
+ packPages,
+ packs,
+ platformSectionFields,
+ settingRows,
+} from "./reference";
/** Headings and paragraphs to add to a page's search entries, by page URL. */
export function extraStructure(url: string): StructuredData {
@@ -30,19 +39,34 @@ function errorCodes(): StructuredData {
};
}
+/** A pack's page: its methods and settings, its section of the Platform, and its intent. */
function methods(pack: string): StructuredData {
const found = packs.find((candidate) => candidate.id === pack);
const all = Object.entries(found?.methods ?? {}).map(([id, method]) => ({ pack, id, method }));
+ const section = platformSectionFields(pack);
return {
- headings: all.map(({ id }) => ({ id, content: id })),
- contents: all.flatMap(({ pack, id, method }) => [
- { heading: id, content: method.description },
- ...settingRows(pack, id).map((field) => ({
- heading: id,
- content:
- `${field.path.join(".")}: ${field.description ?? ""} ${(field.values ?? []).map((item) => item.value).join(", ")}`.trim(),
+ headings: [
+ ...all.map(({ id }) => ({ id, content: id })),
+ ...section.map((field) => ({ id: anchorId(field), content: fieldPath(field) })),
+ ],
+ contents: [
+ ...all.flatMap(({ pack, id, method }) => [
+ { heading: id, content: method.description },
+ ...settingRows(pack, id).map((field) => ({
+ heading: id,
+ content:
+ `${field.path.join(".")}: ${field.description ?? ""} ${(field.values ?? []).map((item) => item.value).join(", ")}`.trim(),
+ })),
+ ]),
+ ...section.map((field) => ({
+ heading: anchorId(field),
+ content: `${fieldPath(field)}: ${field.description ?? ""} ${field.find ?? ""}`.trim(),
})),
- ]),
+ ...intentRows(pack).map(({ path, values, use }) => ({
+ heading: "intent",
+ content: `${path} ${values.join(", ")}: ${use.use === "used" ? use.does : use.reason}`,
+ })),
+ ],
};
}
diff --git a/package.json b/package.json
index 052d842..1992e38 100644
--- a/package.json
+++ b/package.json
@@ -23,14 +23,15 @@
"sync:vscode": "node scripts/sync-vscode-commands.mjs"
},
"dependencies": {
- "@ingestron/authoring": "^2.2.0",
- "@ingestron/cli": "^2.2.0",
- "@ingestron/core": "^1.0.0",
- "@ingestron/definitions": "^1.11.0",
- "@ingestron/pack-adf": "^2.1.0",
- "@ingestron/pack-databricks": "^2.1.0",
- "@ingestron/pack-duckdb": "^2.0.0",
- "@ingestron/pack-synapse": "^2.0.0",
+ "@ingestron/authoring": "^2.3.0",
+ "@ingestron/cli": "^2.3.0",
+ "@ingestron/core": "^1.1.0",
+ "@ingestron/definitions": "^1.13.0",
+ "@ingestron/pack-adf": "^2.3.0",
+ "@ingestron/pack-databricks": "^2.3.0",
+ "@ingestron/pack-duckdb": "^2.2.0",
+ "@ingestron/pack-synapse": "^2.2.0",
+ "@ingestron/standards": "^1.0.0",
"fflate": "0.8.2",
"fumadocs-core": "16.17.8",
"fumadocs-mdx": "15.4.10",
diff --git a/scripts/ci-templates.test.mjs b/scripts/ci-templates.test.mjs
index db4561a..858ebdb 100644
--- a/scripts/ci-templates.test.mjs
+++ b/scripts/ci-templates.test.mjs
@@ -58,7 +58,7 @@ test("every YAML template parses, and the folder holds no other kind of file", (
* The ingestron bundle the templates install: an exact version, the one the
* docs build with (test/standards-and-teams.test.ts checks the cli's version).
*/
-const bundle = "2.2.0";
+const bundle = "2.3.0";
test("each check installs an exact ingestron from npm, then runs ingestron check", () => {
for (const file of [...checks, "github-actions/deploy.yml", "azure-devops/azure-pipelines.yml"]) {
diff --git a/test/areas.test.ts b/test/areas.test.ts
index 30346c6..072e3bd 100644
--- a/test/areas.test.ts
+++ b/test/areas.test.ts
@@ -52,6 +52,11 @@ const areas: Record = {
],
"Standards and teams": [
"/docs/how-to/configure-a-standard",
+ "/docs/how-to/choose-a-published-standard",
+ "/docs/how-to/start-from-a-published-standard",
+ "/docs/how-to/upgrade-a-standard",
+ "/docs/how-to/kits/duckdb-local-medallion",
+ "/docs/how-to/kits/databricks-lakehouse-streaming",
"/docs/how-to/name-things-your-way",
"/docs/how-to/ask-the-platform-team",
"/docs/how-to/share-a-template",
@@ -69,6 +74,7 @@ const areas: Record = {
Connectors: ["/docs/connectors"],
Reference: [
"/docs/reference/documents",
+ "/docs/reference/intent",
"/docs/reference/packs",
"/docs/reference/packs/databricks",
"/docs/reference/packs/adf",
@@ -188,7 +194,7 @@ describe("page URLs", () => {
const built = readFileSync(join(process.cwd(), "test/page-urls.txt"), "utf8")
.trim()
.split("\n");
- expect(built).toHaveLength(127);
+ expect(built).toHaveLength(133);
expect(
source
.getPages()
diff --git a/test/build-a-project.test.ts b/test/build-a-project.test.ts
index f21e52e..3fb45d6 100644
--- a/test/build-a-project.test.ts
+++ b/test/build-a-project.test.ts
@@ -6,6 +6,7 @@ import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { run } from "@ingestron/cli";
+import { intentUseOf } from "@ingestron/definitions";
import { createElement } from "react";
import { renderToStaticMarkup } from "react-dom/server";
import { describe, expect, it } from "vitest";
@@ -117,33 +118,24 @@ describe("how-to/keep-a-table-current", () => {
expect(after).toContain('ORDER BY "ordered_at" DESC, "_ingestron_file" DESC');
});
- it("compares the modes each engine's store method accepts", () => {
+ it("compares what each engine does with the intent, from the packs' own coverage", () => {
const table = page.split("\n## ")[0]!;
- const row = (engine: string) =>
- table
- .split("\n")
- .find((line) => line.startsWith(`| ${engine} `))!
- .split("|")
- .slice(2, -1)
- .map((cell) => cell.trim());
- const duckdb = accepted("duckdb", "duckdb-table", "mode");
- expect(row("DuckDB")).toEqual(["`upsert`", "No", "No"]);
- expect(duckdb).toContain("upsert");
- expect(duckdb).not.toContain("history");
- expect(
- settingRows("duckdb", "duckdb-table").map((field) => field.path.join(".")),
- ).not.toContain("delete_when");
- const databricks = accepted("databricks", "streaming-table", "mode");
- expect(row("Databricks")).toEqual(["`upsert`", "`history`", "`delete_when`"]);
- expect(databricks).toEqual(expect.arrayContaining(["upsert", "history"]));
- expect(
- settingRows("databricks", "streaming-table").map((field) => field.path.join(".")),
- ).toContain("delete_when");
- // A Synapse view has no mode at all.
- expect(row("Synapse")).toEqual(["No", "No", "No"]);
- expect(
- settingRows("synapse", "synapse-view").map((field) => field.path.join(".")),
- ).not.toContain("mode");
+ expect(table).toContain("
+ intentUseOf(
+ packs.find((item) => item.id === pack)!,
+ path,
+ value,
+ )?.use;
+ expect(used("duckdb", "history", "latest")).toBe("used");
+ expect(used("duckdb", "history", "full")).not.toBe("used");
+ expect(used("databricks", "history", "full")).toBe("used");
+ expect(used("databricks", "load.deletes.when")).toBe("used");
+ expect(used("synapse", "history", "latest")).toBe("not-applicable");
+ // The settings each intent stands for, which the page names.
+ expect(accepted("duckdb", "duckdb-table", "mode")).toContain("upsert");
+ expect(page).toContain("`mode: upsert`");
});
});
@@ -222,7 +214,7 @@ describe("how-to/write-a-contract", () => {
it("checks rows by a column's logicalTypeOptions, and unique once checks are on", () => {
const text = prose(section(page, "## Rules about the data"));
expect(text).toContain("such as `customer_id_minimum`");
- expect(text).toContain("when the platform turns on `manage.checks`");
+ expect(text).toContain("when the platform turns on `databricks.manage.checks`");
const cwd = copy("minimal");
const contract = join(cwd, "contracts/orders.yaml");
writeFileSync(
@@ -243,7 +235,7 @@ describe("how-to/write-a-contract", () => {
// With checks on, unique becomes a check over all rows after the refresh.
writeFileSync(
join(cwd, "platform.yaml"),
- "apiVersion: ingestron.io/v1\nkind: Platform\nid: checks\nmanage: { checks: true }\n",
+ "apiVersion: ingestron.io/v1\nkind: Platform\nid: checks\ndatabricks: { manage: { checks: true } }\n",
);
const project = parse(readFileSync(join(cwd, "project.yaml"), "utf8"));
writeFileSync(
diff --git a/test/layers.test.ts b/test/layers.test.ts
index d5715dc..94f048d 100644
--- a/test/layers.test.ts
+++ b/test/layers.test.ts
@@ -95,9 +95,10 @@ describe("concepts/layers", () => {
by: "standard-rule",
when: { layer: "gold" },
});
- expect(table("region_sales").store?.settings.cluster_by).toBe("none");
+ const retention = { "delta.logRetentionDuration": "interval 90 days" };
+ expect(table("region_sales").store?.settings.properties).toEqual(retention);
expect(table("store_sales").store?.chosenBy.by).not.toBe("standard-rule");
- expect(table("store_sales").store?.settings.cluster_by).not.toBe("none");
+ expect(table("store_sales").store?.settings.properties).not.toEqual(retention);
});
it("replaces the list with a standard's own layers, so the default ones no longer exist", () => {
diff --git a/test/page-urls.txt b/test/page-urls.txt
index d035ee6..bbf97a5 100644
--- a/test/page-urls.txt
+++ b/test/page-urls.txt
@@ -89,15 +89,20 @@
/docs/how-to/add-quality-checks
/docs/how-to/ask-the-platform-team
/docs/how-to/check-in-ci
+/docs/how-to/choose-a-published-standard
/docs/how-to/configure-a-standard
/docs/how-to/define-a-data-product
/docs/how-to/deploy-through-environments
/docs/how-to/keep-a-table-current
+/docs/how-to/kits/databricks-lakehouse-streaming
+/docs/how-to/kits/duckdb-local-medallion
/docs/how-to/name-things-your-way
/docs/how-to/plan-and-generate
/docs/how-to/share-a-template
+/docs/how-to/start-from-a-published-standard
/docs/how-to/start-from-a-template
/docs/how-to/upgrade
+/docs/how-to/upgrade-a-standard
/docs/how-to/write-a-contract
/docs/how-to/write-a-transform
/docs/platforms/adf
@@ -112,6 +117,7 @@
/docs/reference/cli
/docs/reference/documents
/docs/reference/errors
+/docs/reference/intent
/docs/reference/packs
/docs/reference/packs/adf
/docs/reference/packs/databricks
diff --git a/test/platform-defaults.test.ts b/test/platform-defaults.test.ts
index e8fe798..4aa36ca 100644
--- a/test/platform-defaults.test.ts
+++ b/test/platform-defaults.test.ts
@@ -24,19 +24,20 @@ function platform(folder: string, environment: string) {
describe("platform defaults", () => {
it("lists each field the Databricks pack reads, with its default, linked to its reference", () => {
const rows = platformDefaults("databricks");
+ // A field of the pack's own section is shown there, where it is written now.
expect(rows.map((row) => row.field.path.join("."))).toEqual([
- "mode",
- "compute.serverless",
+ "databricks.mode",
+ "databricks.compute.serverless",
"schedule.timezone",
"schedule.paused",
- "manage.schemas",
- "manage.tags",
- "manage.comments",
- "manage.checks",
+ "databricks.manage.schemas",
+ "databricks.manage.tags",
+ "databricks.manage.comments",
+ "databricks.manage.checks",
]);
const html = renderToStaticMarkup(createElement(PlatformDefaults, { pack: "databricks" }));
expect(html).toContain(
- 'mode',
+ 'databricks.mode',
);
expect(html).toContain(
"development in an environment named dev, and production in every other, test and default included",
@@ -92,11 +93,11 @@ describe("platform defaults", () => {
platformDefaults("databricks").map((row) => [row.field.path.join("."), row.default]),
);
expect(stated).toMatchObject({
- "compute.serverless": "true",
+ "databricks.compute.serverless": "true",
"schedule.timezone": "UTC",
"schedule.paused":
"running in an environment named prod or production, and where the platform sets mode: production itself; paused everywhere else, test included, though its mode is production by default",
- "manage.tags": "false",
+ "databricks.manage.tags": "false",
});
});
});
diff --git a/test/platform-pages.test.ts b/test/platform-pages.test.ts
index 6c203fe..8784c60 100644
--- a/test/platform-pages.test.ts
+++ b/test/platform-pages.test.ts
@@ -291,6 +291,6 @@ describe("reference/versions", () => {
expect(tools).toContain(`\`@microsoft/azure-data-factory-utilities\` ${adf}`);
expect(tools).toContain(`Databricks CLI ${databricks}`);
expect(tools).toContain(`sqlcmd (Go) ${sqlcmd}`);
- expect(tools).toContain("`ingestron@2.2.0`");
+ expect(tools).toContain("`ingestron@2.3.0`");
});
});
diff --git a/test/reader-journeys.test.ts b/test/reader-journeys.test.ts
index b651d9c..5f5d970 100644
--- a/test/reader-journeys.test.ts
+++ b/test/reader-journeys.test.ts
@@ -310,7 +310,7 @@ describe("platforms/databricks", () => {
// Deploying as one identity: a shared root_path, and binding what another identity deployed.
const identity = section(deployPage, "### Deploy as one identity");
expect(identity).toContain(
- '',
+ '',
);
expect(fences(identity, "sh")[0]).toContain(
"databricks bundle deployment bind -t prod",
@@ -318,7 +318,7 @@ describe("platforms/databricks", () => {
const platform = parse(
readFileSync(join(exampleFolder("environments"), "platform.yaml"), "utf8"),
);
- expect(platform.workspace.root_path).toMatch(/^\/Workspace\/Shared\//);
+ expect(platform.databricks.workspace.root_path).toMatch(/^\/Workspace\/Shared\//);
expect(section(page, "## Deploy and run")).toContain("(/docs/platforms/databricks-deploy)");
const ci = section(deployPage, "### From CI");
expect(ci).toContain('file="github-actions/deploy-environment.yml"');
diff --git a/test/site-journey.test.ts b/test/site-journey.test.ts
index 699e6ec..f992915 100644
--- a/test/site-journey.test.ts
+++ b/test/site-journey.test.ts
@@ -54,6 +54,10 @@ it("opens with one learning path, then areas by the reader's job, with no index
]);
expect(JSON.parse(read("how-to/(standards)/meta.json")).pages).toEqual([
"configure-a-standard",
+ "choose-a-published-standard",
+ "start-from-a-published-standard",
+ "upgrade-a-standard",
+ "kits",
"name-things-your-way",
"ask-the-platform-team",
"share-a-template",
@@ -88,6 +92,7 @@ it("opens with one learning path, then areas by the reader's job, with no index
]);
expect(JSON.parse(read("reference/meta.json")).pages).toEqual([
"documents",
+ "intent",
"packs",
"errors",
"cli",
diff --git a/test/sql-watermarks.test.ts b/test/sql-watermarks.test.ts
index 0f3fb9d..257e97a 100644
--- a/test/sql-watermarks.test.ts
+++ b/test/sql-watermarks.test.ts
@@ -42,15 +42,21 @@ it("describes the watermark example as the sample has it, on every adf-copy page
expect(item.ingest?.settings.mode).toBe("incremental");
expect(item.ingest?.settings.every).toEqual({ frequency: "Hour", interval: 1 });
}
- // "using `modified_at`. Customers inherit that default.": the standard sets it, the flow does not.
+ // "the standard copies changed rows once an hour, and the flow reads them by `modified_at`".
const standard = parse(readExampleFile(watermarkExample, "standard.yaml"));
- expect(standard.ingest.settings["adf-copy"].watermark_column).toBe("modified_at");
- expect(words).toContain("using `modified_at`. Customers inherit that default.");
- expect(customers.ingest?.settings.watermark_column).toBe("modified_at");
- // "Invoices override the watermark with `updated_at` in the flow and set the store's ordering column to match."
+ expect(standard.load.default).toBe("incremental");
const flow = parse(readExampleFile(watermarkExample, "flows/erp.yaml"));
- expect(flow.tables.customers.ingest).toBeUndefined();
- expect(flow.tables.invoices.ingest.with.watermark_column).toBe("updated_at");
- expect(flow.tables.invoices.store.with.sequence_by).toBe("updated_at");
- expect(words).toContain("Invoices override the watermark with `updated_at` in the flow");
+ expect(flow.load.cursor).toBe("modified_at");
+ expect(words).toContain(
+ "the flow reads them by `modified_at`. Customers take the flow's cursor.",
+ );
+ expect(customers.ingest?.settings.watermark_column).toBe("modified_at");
+ expect(flow.tables.customers.load).toBeUndefined();
+ // "Invoices give their own, `updated_at`, and order their changes by it too."
+ expect(flow.tables.invoices.load.cursor).toBe("updated_at");
+ expect(flow.tables.invoices.sequence).toBe("updated_at");
+ expect(invoices.store?.settings.sequence_by).toBe("updated_at");
+ expect(words).toContain(
+ "Invoices give their own, `updated_at`, and order their changes by it too.",
+ );
});
diff --git a/test/standards-and-teams.test.ts b/test/standards-and-teams.test.ts
index c6d7f03..0b8cb5b 100644
--- a/test/standards-and-teams.test.ts
+++ b/test/standards-and-teams.test.ts
@@ -324,7 +324,7 @@ describe("how-to/upgrade", () => {
it("pins the exact bundle these pages describe, as the templates install it", () => {
// The ingestron bundle these pages describe: the installed cli's version.
- const bundle = "2.2.0";
+ const bundle = "2.3.0";
const choose = section(page, "## Choose the version you run");
const version = JSON.parse(
readFileSync(join(process.cwd(), "node_modules/@ingestron/cli/package.json"), "utf8"),
diff --git a/test/standards-kits.test.ts b/test/standards-kits.test.ts
new file mode 100644
index 0000000..69be4cb
--- /dev/null
+++ b/test/standards-kits.test.ts
@@ -0,0 +1,244 @@
+// Published standards (kits), intent and the field reference's help: the pages
+// show what the packages and the installed kits hold, and what the CLI prints.
+import { execFileSync } from "node:child_process";
+import { appendFileSync, mkdtempSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createElement } from "react";
+import { renderToStaticMarkup } from "react-dom/server";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { documentReference, intentPaths, intentUseOf } from "@ingestron/definitions";
+import { FieldTable, PlatformSection } from "../components/reference/document-fields";
+import { KitFacts, KitIntent, KitList } from "../components/reference/kits";
+import {
+ IntentAliases,
+ IntentOverview,
+ IntentSupport,
+ IntentVocabulary,
+} from "../components/reference/pack-intent";
+import { kitBlanks, kitById, kitPage, kits, latestKits } from "../lib/kits";
+import {
+ fieldPath,
+ intentRows,
+ intentVocabulary,
+ packs,
+ platformSectionFields,
+} from "../lib/reference";
+import { cli, fences, read, section } from "./pages.js";
+
+const html = (element: Parameters[0]) => renderToStaticMarkup(element);
+
+// No test reaches the network: the catalogue is a repository that is not there.
+let saved: string | undefined;
+beforeAll(() => {
+ saved = process.env.INGESTRON_STANDARDS_REPOSITORY;
+ process.env.INGESTRON_STANDARDS_REPOSITORY = `file://${join(tmpdir(), "no-such-catalogue.git")}`;
+});
+afterAll(() => {
+ if (saved === undefined) delete process.env.INGESTRON_STANDARDS_REPOSITORY;
+ else process.env.INGESTRON_STANDARDS_REPOSITORY = saved;
+});
+
+describe("the field reference's help", () => {
+ it("shows where to find a value and a suggestion, where a field has them", () => {
+ const field = platformSectionFields("databricks").find(
+ (item) => fieldPath(item) === "databricks.run_as.service_principal_name",
+ )!;
+ expect(field.find).toBeTruthy();
+ expect(field.suggest).toBe("");
+ const shown = html(createElement(FieldTable, { label: "x", fields: [field] }));
+ expect(shown).toContain("Where to find it: Settings, then Identity and access");
+ expect(shown).toContain("Suggested: <application id>");
+ // A document's field too: an alert's address.
+ const to = documentReference("Platform").find((item) => fieldPath(item) === "alerts.*.to.*")!;
+ expect(html(createElement(FieldTable, { label: "x", fields: [to] }))).toContain(
+ "Suggested: <team>@<domain>",
+ );
+ });
+
+ it("gives each field of the Databricks section its anchor on the pack's page", () => {
+ const page = read("reference/packs/databricks.mdx");
+ expect(page).toContain('');
+ const shown = html(createElement(PlatformSection, { pack: "databricks" }));
+ for (const field of platformSectionFields("databricks")) {
+ const [path, anchor] = field.docs.split("#");
+ expect(path).toBe("/docs/reference/packs/databricks");
+ expect(shown).toContain(`id="${anchor}"`);
+ }
+ });
+});
+
+describe("intent", () => {
+ it("has a support matrix on every pack's page, from the pack's own coverage", () => {
+ for (const pack of packs) {
+ expect(read(`reference/packs/${pack.id}.mdx`)).toContain(
+ `## Intent [#intent]\n\nWhat this pack does with each value of [intent](/docs/reference/intent)`,
+ );
+ expect(read(`reference/packs/${pack.id}.mdx`)).toContain(
+ ``,
+ );
+ // Every value of every path, each covered once.
+ const rows = intentRows(pack.id);
+ expect(rows.flatMap((row) => row.values.map((value) => `${row.path}=${value}`))).toEqual(
+ Object.entries(intentPaths).flatMap(([path, values]) =>
+ values.map((value) => `${path}=${value}`),
+ ),
+ );
+ }
+ const databricks = html(createElement(IntentSupport, { pack: "databricks" }));
+ // A preview feature shows its release level, linked to the page, and the date checked.
+ const slow = intentUseOf(packs[0]!, "operations.alerts", "slow");
+ expect(slow?.use === "used" && slow.status?.level).toBe("preview");
+ expect(databricks).toContain(">Public Preview");
+ expect(databricks).toContain(">Generally available");
+ expect(databricks).toContain("2026-11-09 | ");
+ // Where a method differs from its pack.
+ expect(databricks).toContain("Managed ingestion reads the source, not files.");
+ });
+
+ it("describes every intent path, and shows every pack's use of each value", () => {
+ const vocabulary = intentVocabulary();
+ expect(vocabulary.map((row) => row.path)).toEqual(Object.keys(intentPaths));
+ const history = vocabulary.find((row) => row.path === "history")!;
+ expect(history.places).toEqual(["layer", "flow", "flow table", "transform"]);
+ expect(vocabulary.find((row) => row.path === "load.cursor")!.places).toEqual([
+ "flow",
+ "flow table",
+ ]);
+ const page = read("reference/intent.mdx");
+ for (const component of ["", "", ""])
+ expect(page).toContain(component);
+ expect(html(createElement(IntentVocabulary))).toContain("Only the latest row per key.");
+ const overview = html(createElement(IntentOverview));
+ for (const pack of packs) expect(overview).toContain(`/docs/reference/packs/${pack.id}#intent`);
+ expect(html(createElement(IntentAliases))).toContain(
+ "mode: upsert is history: latest",
+ );
+ });
+});
+
+describe("published standards", () => {
+ it("has a page for each installed kit, titled as the kit, with its current version", () => {
+ expect(latestKits().map((kit) => kit.id)).toEqual([
+ "databricks/lakehouse-streaming",
+ "duckdb/local-medallion",
+ ]);
+ const list = html(createElement(KitList));
+ for (const kit of latestKits()) {
+ const path = `${kitPage(kit.id).replace("/docs/", "")}.mdx`;
+ const page = read(path);
+ expect(page).toContain(`title: ${kit.title}\n`);
+ expect(page).toContain(``);
+ expect(fences(page, "sh")).toEqual([`ingestron init shop --kit ${kit.id}@${kit.version}\n`]);
+ expect(list).toContain(`href="${kitPage(kit.id)}"`);
+ expect(html(createElement(KitFacts, { id: kit.id }))).toContain(kit.summary);
+ }
+ expect(kits().every((kit) => kit.features === "ga")).toBe(true);
+ });
+
+ it("explains every REPLACE_ME of each kit with the field reference the hover uses", () => {
+ for (const kit of latestKits())
+ for (const file of ["platform.yaml", "project.yaml"]) {
+ for (const blank of kitBlanks(kit, file)) {
+ expect(blank.help, `${kit.id} ${file} ${blank.pointer}`).toBeDefined();
+ expect(blank.help!.find, `${kit.id} ${file} ${blank.pointer}`).toBeTruthy();
+ expect(blank.help!.suggest, `${kit.id} ${file} ${blank.pointer}`).toBeTruthy();
+ }
+ }
+ const kit = kitById("databricks/lakehouse-streaming");
+ expect(kitBlanks(kit, "platform.yaml").map((blank) => [blank.line, blank.help!.field])).toEqual(
+ [
+ [12, "to"],
+ [18, "service_principal_name"],
+ [22, "group_name"],
+ [25, "group_name"],
+ ],
+ );
+ // Each kit page shows the files that hold them.
+ const page = read(`${kitPage(kit.id).replace("/docs/", "")}.mdx`);
+ for (const file of ["platform.yaml", "project.yaml"])
+ expect(page).toContain(``);
+ });
+
+ it("shows where each table's intent came from", () => {
+ const shown = html(createElement(KitIntent, { id: "databricks/lakehouse-streaming" }));
+ expect(shown).toContain("silver.orders");
+ expect(shown).toContain("history: latest");
+ expect(shown).toContain("the table's layer in the standard (standard.yaml:");
+ expect(shown).toContain("this table (flows/orders.yaml:");
+ expect(shown).toContain("the pack's default");
+ });
+
+ it("prints what Start from a published standard shows", () => {
+ const page = read("how-to/start-from-a-published-standard.mdx");
+ const create = section(page, "## Create the project");
+ const [command] = fences(create, "sh");
+ expect(command).toBe("ingestron init shop --kit duckdb/local-medallion@2026-11-10\n");
+ const cwd = mkdtempSync(join(tmpdir(), "docs-kit-"));
+ const { code, output } = cli(cwd, command!.trim().split(" ").slice(1));
+ expect(code).toBe(0);
+ expect(output).toBe(fences(create, "text")[0]);
+ const connection = section(page, "## With a connection [#with-a-connection]");
+ const [warning] = fences(connection, "text");
+ const sales = cli(cwd, [
+ "init",
+ "sales",
+ "--kit",
+ "databricks/lakehouse-streaming@2026-11-10",
+ "--connector",
+ "sqlserver",
+ "--connectivity",
+ "public",
+ "--via",
+ "databricks",
+ "--yes",
+ ]);
+ expect(sales.code).toBe(0);
+ expect(sales.output).toContain(warning!.trimEnd());
+ });
+
+ it("prints what Choose a published standard and Move to a newer one show", () => {
+ const cwd = mkdtempSync(join(tmpdir(), "docs-kit-"));
+ expect(cli(cwd, ["init", "p", "--kit", "duckdb/local-medallion@2026-11-10"]).code).toBe(0);
+ const project = join(cwd, "p");
+ const git = (...args: string[]) =>
+ execFileSync("git", args, {
+ cwd: project,
+ env: {
+ ...process.env,
+ GIT_AUTHOR_NAME: "t",
+ GIT_AUTHOR_EMAIL: "t@example.com",
+ GIT_COMMITTER_NAME: "t",
+ GIT_COMMITTER_EMAIL: "t@example.com",
+ },
+ });
+ git("init", "-q");
+ git("add", "-A");
+ git("commit", "-qm", "init");
+
+ const choose = read("how-to/choose-a-published-standard.mdx");
+ const listed = cli(project, ["standard", "list"]);
+ expect(listed.code).toBe(0);
+ const reached = "Could not reach the catalogue; showing the kits installed with ingestron.\n";
+ expect(listed.output.startsWith(reached)).toBe(true);
+ expect(choose).toContain(`\`${reached.trim()}\``);
+ expect(listed.output.slice(reached.length)).toBe(
+ fences(section(choose, "## List them"), "text")[0],
+ );
+
+ const upgrade = read("how-to/upgrade-a-standard.mdx");
+ const [refused, printed] = fences(upgrade, "text");
+ const install = cli(project, ["standard", "install", "duckdb/local-medallion@2026-11-10"]);
+ expect(install.code).toBe(0);
+ expect(install.output).toBe(printed);
+ // The new file is what the project had: the same version.
+ expect(git("status", "--porcelain").toString()).toBe("");
+ // A standard with changes that are not committed is refused.
+ const standard = join(project, "standard.yaml");
+ appendFileSync(standard, "# a change\n");
+ const stopped = cli(project, ["standard", "install", "duckdb/local-medallion@2026-11-10"]);
+ expect(stopped.code).toBe(1);
+ expect(stopped.output).toBe(refused);
+ expect(readFileSync(standard, "utf8").endsWith("# a change\n")).toBe(true);
+ });
+});
diff --git a/test/troubleshooting.test.ts b/test/troubleshooting.test.ts
index c1c3dd9..a5fc809 100644
--- a/test/troubleshooting.test.ts
+++ b/test/troubleshooting.test.ts
@@ -180,7 +180,10 @@ describe("generated files", () => {
const project = copy("private-sql-server-filled");
cli(project, ["apply"]);
appendFileSync(join(project, "build/databricks/databricks.yml"), "# my change\n");
- appendFileSync(join(project, "standard.yaml"), "layers: [raw, curated]\n");
+ appendFileSync(
+ join(project, "standard.yaml"),
+ "store: { settings: { streaming-table: { properties: { team: erp } } } }\n",
+ );
const { code, output } = cli(project, ["apply", "--no-diff"]);
expect(code).toBe(1);
expect(lines(output)).toContain("Applied: adf. Not applied: databricks.");
@@ -633,7 +636,7 @@ describe("Databricks", () => {
it("quotes a change mode without a sequence column, and without a key", () => {
const project = copy("change-data");
edit(project, "flows/crm.yaml", (text) =>
- text.replace("mode: upsert,\n sequence_by: changed_at,", "mode: upsert,"),
+ text.replace("sequence: changed_at # the newest change to a key wins\n", ""),
);
const { code, output } = cli(project, ["plan", "--no-diff"]);
expect(code).toBe(1);
From 18d846b85536a7fb6492079a43b8c735720f50c7 Mon Sep 17 00:00:00 2001
From: The Architect
Date: Tue, 10 Nov 2026 00:23:39 +0000
Subject: [PATCH 02/17] Examples and pages: intent words, no stale screenshots
Signed-off-by: The Architect
---
components/replace-me.tsx | 4 +--
components/screenshot.tsx | 2 +-
content/docs/(reference)/reference/intent.mdx | 2 +-
content/docs/(reference)/tools/vscode.mdx | 28 ------------------
content/docs/concepts/settings.mdx | 7 +++--
.../how-to/(build)/start-from-a-template.mdx | 7 -----
.../(standards)/ask-the-platform-team.mdx | 2 +-
.../kits/databricks-lakehouse-streaming.mdx | 6 ++--
content/docs/how-to/(standards)/upgrade.mdx | 4 +--
examples/change-data/README.md | 16 +++++-----
examples/platform-setup/README.md | 18 +++++------
public/images/vscode/completion-dark.png | Bin 127136 -> 0 bytes
public/images/vscode/completion-light.png | Bin 128024 -> 0 bytes
public/images/vscode/editor-dark.png | Bin 184681 -> 0 bytes
public/images/vscode/editor-light.png | Bin 186264 -> 0 bytes
public/images/vscode/hover-dark.png | Bin 210738 -> 0 bytes
public/images/vscode/hover-light.png | Bin 212126 -> 0 bytes
public/images/vscode/new-project-dark.png | Bin 39258 -> 0 bytes
public/images/vscode/new-project-light.png | Bin 39351 -> 0 bytes
test/screenshot.test.ts | 4 +--
20 files changed, 35 insertions(+), 65 deletions(-)
delete mode 100644 public/images/vscode/completion-dark.png
delete mode 100644 public/images/vscode/completion-light.png
delete mode 100644 public/images/vscode/editor-dark.png
delete mode 100644 public/images/vscode/editor-light.png
delete mode 100644 public/images/vscode/hover-dark.png
delete mode 100644 public/images/vscode/hover-light.png
delete mode 100644 public/images/vscode/new-project-dark.png
delete mode 100644 public/images/vscode/new-project-light.png
diff --git a/components/replace-me.tsx b/components/replace-me.tsx
index ba6b148..2d47b56 100644
--- a/components/replace-me.tsx
+++ b/components/replace-me.tsx
@@ -1,7 +1,7 @@
"use client";
// A value still REPLACE_ME in a kit's file (components/reference/kits.tsx):
-// marked by an outline and its note's number, not by colour alone, and
-// focusable. On hover or focus it shows the field, where to find the value and
+// marked by a dashed underline, a light tint and its note's number, not by
+// colour alone, and focusable. On hover or focus it shows the field, where to find the value and
// a suggestion, in a tooltip drawn above the page, so the code block's
// scrolling frame does not clip it and Copy does not copy it. Escape closes
// it, and the pointer can move onto it. The same text is the note the mark's
diff --git a/components/screenshot.tsx b/components/screenshot.tsx
index 8a3ff26..772d048 100644
--- a/components/screenshot.tsx
+++ b/components/screenshot.tsx
@@ -56,7 +56,7 @@ type Captures =
/**
* One framed screenshot.
- * @example
+ * @example
* @example
*/
export function Screenshot({
diff --git a/content/docs/(reference)/reference/intent.mdx b/content/docs/(reference)/reference/intent.mdx
index 5d093bb..a722e35 100644
--- a/content/docs/(reference)/reference/intent.mdx
+++ b/content/docs/(reference)/reference/intent.mdx
@@ -111,7 +111,7 @@ would.
`plan` reports each such setting as
[`SETTING_MOVED`](/docs/reference/errors#SETTING_MOVED). In a flow's or a
-table's `with`, its fix, **Write as intent**, writes the intent there and
+flow table's `with`, its fix, **Write as intent**, writes the intent there and
removes the setting. Anywhere else, such as the standard's settings or a
rule's `with`, write the intent by hand where it should apply: written for a
whole standard or layer, intent also reaches tables and methods the setting
diff --git a/content/docs/(reference)/tools/vscode.mdx b/content/docs/(reference)/tools/vscode.mdx
index 603f523..0577f09 100644
--- a/content/docs/(reference)/tools/vscode.mdx
+++ b/content/docs/(reference)/tools/vscode.mdx
@@ -165,13 +165,6 @@ marked **fixed by the standard**, and marks the others **not allowed**. If a
definition sets a fixed value anyway, the `FIXED_SETTING` problem has a quick
fix, **Remove this setting**, so the standard's value applies.
-
-
It also fills in what the project already decides:
- **A new, empty file** offers a document to start from: the project
@@ -210,13 +203,6 @@ it resolves to:
Turn these off with VS Code's `editor.inlayHints.enabled` and
`editor.codeLens` settings.
-
-
## Why is this value what it is?
Hover a flow table or a transform to see its full name, the method for each
@@ -241,13 +227,6 @@ docs, and each problem's code in the Problems panel links to its entry in
`ingestron.docs.url` setting; set it to a local build, such as
`http://localhost:3000`, to read the docs offline.
-
-
## Project view
The **Ingestron** tab in the secondary side bar, on the right, opens with a
@@ -433,13 +412,6 @@ README lists each with where to find it. When the folder is outside the
workspace, the message offers to open it or add it to the workspace. See
[Create a project](/docs/how-to/start-from-a-template).
-
-
## Check files with the YAML extension [#check-files-with-the-yaml-extension]
Without the Ingestron extension, the
diff --git a/content/docs/concepts/settings.mdx b/content/docs/concepts/settings.mdx
index c847e53..399793e 100644
--- a/content/docs/concepts/settings.mdx
+++ b/content/docs/concepts/settings.mdx
@@ -47,9 +47,10 @@ still unset comes from the chosen method's own default. See
Some settings are other names for intent, such as `streaming-table`'s
`mode: upsert`, which is `history: latest`. Written as a setting, one still
-works, in the place it is written, and `plan` says
-[`SETTING_MOVED`](/docs/reference/errors#SETTING_MOVED), with a fix that
-writes the intent instead; see
+works, in the place it is written, and `plan` warns with
+[`SETTING_MOVED`](/docs/reference/errors#SETTING_MOVED). In a flow's or a flow
+table's `with`, its fix writes the intent there instead; anywhere else, such
+as the standard's settings, write the intent by hand; see
[Settings that are intent](/docs/reference/intent#settings-that-are-intent).
## Names, layers, types and quality
diff --git a/content/docs/how-to/(build)/start-from-a-template.mdx b/content/docs/how-to/(build)/start-from-a-template.mdx
index c81f06f..702b863 100644
--- a/content/docs/how-to/(build)/start-from-a-template.mdx
+++ b/content/docs/how-to/(build)/start-from-a-template.mdx
@@ -81,13 +81,6 @@ panel.
-
-
The project holds the connection, its example flows and contracts, and the
platform settings the route needs. What Ingestron cannot know, such as the
source's server, the tables to copy, the storage account or the integration
diff --git a/content/docs/how-to/(standards)/ask-the-platform-team.mdx b/content/docs/how-to/(standards)/ask-the-platform-team.mdx
index 216be6e..9938304 100644
--- a/content/docs/how-to/(standards)/ask-the-platform-team.mdx
+++ b/content/docs/how-to/(standards)/ask-the-platform-team.mdx
@@ -32,7 +32,7 @@ Each section appears only when it has something to say, in this order:
[platform prerequisites](/docs/reference/packs/databricks#platform-prerequisites).
- **Used by the definitions, but switched off:** features the definitions use
while their switch is off, such as a contract's `unique: true` without
- `manage.checks`.
+ `databricks.manage.checks`.
- **Shared infrastructure:** the platform's settings for the methods the
environment uses, with this environment's values. A secret reads "a
secret"; its value is never shown.
diff --git a/content/docs/how-to/(standards)/kits/databricks-lakehouse-streaming.mdx b/content/docs/how-to/(standards)/kits/databricks-lakehouse-streaming.mdx
index 6e00497..cf06c23 100644
--- a/content/docs/how-to/(standards)/kits/databricks-lakehouse-streaming.mdx
+++ b/content/docs/how-to/(standards)/kits/databricks-lakehouse-streaming.mdx
@@ -52,8 +52,10 @@ came from:
What the Databricks pack does with each value, with each feature's release
level and the date it was checked, is in
[the Databricks pack's intent](/docs/reference/packs/databricks#intent).
-`features: ga` keeps out features in preview, so the job alerts on failure
-but not on slow runs: Databricks' run duration alert is in Public Preview.
+With `features: ga`, Ingestron warns about any value that needs a feature in
+preview ([`PREVIEW_FEATURE`](/docs/reference/errors#PREVIEW_FEATURE)). The
+standard writes none: the job alerts on failure but not on slow runs, because
+Databricks' run duration alert is in Public Preview.
## Why each choice
diff --git a/content/docs/how-to/(standards)/upgrade.mdx b/content/docs/how-to/(standards)/upgrade.mdx
index 82f40d0..3be05df 100644
--- a/content/docs/how-to/(standards)/upgrade.mdx
+++ b/content/docs/how-to/(standards)/upgrade.mdx
@@ -365,8 +365,8 @@ some settings to be written in their new places:
[intent](/docs/reference/intent#settings-that-are-intent), such as
`history: latest` for `mode: upsert`. They still work, and `plan` reports
each as [`SETTING_MOVED`](/docs/reference/errors#SETTING_MOVED), a warning.
- In a flow or a table, its fix writes the intent and removes the setting;
- in the standard, write the intent by hand.
+ In a flow's or a flow table's `with`, its fix writes the intent and removes
+ the setting; anywhere else, such as the standard, write the intent by hand.
- **The `databricks:` section.** The platform's `workspace`, `mode`, `run_as`,
`compute`, `permissions`, `manage` and `grants` now go under `databricks:`,
and `notifications.on_failure` becomes an email alert in `alerts`. At the
diff --git a/examples/change-data/README.md b/examples/change-data/README.md
index df37345..b3cb86f 100644
--- a/examples/change-data/README.md
+++ b/examples/change-data/README.md
@@ -1,15 +1,17 @@
# Change data
The source files hold one row per change: a key, a time and an operation.
+The flow orders each key's changes by `changed_at`, with `sequence`.
-- `accounts` uses `mode: upsert`: [the table](build/databricks/src/crm/bronze/accounts.sql)
- keeps only the current row per key (`STORED AS SCD TYPE 1`); rows whose
- `operation` is `DELETE` remove the key.
-- `contacts` uses `mode: history`: [the table](build/databricks/src/crm/bronze/contacts.sql)
+- `accounts` keeps `history: latest`: [the table](build/databricks/src/crm/bronze/accounts.sql)
+ keeps only the current row per key (`STORED AS SCD TYPE 1`); a change whose
+ `operation` is `DELETE` removes the key, by `load.deletes.when`.
+- `contacts` keeps `history: full`: [the table](build/databricks/src/crm/bronze/contacts.sql)
keeps every version (`STORED AS SCD TYPE 2`), keyed by two columns in
`primaryKeyPosition` order.
Each also gets a `_changes` streaming table with the records as
-received. Keys come from the contract's `primaryKey` columns; `sequence_by`,
-`delete_when` and `except_columns` are settings of the `streaming-table`
-method. Without a key or `sequence_by`, loading fails with `METHOD_CHECK`.
+received. Keys come from the contract's `primaryKey` columns. `history`,
+`sequence` and `load.deletes.when` are intent; `except_columns`, the columns
+left out of the kept table, is a setting of the `streaming-table` method.
+Without a key or a `sequence`, loading fails with `METHOD_CHECK`.
diff --git a/examples/platform-setup/README.md b/examples/platform-setup/README.md
index 4c4ec6a..6037880 100644
--- a/examples/platform-setup/README.md
+++ b/examples/platform-setup/README.md
@@ -2,14 +2,14 @@
`platform.yaml` holds what a platform team sets up for a project:
-| Setting | Generated in `build/databricks/` |
-| --------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
-| `workspace.host`, `run_as` | each target of [databricks.yml](build/databricks/databricks.yml) |
-| `permissions` | each target's permissions, applied to every resource |
-| `compute.budget_policy_id`, `tags` | [the pipeline](build/databricks/resources/hr.pipeline.yml) |
-| `schedule`, `notifications` | [the refresh job](build/databricks/resources/hr_refresh.job.yml) |
-| `manage.schemas: true`, `grants.bronze` | [schemas.yml](build/databricks/resources/schemas.yml); the pipeline refers to its schema so the bundle creates it first |
-| `manage.schemas: true` | [prerequisites/prod.sql](build/databricks/prerequisites/prod.sql): the grants an administrator runs once in prod |
+| Setting | Generated in `build/databricks/` |
+| ------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
+| `databricks.workspace.host`, `databricks.run_as` | each target of [databricks.yml](build/databricks/databricks.yml) |
+| `databricks.permissions` | each target's permissions, applied to every resource |
+| `databricks.compute.budget_policy_id`, `tags` | [the pipeline](build/databricks/resources/hr.pipeline.yml) |
+| `schedule`, `alerts` | [the refresh job](build/databricks/resources/hr_refresh.job.yml) |
+| `databricks.manage.schemas: true`, `databricks.grants.bronze` | [schemas.yml](build/databricks/resources/schemas.yml); the pipeline refers to its schema so the bundle creates it first |
+| `databricks.manage.schemas: true` | [prerequisites/prod.sql](build/databricks/prerequisites/prod.sql): the grants an administrator runs once in prod |
-Without `manage.schemas`, no schema is generated, because schemas are often
+Without `databricks.manage.schemas`, no schema is generated, because schemas are often
shared infrastructure that one project must not change.
diff --git a/public/images/vscode/completion-dark.png b/public/images/vscode/completion-dark.png
deleted file mode 100644
index 19b8bac7bbd8e11492202ecd872631e90382cbf3..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 127136
zcmcG#RaBfy)FuohKnRvVa0?O$8r&_oySuvtcMB5SU4y%&aS!g+K;!N%jZ06Tlbr9L
z^{+J-b1{9_Z`WJ3d)KZl&weUQK~5a?E&f{=7#LJZ2@xe27z9BWm{*H$;Gj&VU1iy1k&
zs3CdE?SM2jkYQcv;VpfIkFKG2hSj{T^u6{*v!{+ad+*g)Y_(6-5AUt3q&@dd?VOc-
zDaogga_m2v_{rQ)RoZ
zqzgn@TxCEg1x;jTI2-0)E0=#SfLbMH68xw@CeFA8*zvw$erR|uo4eOGeR;EHso1^_UKH^3&FJZoPO2STNn4<
zT0fO7I!0VR{#}PrC-vKqQSId9P#p7u{X9GxYY#JS3$OY{6B}s(tpo*zXNLUoRFaP{
zxFN?b4n{}BB1Dv2i?Sszg+sRfKUpGQCk&aeXAoV>WqPQXn-}K`siJ4gP#$@7Z%G!y
zf~&HN^rP7%-7pZoDdys^KfBMD(_FfPzZc_S9~#2`5hU`Br#-{FHx@9W_OofpN?_`C
zRkx@DPdGa6j3RY#@~V}&;WRDWw+o{(Y-T)c<%QV4@^?vm1K;1`Fx{K6r^w#6_et+t+V4e0(`Jd6o+zpOFN;g`Rb;~Rf_c;<2yN({w%fsL87B}LJ
z7;*Oz11%*a{*E@-o?Moq!)Ceo5LISj&?nP~r~&3;+5Nd-Lm6GQj?`7yYr%>kJ}Ld*
za1kavY)mCCF$2Ku@wjRBJ8_aT9R@^t#0?A)0W6sl5go|ik>t@MJ8y&G@KxhZ=fkD#
z7vbSx9U@0UC{>_a5wGk0`tPtOSQZ9S<)0f^DHD5`;-0~z^Xp0>`~Mtu=%=fCax~nM
zS`=A_IOZ(Xq6d{%XE-(HHMjq(Kfj-2MpFjfJUZj+U3l<%acDHz|Bxo+|Hmu*^LYue
z{36jq%kU#G4)o^_y3F}r}^~B^kBNMwzgKPFg~-jp`jr>Jly4IE{n@4L?@Dy
znz|J1Czc-oqqAtjoHk|g;cf6=1t7fo(epJ&^x2{Ba;LnyBi)h#G&{TFf>*2IJ=YAj
z@w>VP3A}(Yn?rGlflZd`Dx<^$?5U&
z)8iUM?g>fu^X_<7tMh?UrJA;iN}cP8R+N~qn4Farjcj#=+7XVvcdgT&oM(RHoRDHS4<7zL*PmV(rz~wlFcZsi|_XqJ2=64v)4&
zULL7r)14WX$eWv+zqErf@N9rZviT67ttgKDQ(cdpB;{f`-FDAws2Cj&hvhzxH>iFY
z1M~A54{Ohlw+XOI0P_j@#^r9Cb264~ST+9=sxGTD8@BY;!EeoUpX^-3nCZP^y|nf$
z7?FHhy{2&=)--|%VX^-xD3g{rCM7jhBA*uDUD<6fA_|0Jd?|Mt|Bz_
zO_G$Tz(jq0J&)Vza=TaKP>=$~zfyL^v|aQQu(lhOK_kBipTvJw*Rn_*Orj*cC7n-LEWo^Za#`>*Iy@HL{+Y2a{DUaPJ@E3*{B
z?FCc3N7=U${ee
zXL|4^(pS!ged*=-F@})Kaltf&UcZBvme%^|?jnQ3{sn5`^Y|u=zd~}64CPVjUTffB
zqW!2K&T3<%4naYEFuj1n6
zlD)@?R<;BA1h*g9ybi7+L!`biq;auP>hw0cLCoqGXcia!TJ6lE#E*r9OV-AQ9mm{S
zMt7^_u-h9TsYga26AA%t+6ZjBXHVr8?)~2fTYW3}UihQQZZX^KZWJ+hjFfbDE2_wr
zhE5KBm;z@`8D5XQQ=RF|$=EL~CtZu*nK$1YWTEiHVMmE1Q+q7l;bqc1o@R$zMjK
z&E;dF7&R-T`R`sjX?H&%(l!&K0U&;hZGPgKE^RdD3&FsBXETo$ij@EXdp^N&9Dq;$
z!ewWsvRe5G2atJaz4@RhV;|J3iIWdi736-x?~nPy`2SYc)01;5uc)Xvs_x{cqdR_i
zJo`dN_XGyYOG`^5MF!SKD_B-p~yF7BD*%NNcs@hMx6pDF{4Z
zE^h>+4exj>`_m3LMar8~bFG47VWOv^>GMSMhl!&3bjG?anX7i{o*Y
z>TLYHhldc;i($b=-AxBB`)V!0J>~-p0OX4vyOXaGm`eSNA&_=+-{bjm=B^k=j(l
zq#m2i+81)A=P6>MK%F#Py0Gx9!^y*=U8wZ_kIkI3D3yY12Z+7a0}uz{X{o68E(S;-
zPgfH>JUq(E$`Gh?*Ku2Mc`epGIoB26;_N>=ttXBUDALtQ?exK6#^vE^?zwf?ifkKJ
zz2MU%U04x2x*W-bh(J}5$dB2&qt&q?9;Kys#jyfR=8hKr6oQ}Koz3RX*iQ_T!&C<$
zPJO(Tw+mWI&7~!IUqQ_unao_oFvk3=WaRU--64aPjr72_$Pn2hf(*5_-&2!
zw^Ik`aybg~Gs()p_3L-+SV=(CX&Eb}#D{M#6U;PWp|SkYxvzrtsU;)ApqdcN-`
zKf~7UMNuueR{xoLcMj>md3bYszSD-!ph0V>Ilp=Kr?@)GLt)p+twJd;@t04ZcGiCf
z{tYcpAvz3x;3@H~&CT$Lh@s$78}@4bPT$+pm2$PpbPoH~=|btF!$a5O`L7Io#{H4a
z8zBT18-NLZ9*@gy4|xHB9$W(#l=1U~hgw7lf0qC
zaSKP?LPe0rGtN<(M2Qse>xjTwM>vCk7Ev$5RqKK#Ty{GK5Qpk5qD|?ySf#w8tpO(2JQtwP{!B=!z
z8hqRfY+Xi_N;$-|25*suyB1xu<_w;8(6!S%wLev+pIu5dta)JNj1b`vZ#G}}VAZ2n
zITdN$zBHY*5~PU2FSe4>+E}$8J<^Y^;;`0-yp~Aj{?$-x_*>kXAWT59e0HzS+rr}W
zpN!W(&K2@}iwFb*=bKN9TrD}
za(PWj*=~eOpB$)q!^)^YVUqFo*NfA6&=GKDXI$3DHE)v5YYE85$0j2A=RCq7&r`o57l|-nnscH?+f9HNHC@n396$1yria;v^H;a)jpg{
zb6e!icG6P2LCX5P;nZwauG+gt-Al&JSDoAEbPdgW8oBV7O-0M(ul3dq>Qk&tq3YjK
z4RGFKdh1bGl;wUkR+^_TKjv&JS`r^>M0A44mm!ED-ZPYFP^d(w65jW=_M4P5E=yKN
zxOE|;*0{Zui>{(EBV(}gHGTVp;14HclYIm^UU^hhROo&h=ljim{F8Dwu|7XWarivF({~SYy?gof;wN=Qbv59uR&j;ThzVz
z{igCnY1OL|C5|hb<|?(_EE`A)OZYF%Lxbw6Bw{rmB}Zb)Gt=p5}{H?#G%CJ
zyp?jY!nYeHtf$~gF6HXmb2J|ipBgBCbQIuM6C6*eT*G`^ln}_vL(4yK*9A76ziYtS
zxKu;}ccOCpNLo)&J$<Y6zf4=r0Zh&L2+SW;Xj@oL4!+Fdp{5*V9|9&z?CEy
zJy?TNfN8e@=|HCFD#LOM>U`~|-Liml(X(t!!Be?kvz^#4RiX9ZA1r{S<Yim`-
zRz`zysi4;UzQ&*^cFBBiKPT&R0W3B!yrxOCr;g<5#Tlv9U+1&IG;KoP!=tR%K5}h+
zNwyZFi=-4`v%Amb5LH@Q8Z4~&(19~Ik#79o6M<6J)YL>^-^N5wAC-J`bkzBDInwh3
z9TS&Jcq#0Wanc#DbA-0zG9iZ@N1ajGtWPw~$%Ik64#5ka3FLsJxiGaMLt5P5ZeKK{
zXqY_`6H~(>QDCSX9CH&GY5Pf)HH|F#ACt&DLG0vS0OH%}8KLkvsCz_N6LcZP;oLWQ
zxwyDM9zdNkl*%L{lg-X}Efjx?b6g6k!pm`kld5nDJ
zJLQ}Mm!!%=jp@2~EN^ZCiwU$XhzA(Dj))Bz1wViD3Db0ldp^cE`ktJT@i-vxQrq0T
zaHPYX1x+2_{Y?U-nbRUyoA{p2d$!?7KfG11(75PE{xX`)w`5gsx7>6wB~DTZsRy5m
z6NvRHu#qN)VRh;}v}qWeZO~C0*=UU>HnKlWnE~Qq^QkA=L>@yC&UEX~`&PtnRA(GH
z3GvZa_^1QrShXvch)O4$A0EduC#u?Iy0-KL7}zKT+b(286FQvqguwO2%1zN94?2Iy
zxt}0D-vX#iO9DBtldd?IN0f|GxLSVYWv@7d>h3~{BHMj7EwoZGNnAj;e6}8yZ~qpw+M2*=eiq-&!p4W4hk+sat16R8*mxYwy{V0qLKr
z1{JNr?QLR0!c`xR{tO6&c}^gVWzbg&Oaq6%$H&F1mitmEAc}B&g&EBH8!0~YV5?d5
z^x3a7#)##f4B!e0Hf0GkwHAitwVB1~1yS@M&ODCv@6F@0_>rw6)@J>p)4DC@&|1>K
zJgMFa>fC$nKd{z6uch0XyqQe2%6RqHa3ME?}Jz=t7TLc>AF2OW(e@DI>n{6CeTf!E9(s)@#=xVU5i<>
zi)f%Quqr>7+Igh33yr!x3IBC9pSbRAuSxB67|G6VE5prc9B;b(R5#VYyk&Z7v-U>SAI$hq;VwW_6|8*Y1Y{+Kw!3kQt(1MC>AA0^yae{5R^tfZ{$
z_fPaWP_?x^LH^Pj7|2^65Yxv_u(G$;*IwF{UDyQ0Aq>TFpH
zK31|GJ(kS#aH8F>RSjByt1OmrHCjed_;
zr6Jf_p|OY}@Cd92th_-sQaqaQ
zpY&_6uiK*T_9Lr`?3>cI@K@m9JC_x6xrZk*){HmrA5csH0}~Da*1fzKer!h1h$u@s
z7Y{2!`kA_;2mV-q9m(HFYU1MT{13F#c|T^~q;qOr-M*`dZV?Js>U0g17nsu3ZE-GD
zE2pQS$va3A%O@o_e}1^eq?FO<@aCdFTL28GKT}8N|FEmoICiO0D$V3}ImA-eF$63?
z!>>34x3I>mBeGr1S4TRPQu6qfXj-WsdLkL`219gtMy%dMexCV@)%m~w4?YVkLHJ&z
zzUKXW9V758!aydh_X_#pe7_>hy9)5?7c@dhRR0Y`44SNtP;5)1#o5xv=J?(vWOi;&
zO;NGdZl#6Ad^|SdJzT|A9BBSmaW^p+Tc&-C3Rb!+QjR{bYDr1w{<;*|Ktl2FOt{>v
z?k%a@Rrt;pLT}cCD1-h!hvJVygMU#6=p$d0{{SHvm^-%D8h=N@tb-=Qcm&a%{o(&S
zS^uH%3b|{@4Dh$8gE<#?Qv;J(p+@`nz{5ZSVUW9!{+rF;PsksB9STkVH5`~b66oY|
z*b(qaHGkK5dIv}6PfVE*_V>Gg&R2i<$v^}R+=#zHqigMB7uHXPBN)Zs<=W(!UG9+;RQyih>I8-^%*J|LF^=
zS=9e@@&DKC|LKJ&`E`t*jmy^9QnURQax
z3bK0CEcmV49qorO@(HSGNVJ;%lA>5t-V{3h{?g$!J=^wowlj{i3lR)0&buiRtzr3C
z>7}0X-KeOTpTVeH_at_FA}}rRLj~0kD>nLho9!Y$Z;=p1A&poaqls}EeC;LYEE5IE
zAh>#^@kCkz?aBsezCMJ1lyffsrp8n8hq!6vh*H`2wBqY}RWG*}83yLej>RZPkEbDy
zmn2TOip2z;Xdj3?$RaWP7|T{#+|?qoy%`%xGCm^EcH>oyzWx=
zj2J)l(U#+yISa@(fQ@vY~&(9%h|n^{0RjNPRfzF{JV3Hx>#iu
zO_e1PgKZramW2CX2h8biXiA=iYJ+y^9F|IsC>YxROzn>h4@XD#Askz$L5eOeRywe%
z^jCXsCbkx>oTu*3Q*gG4xc{g4($^TxrGOQuv~w`tc>LJhx7}_#i#Z)vojjEM;Hn#J
z#~~hF&Fa`i0c2BDPJ+pZRf7r(oBROz;wh03GYSCIJgd9(gw#ek%<3??Q5@4z;M~sI;9hYm9
zMm*E;^b0k`ARTjTW4GwZ8lEf_Y116kaBU&PCDiiAI?urrT$;~K$nI5
z{2xWuW!xRRh0*AeaZzcdg!Rn@G&Bi8Yw0D0}&oA1~HRiKrsl82nZ$kN_+gmK(*w
z9z1f`YxVAj`}2%NBTgAnD(v;`=hw-}Xzd^uV_{W@+r4x80pqL{H#aKP$k9u?>n*wZ
z7&2;I&a9tzV`bH>^mjt6iY6aJ*Gw&}kFNcCEeyuKsyzhOZjV+|V%2RPY1E)mT>0Cv
z`ttm_=<&?FVKziLc1xGz-kFj~#H!RWe$!VeYr5@S7R;@7H&HHLa;h^P9m~obUr^&0
zjvV4He`yt}Whu63z~
ztyUN7)=XSG*?Wb4M>JNiJBCi#c$}Y{9o&4GkSg8_I?&7N=xuzAhB%*pz~i-?G6X-n
zWJt*0!o)Zo`LRhT2EWD1zVtXDO=&Wfm4b#w&Ip;A$iMmU5raqks~)&Kt0odPIjG
z{=5_NsZLOSZ;RwN|czX9^&pBOeNXS@eceO%ph#x#eGS;t`-zObNCK7P7@jh^CAb91f
zMQ~AhDVQq$HI>Qu^YxO~nvVuZXQ$aTJ>zwdNP=%US8Wth&3^JHDYmFtojqnIOw$?5
z*2NX^vQWKz(=znnYGh6+3I@`Sm)_UZL!73QJti&ECpduMSpp12MMCk4Cx@|yQ3U
zGRLA@%>ufey(fv&AI)a#YL}vO9HQ*2vg15v!K)yy0e;ri(a2s0Bt?0@+S0O8bscF;
zJkP4s%ZkWeefq488w`%|GzmaWBl{_NAcYKd1d1!kVz+BUcF8`$^Lmog!g(|)7SgoQ
z%$K$H!}=UoqR*eHo@+ckDV685)Hkt}*)6&E`+73Z@3)lZ2*K0&B&c^nSg@mLpgN2B
zgTRc7f8$Hnxgciu``wamWwo5n&cfp$$+HT9!ceKSixH4dr4Z&P%J#!)R<;yk7H2m;
zIpvH4xAtwV_hGZE3Su~B#MoO4wCQX~gr?RA!Nn8K86#rSQKF|SjXZ05a2RD1)`npckZQA8Pu7M-5
z5tQ_+l}a&KJUUHUOv}>837|H?TQTRfPnoex$J(AJ4xF4*R9P}%<*LTRr(I@Tde&8=
zHHoRpo+wg}!e1G>)QTGSWh)X1VLnJwcUI<@aDwt3@b#i>#UskCwKGX^WNowCe8bF_
zx(N16pz@BDIZBtg<%g+up5rP}4+gy=m8hEEiz;v-e{&vqw-8Nk=VW^WOO|EG
z$7fQlZQ$i6psL8zG=~1~9vj@{BFstDXjuDSTTWjh0_7d!mrj&pNcA`m7ilkB0rj8D+xGT~ZA
zlr&xY`J`whZ9cRJy{(=QwCmV7n$!EiLgVT%hG&7BlQpM>jHd^5lLQ;TqnznvDRXt4
zA^>1ZRmvC2X)=2++hIX@IWkfVxoXu@->R!~FZz~OanoGQ-Z~!sR!v91>$^5zb$rXN
z_}WS7^6l?H4(56oRu4q_erIJ2
z#`?;&g{_g*r6K3#{v<)5T?*mVScVV=730qI^IENW$o$byB8K?~Smh#*937%F7i4hP
zt=~Crg5G;^cJ;a=ry@GFpm=1wsHk}}J4!$>0=7)r?g!EUW>+Rre@<*>6kI1HjL)
zr{64_`}?i(&x=Vxv1jAiOh>U-gv0`jykS|y?86KBPTHm&`9$(DB^ME?LdJ?{V;Rfr
z62gUx3FG@zoQAvyz%UYh*QF*Zz|g|HR2FkEO5ysQOoL9#8Bb#~)(_AxSva5i=!%K8
zy*Wof?)gV$gO-ikvhwl~tJ<2F=xDS}^STFy#?w(t_Fh-MQ|`|jsq@)dx&oeHK@oEL
zO<}Ry@SM{@Ch)0D7N=6ioU1t9*Ee?s{Ycj;ou=adZ)outMR{PF%C#ghATfA38jvH+2zUp@yBy@B?v8KuZgZC1t}
z*U?c{sim{*?ea0-$#JNUnppfr+l(zK7;mS;BxnAj!G9g%D?aqGJ9`>4K7KlgRu2
zx{Ul@X7f*!RWTMlR_yg0iOsS$bcY8&_&;)%Rkp;UL@E|DS!AT?&GrDzEsnyo9tpGh
zKUhIcwUSJ=Bu(hsQxo28+b(y+R*5LtAwR3A2*yRFGO(nQmed?Lr9i2^MDR{B@vyY<
zx!q)~aqN>gy%{;44cMC*-cu;j^VZAq5hh}6R)*vr_V=pS@NMLkjyU~xM1jo?o%+ts
z4oJs~{=%4TOfLj^hCt@6_!dv0u>`-ENGyQ-Te^pbYhkae*|r$oN5gWG%0Xg)g||FC
zf!r-h^Ni+&l}g=WeTRHTmGaJf0GDyXz4z+wa(7<3Sh+1{US#`yA@-4`r;ZYB=4^yq
zBFJ-`PftzwHj%5h#2^@Ch!N8h^oH>l<-sM8o3_fkQ2ifI`%-1(c=Hcsdc#TFxDoY
z^ML~lotkJMC;*(%{}{pKdWGDAf=Pc*7W?{!sS!|_4Ho|0fKJYA#^QEAoC*$O^7|Ow5T2gS
zGCRP|@qzd_Kcmu#OqHuV92dN5{e2S01wn)6tXOdU`4@LbFL0vV)H?GV
ztfmw?{tyotZQdC~!GtMg`5XbNdf>HW5S&uRcT4I?V~5ksObN4Z9Tr9r&SZxwuvub|
zY7$qMl(NbXx0X_2q_HuCZ|BVFgNi2%%v>7SQd>M%+*@yUOqiToT%>l2&O)Wul!PC0
zrGf7I2?DZhyXZFG#0cB_fK5^Sk=$be=)Tp~rIfEI5BOR0t6`+~j=Lo(ZdWT~`kW0B
zRLqT!?daXrXwm@RbE`(Yu(%q!;qf
z(^iEA_OvBs{FvWLL?%l*v|z)d*o(5uY3xi!U@%6!bqr(ewhFBY|8)?{?Bt}|=RIr3
zXm-M(;)DJfMF$`d6Qd_kSD%|+%m;!6(0_8J#e=h%*!20ikfUJ%dAN2FN@aQ%c1AHM
zP@8l>v7$TEYM8*IueclK^eYuEiY^A6e|0S@j+_cZt3VTuq^ic=F!hE`W#g%&O29k)
zPjM*413O%tqr@CVBiux?vcF
zx;)mT?O&*`%41sB?zKn7%ok0oe-%*n3_u~Uk3R1P@Inb$+-ANnrJ?n`R`L+(KTc0r
z8tR(24&o9c{ebYfo#A6VB1`ai12^32jO*>@HoMwKVakiu6>Uu)WNox~No{WMHX)wO
zE);S&2;}sf(c-oQPfH%iWK|$ZDk{bO6fguTSO8r@kWVU4zVECGjAP`oz?4}}#eWrQ
zsf73cz9~i@pY?^lDZw^mpE6aNyloI^uyQkGiU}c
z0>4Ciq|46qTp9J4EaQ9w1Xaue+Kv7cOTlPR?Iiqgi3u|UctGm|_-@R{Rk$e2AlSG0
zRq8CCG%e_~i4$F&BGVIipjzg15K_=VP<9wKVy^LNX+4}~mkcDlt%{>Ip_n8p=WVHF
zw+FSY`F8bsILjjt8ysk5uGmDp3QQ|XY~OQA&8G~+CsMMif3Wv^BzUYVD(b&J2o65M
zrct0NLT@~6r=l8g*Bz<9y6g>IPiEO8G-%FCn=0mUyunH2^BGYxlWODsTs`e&t=7=!
z7H1&QL6K2Mv=M!p7f&X{yC+rxw?Je0Y#BQ*pPx5#tIsszGTaF|T}@DzLXg)jXJLGs
zEbXCFrEPvg=mknPU>522eXwYoF2t*oi#Tc}XAdJ|
zh1P>b%Syh6bR8T2VFA-e-IYn>QyowUF84hF9;sQN-Wn0tMGywFLHZ$Fksz>q{~}^}
zH>6nB_^87pnw|3-t}VoaV3&4!nE^^%iLUFb7L20a@`Gh`5iC3RU^
zsR+>qKt|kwfB@~LvT9hwac#>ifTN~O33ExS5{5OjP`7DraGA6+Kz-FP;LMQ1;%4(}
zX?du$R%Sr2_d={`tWa~K?Nqg6%`Fe2>pfKx;UQm02D>6;aNCtAKtliVUj=-t74&CY
zuDf2jCgRNOJt)^TADX=|R)0zc3wO`Z>Xz$JlPO!8^(L0b*g7oVp`!t66Hb))Xn=SI
z_hmq5PoOq?M83KWn)&e9L?|!*vcrA%-7)Rn(}Qh*h#Ugl)v!1;fBw1ToWXz`W|+a<
z074hS8MmE
zC{>}b8Vk}oj+Yxk19#eNgCBGBl2nKiyWC)Kxn|d~?2;Ed;dS}&NpqFPb;>_5s`3Fh
zr>by8bLl9Y8ZY|`_csAT&Y@k47(KQ%m*uX=`E$JdESFHU4=T7JMaB1|N-Dtl$z2-)
zci=KzeV)06#Dm;Qy*rKa+#zcAvo8U$qO`|_@yp-siQg$x#s_Ab1lF|R6ES*)0
zgq-w`e-3^l`{o1=tLy;aWi=N{G5+?w;rpo+T?cv|pNZ(Glf?k`z1nG+l6twkU+93O
zf1pff&y|A}-VO<@3`6u>Z*?SxzKsj>{#Napj%NZsQtv#VN!|3vnC!E-Y&C|YyX9~UuQ)Jhc$5<>mnEytRgVIH{
zgl-=5tGEh0)(i>P1WjaX0Q167RZC5LX5&Z7D+Dz4cqqqLC7e#@w}+@}5#J6=A7*d`
zVUl1c;$mmMvs>F>Fs@GgF|k$HS8@<2C!nRItGA}Kk{1^ystnv@#?0iKoUL!TjYB;d
z`63%$<7Qi=HoV#bX~|pcZ-|!Lb$)U`S5Z|az{73--E?x7cAs|B-{4?1OM9zRf287&
zpq4^R#Lh;a{fvKSaYV}O2rZrg%3Jfrm#0<3`==|+_VGvY8BWLx=|*Sv{CD%SSC(Jt
zwmaPi!gY(Kqz==KXUi7wBxO3UdmWgX+OcCsG}$d#m0!3TCvCVIYjN0#hl;(Hyiy*@
ze6L$)k(F9f_F7*jvfi?~ad?eZmq7*^m;SUD?+{;WxWWYpl3hwau5D?rb|%g-)0o1<2L3pzP(Fef&$No$AOY1zgxXa^
zbBCmLZr*~ZMO*pvNfVch*r*y?5j2RRlCS}{8$Bhz0<9^AYnrIl--!qEcz*OD8R?o*
zN0LmJAjt+qS9>fNIahxtMUHmmu~M^e#FiX4$_dA^V(+q|8!zQ<3FBJQ>G){2i6NBS
zR3*JupN@&M-F{kTxz$!wv_W&fli;a!7%|h+NAoWi0ll{mzOY}{ZbB1u%j|A2
zryNH&g+-(?mdh*jnv$5ty#gpGRzG18-k9e$B85-v+jKqY0oE1>weZ~WL~U|}B2)AI
z2pZ4=b}1TXjKJI;<@5U0@tL|=_3e@1S9sa_W%eeEr_Z}^KM`NXvW>S1iRkDWwal4Q
zumSIj_I&mCcj&L#<;YeLNgeclYz&iH^8LAwvxf*h`xSG(_052V@k%BCFGR9Qk86H2
zKV8n?`b&VvK6|7Me$E+vn=UT8%2-idnNim*nld56IlP)4CV_jG*-2&IzdX3miK4Ku
zg{dCvPQj=A9ESE1O}8g&u2%T%pH%B1BZ^bHGEDP?I#$ImY2&L2|M*M?YG8;F8Nd2yrg4V&&kH-79xEOxz^XAHedwc
z8voi||018=Y{h&^euj^+ZuQ8t1v?41a_P}!cHM5WiooSKx()f)>V2lCUB9(yir`by
z)<%?wnBlL-c)#W!u6?FWS)WgzNkGKIS2{Z798ViP^#ptvJsq8;^F&uJrS2X2KI+bg
zer%r`#~0TYn_@bODFJ;>f%a8vkHhIi+1eUVheCZkPJ6(b?F26yG3#1aVE!if`(a2&
zhMv~r1+nJi?Vo#2UFYis1@m4Z?D&XI6fXP4T{XR3p%h_4D{8uvO1|Gi6nyA1c^?rT
zK1QQhOe~?l6K83k+?K>PR#zTU^s%PDVA1&pa!`EFCG|P(i0%3z8Q)e{a&5q!_)aK{
z%qmd@j}i@qiGatl1+=O$w=gFdg5pD}>deh|-?Do+H{2#S_O4@2ln$BzAw^HcIlQ@V
zyCrFhOY^-F;6EkjgraCe?{kqheiI5EE-gZ<&>d8IpI(%&%$D97v09|cuO%#4)mAE~
z?QNidZFMz|C!Hec@Jxw9l8lzpl5b1vykZp_HAB1I6_H*5>`FF}7a!GCdJR-p_TTnv-e|<`tHL@#bC_raUw6K`f
zNner)kxywTWw*x(2yi$t@X~SXRM8c4TAS5_?)=3s{D+Uv!0B9)1vU3)L%=QPD*NX#
zjAvQtR$qPJ1x*}iH|up#thQ1V_|KVW6T%dN!KK{7%r`NLTM{8oaXNY$lHuws%kKI_
zsX&8$>A^Z#5EIitO!;pX0<-!P0UHJ-0n+L%_ESYgW2NcV*-}$D;LQQBy3!cZIy#us@BRBu|y<1RMKlA=;!0$^Ke3K-$
zy2Kf_NBm5*KkxYC1;6{>NEOtSUEQDms{vbmSK7fbWcbB|>vhxge)dG$F08M8mu
z1ie3q2%C%!4(l{blCWEwerwb
zY)obLS@S%L{*!lEUi2UB!}vqK`ilw)^&}{X{}AtU+*D+M@}=#RKwzw9olupv
zq2qi^?dCvOeSGI?5|i-nc0=hjv%=LNwmDE;J*%6`fvKaY)q_TWphR)Rr}&CvO~f0U
zPrhOxm4#IWIpj(#s-eQlf~p0J2KOw}L%tQ8TBuKKrlC!?|&S0Uwo;`;<8>pau
z!d=_MXg{6@y$%|lOrF05XX2!D_EnXb(_rG`by>XErj2;uP
zU8?J+Z_lH{zS{8NnJ+H;3An}a&0YWI@hcX3!OoXqV&~%3_xuCT*MxVxb|pfT>$kVB
z|L$FvAoQ3qrA2m_S+Z5CWkek;>*9UTQB@QQ(NkESuch@JadDp1^V>C8eSkEL%ci=j
zI`0ZKs%p{5Wog~oS8F574&wukL3NW_F5I_EIAUX+^mMi@Dp+!ah^G4%d^alsP{z(X
zl4bPBN_32WXxjuzpcdWP>jnsB`G;-0?zw9<^)_p6>JB9Ym|B|m@
z+y(t-aTrnt&}8RJfpNLwS49;^InG7WC-mkGe#Z^|%vLIb608RtwPY$EaAlCgL$^n1
zNpdX{k95*yWSe)=jMq9eG8#v0MTzw}?{|xnrY!jqI0-c^MtiLxlREKUbD*f0n3R#7
zV_;1UqhmluEbjGBDBPtK_mN)g`EsRP@6i$z@ZE7w1x<0N5DbAj;BkBdS=k74>|F;%
zqQ950x_FjZZbh*OyOT5Uj|Sf%6R;)8R&y`m|I;r}F7~R{hEkz)(8;xc{6=OCGJ)!Mqyb{Lb6Mf;XzG5UYTz13Z;6Asw7r;U^;S2vV{7`M%H4s|O{WdK_`+0n1@&
z1lp3JT?wE-XHQ2s8QFr%P#t_D3w%c3XpwqiRrBG#`UD1~XU_rNJ9|&~^H01f!7rEp
z^fSC)E9-fEj`^R4h91Gtd(zt68ya;=Kw-V^gbX1j&gww2xG0x6!X$yrJw@=ve^`Ll
zGuKS){O$8sjF6HQwWiob-Y1{L4wN<{wqog$rsYh!13Mku^kFGtQ3adq=p#6(`qv7`$5{Pu<3z=0BH>T$V;7M$xEwdt{MazRH!Z!WZ0jxr;|
zPLOhaxt)h?PmA*5k7-RPMVN!14#kQX*;l!OM>f3gCS`N@Jx2!6*GAbzP4=)s?EgcX
zgq3~MzJ75J*!{9|;a+D`tKn+DW>Bv+G#N_VDS`XbU^pwi|C!_|T8zQu4U`3_oZ=+_
z*3~k%BYV3)tDT^*dFr5C;=dm^DpTp^3>ybBzpXJHK0HAjbq2x$`j8C
zPbGwe<_1v$Ji>hq=GZj%iaJj%EkgPNQJ#d~(6}OF*SF(HNg+x?Q}J^LoLY0bGxc9+
z^_MRR*op4h7*7R=P`XjRZ*8>QpF!Wa3xxLBB;kbNdN2kj8Qomn)8_b(F;RES7K0UrbBkWIBh3;7C=g?Y1(6ULONytkG{%R%*29Sn&oq|=&9bZ;NM->%cAJ7E(nNYA*_*EK@E+D+Y8fg1
zL6xm5!~50#Tdsq&>yU}k$x%n>tunvIv4V#rDV`@ARYT6gg{1IC0F?HaMtwL#t+=(D
z5NFKfT1(Uzft8z$+@vfUkc6DGO5*$?PFdm00pA7oz{$S2gvzmCK@c9@%*`g>jaQZySyu3r>*$+Pv
zd%jCg`Hsn)#!%Z&2W53^nwT#!Hs~FXEf?ZQoDd0=<^hkcZkxwYQRKOgu%UGAj#}e^
zU-D3*J7jlv7XxC0>_-O@;RcQ?{9v_T3;cgrAB(m8~5Gt|%sNDMIu41)}{FYo8}zn-;Tt+n@OuXzc~
z#mp7wdBpGcJ@!q_g+&Lyy28Fq#gpo<3c*)qf;?p-dya@_MycKnn&>PhQ;~@X^XonJ
zp4}EPGEx1z%L-0+)+no}QH#$Cmz%xP3|={+0qwi!4Hu|iJ5&|R@IK;n2TEO@LeJ1h
zZ$3Tl4_`d+m+bD~$N_z^S~ms`4vuL#9UammWmO}4qN`F(38H9uHI=a8O`t`vFOVj$
zDSbPhC2jiD+ueMskjHpmlOncnvhbjYkdB$XKOt;B`}xJ3A&kuhjAR-@B6<
zi=CJxEjfcfmPJ36f_`{jL0&OH#(%AWC<8#m_n^-
z>z?N$7Wda^Nq&13EdxI}RzW%bSVGsahA=?6sl{fq&u$y~v$9t^qejnNXflAiYe?SS
z-6}Bm>RrQy&b_O{m8s&M6{`epS90owg^IZ8zt=NRibNX36F_dP7a?cbz7DR6nVHd}
z6R(RPs0JGBKvRa`T`yv%HgBhWatc`(ood2-aUle5|!9V9(skVZbCIaFGgL)@ldEzOo1dPpvWn*j}
zbJeT6L{>w9TrVgK%2KGeI