diff --git a/ci-templates/azure-devops/azure-pipelines.yml b/ci-templates/azure-devops/azure-pipelines.yml index 5326bce..5b58a29 100644 --- a/ci-templates/azure-devops/azure-pipelines.yml +++ b/ci-templates/azure-devops/azure-pipelines.yml @@ -33,7 +33,7 @@ stages: version: 22.x # An exact version: change it when you upgrade, as # https://docs.ingestron.io/docs/how-to/upgrade#choose-the-version-you-run says. - - script: npm install -g ingestron@2.3.1 + - script: npm install -g ingestron@2.4.0 displayName: Install Ingestron - script: ingestron check displayName: ingestron check diff --git a/ci-templates/azure-devops/ingestron-check.yml b/ci-templates/azure-devops/ingestron-check.yml index 0e4dab4..ee04b68 100644 --- a/ci-templates/azure-devops/ingestron-check.yml +++ b/ci-templates/azure-devops/ingestron-check.yml @@ -15,7 +15,7 @@ steps: version: 22.x # An exact version: change it when you upgrade, as # https://docs.ingestron.io/docs/how-to/upgrade#choose-the-version-you-run says. - - script: npm install -g ingestron@2.3.1 + - script: npm install -g ingestron@2.4.0 displayName: Install Ingestron - script: ingestron check displayName: ingestron check diff --git a/ci-templates/github-actions/deploy.yml b/ci-templates/github-actions/deploy.yml index 7c6c959..bd2ae6c 100644 --- a/ci-templates/github-actions/deploy.yml +++ b/ci-templates/github-actions/deploy.yml @@ -29,7 +29,7 @@ jobs: - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 22 - - run: npm install -g ingestron@2.3.1 + - run: npm install -g ingestron@2.4.0 - run: ingestron check build: diff --git a/ci-templates/github-actions/ingestron-check.yml b/ci-templates/github-actions/ingestron-check.yml index e238faa..47ba79f 100644 --- a/ci-templates/github-actions/ingestron-check.yml +++ b/ci-templates/github-actions/ingestron-check.yml @@ -18,5 +18,5 @@ jobs: - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 22 - - run: npm install -g ingestron@2.3.1 + - run: npm install -g ingestron@2.4.0 - run: ingestron check diff --git a/components/mdx.tsx b/components/mdx.tsx index a344809..8cf48d4 100644 --- a/components/mdx.tsx +++ b/components/mdx.tsx @@ -37,6 +37,7 @@ import { ZonesNotApplied, } from "@/components/reference/pack-intent"; import { KitFacts, KitFile, KitIntent, KitList } from "@/components/reference/kits"; +import { RecipeCatalog, RecipeList, RecipePlaceholders } from "@/components/reference/recipes"; import { ErrorCodes, LoadingCodes } from "@/components/reference/error-codes"; import { ExtensionCommands } from "@/components/reference/extension-commands"; import { ExamplesList } from "@/components/reference/examples-list"; @@ -106,6 +107,9 @@ export function getMDXComponents(components?: MDXComponents) { KitFacts, KitFile, KitIntent, + RecipeList, + RecipePlaceholders, + RecipeCatalog, CliCommands, BuiltInTemplates, ProjectSources, diff --git a/components/reference/connector-page.tsx b/components/reference/connector-page.tsx index 7c85f5e..1f52979 100644 --- a/components/reference/connector-page.tsx +++ b/components/reference/connector-page.tsx @@ -20,6 +20,7 @@ import { ConnectorSettings } from "./connector-settings"; import { ExamplesList } from "./examples-list"; import { Inline } from "./inline"; import { NativeReadSettings } from "./native-read-settings"; +import { ConnectorRecipes } from "./recipes"; import { SettingsSections } from "./settings-sections"; /** Words only one connector needs, placed in one of its page's slots. @@ -55,6 +56,7 @@ export function ConnectorPage({ id, children }: { id: string; children?: ReactNo {section.id === "routes" && } + {section.id === "source-metadata" && } {section.id === "examples" && } {section.items.map((item) => "slot" in item ? ( diff --git a/components/reference/recipes.tsx b/components/reference/recipes.tsx new file mode 100644 index 0000000..fb1cdc0 --- /dev/null +++ b/components/reference/recipes.tsx @@ -0,0 +1,359 @@ +// Source metadata recipes, from the installed @ingestron/standards package: +// the list, each recipe's section, and the short section a connector's page +// shows for the recipe its connections use. +import { Fragment } from "react"; +import { ServerCodeBlock } from "fumadocs-ui/components/codeblock.rsc"; +import { Heading } from "fumadocs-ui/components/heading"; +import type { SourceRecipe } from "@ingestron/core"; +import { KindBlock } from "@/components/code-block"; +import { TableFrame } from "@/components/table-frame"; +import { codeThemes } from "@/lib/code-themes"; +import { + connectorRecipes, + howBlocks, + kindWords, + placeholdersUsed, + readerWords, + recipePage, + recipes, + reservedPlaceholders, + type ConnectorRecipe, +} from "@/lib/recipes"; +import { connectorEntries, connectorIds } from "@/lib/reference"; +import { Inline } from "./inline"; + +/** A connector's name, as the packs title it. */ +const connectorTitle = (id: string) => connectorEntries(id)[0]?.connector.title ?? id; + +/** The connectors whose page shows a recipe, and how each uses it. */ +function usedBy(recipe: SourceRecipe): { connector: string; how: ConnectorRecipe["how"] }[] { + return connectorIds().flatMap((connector) => + connectorRecipes(connector) + .filter((item) => item.recipe.id === recipe.id) + .map((item) => ({ connector, how: item.how })), + ); +} + +/** How a recipe names a table. */ +function namesWords(recipe: SourceRecipe): string | undefined { + if (!recipe.names) return undefined; + const { parts, caseSensitive } = recipe.names; + return `${parts} ${parts === 1 ? "part" : "parts"}, ${caseSensitive ? "case-sensitive" : "not case-sensitive"}`; +} + +/** + * Every recipe installed with Ingestron, linked to its section: what it reads, with which + * reader, what it asks you to fill in, and the connectors that use it without being told to. + * @example + */ +export function RecipeList() { + return ( + + + + + + + + + + + + {recipes().map((recipe) => ( + + + + + + + ))} + +
RecipeGetsInputsUsed by default for
+ {recipe.title} ({recipe.id}) + {kindWords[recipe.kind]} + {Object.keys(recipe.inputs).length === 0 + ? "None" + : Object.keys(recipe.inputs).map((name, index) => ( + + {index > 0 && ", "} + {name} + + ))} + + +
+ + ); +} + +/** The connectors a recipe is the default for, linked, or "None". */ +function DefaultFor({ recipe }: { recipe: SourceRecipe }) { + const connectors = usedBy(recipe).filter((item) => item.how === "default"); + if (connectors.length === 0) return "None"; + return connectors.map((item, index) => ( + + {index > 0 && ", "} + {connectorTitle(item.connector)} + + )); +} + +/** + * What the request fills in where a recipe writes a name in braces: the reserved names the + * catalogue uses, and the recipes' own inputs. + * @example + */ +export function RecipePlaceholders() { + const used = placeholdersUsed(); + const unknown = [...used].filter( + ([name, ids]) => + !(name in reservedPlaceholders) && + ids.some((id) => !(name in recipes().find((recipe) => recipe.id === id)!.inputs)), + ); + if (unknown.length > 0) + throw new Error( + `Recipes write placeholders that nothing fills: ${unknown.map(([name]) => name).join(", ")}.`, + ); + return ( +
    + {Object.entries(reservedPlaceholders) + .filter(([name]) => used.has(name)) + .map(([name, meaning]) => ( +
  • + {`{${name}}`} is . +
    • + ))} +
  • + Any other name, such as {`{${firstInput()}}`}, is an input of the recipe: the + value the connection gives in metadata.with, or the input’s default. +
    • +
+ ); +} + +/** The name of one recipe input, for an example. */ +function firstInput(): string { + const name = recipes().flatMap((recipe) => Object.keys(recipe.inputs))[0]; + if (!name) throw new Error("No recipe has an input."); + return name; +} + +/** The recipe's query or call, in a block that says it is not a command for the shell. */ +function Step({ recipe }: { recipe: SourceRecipe }) { + const { run, call } = recipe.ask; + const code = (run ?? call)!; + return ( + ( + + {children} + + ), + }} + /> + ); +} + +/** A command a recipe's `how` shows, in a block of commands. */ +function HowCommand({ recipe, text }: { recipe: SourceRecipe; text: string }) { + return ( + ( + + {children} + + ), + }} + /> + ); +} + +/** One recipe's section: the request's step, its inputs and where it comes from. */ +function RecipeSection({ recipe }: { recipe: SourceRecipe }) { + const inputs = Object.entries(recipe.inputs); + const names = namesWords(recipe); + const users = usedBy(recipe); + return ( +
+ + {recipe.title} + +
+
Recipe
+
+ {recipe.id} +
+
Gets
+
{kindWords[recipe.kind]}
+
Reads
+
{readerWords[recipe.read]}
+ {names && ( + <> +
Table names
+
{names}
+ + )} +
Who
+
{recipe.ask.who}
+
Access
+
{recipe.ask.access}
+
Save as
+
+ {recipe.ask.save} +
+ {users.length > 0 && ( + <> +
Connectors
+
+ {users.map((item, index) => ( + + {index > 0 && ", "} + + {connectorTitle(item.connector)} + + {item.how === "default" && " (by default)"} + {item.how === "fits" && " (if the source fits)"} + + ))} +
+ + )} +
+ {(recipe.ask.run !== undefined || recipe.ask.call !== undefined) && } + {recipe.ask.how && + howBlocks(recipe.ask.how).map((block, index) => + block.command ? ( + + ) : ( +

{block.text}

+ ), + )} + {inputs.length > 0 && ( + + + + + + + + + + + {inputs.map(([name, input]) => ( + + + + + + ))} + +
InputWhat to writeDefault
+ {name} + {input.description}{input.default === undefined ? "None" : {input.default}}
+ + )} +

+ Documentation: +

+
    + {recipe.sources.map((source) => ( +
  • + {source.url} (read {source.read}) +
    • + ))} +
+
+ ); +} + +/** + * Each installed recipe's section, with an anchor of the recipe's id. + * @example + */ +export function RecipeCatalog() { + return ( + <> + {recipes().map((recipe) => ( + + ))} + + ); +} + +/** + * The short section on a connector's page: the recipes its connections use, who to ask and + * what the recipe needs. + * @example + */ +export function ConnectorRecipes({ id }: { id: string }) { + const found = connectorRecipes(id); + if (found.length === 0) throw new Error(`No recipe is shown for connector "${id}".`); + return ( + <> + {found.map(({ recipe, how }) => ( + +

+ {how === "default" ? ( + <> + A connection with no metadata uses the{" "} + {recipe.title} recipe. + + ) : how === "named" ? ( + <> + Name the {recipe.title} recipe in the + connection’s metadata.from: {recipe.id}. + + ) : ( + <> + If the source is one the {recipe.title} recipe + describes, name it in the connection’s metadata.from:{" "} + {recipe.id}. + + )}{" "} + It asks someone to save {recipe.ask.save}. +

+
+
Who
+
{recipe.ask.who}
+
Access
+
{recipe.ask.access}
+ {Object.keys(recipe.inputs).length > 0 && ( + <> +
Inputs
+
+ {Object.keys(recipe.inputs).map((name, index) => ( + + {index > 0 && ", "} + {name} + + ))}{" "} + in metadata.with +
+ + )} +
+ + ))} + + ); +} diff --git a/components/reference/vscode-commands.json b/components/reference/vscode-commands.json index 8998e6d..5e02f39 100644 --- a/components/reference/vscode-commands.json +++ b/components/reference/vscode-commands.json @@ -58,5 +58,20 @@ "command": "ingestron.draftContract", "title": "Draft contract from sample", "category": "Ingestron" + }, + { + "command": "ingestron.requestSourceMetadata", + "title": "Request source metadata", + "category": "Ingestron" + }, + { + "command": "ingestron.importSourceMetadata", + "title": "Import source metadata…", + "category": "Ingestron" + }, + { + "command": "ingestron.addAssistantFiles", + "title": "Add assistant files", + "category": "Ingestron" } ] diff --git a/components/reference/vscode-extension.json b/components/reference/vscode-extension.json index bf38d5b..d9f6f21 100644 --- a/components/reference/vscode-extension.json +++ b/components/reference/vscode-extension.json @@ -1,3 +1,3 @@ { - "version": "2.2.0" + "version": "2.3.0" } diff --git a/content/docs/(reference)/reference/cli.mdx b/content/docs/(reference)/reference/cli.mdx index 70f78f2..48ec962 100644 --- a/content/docs/(reference)/reference/cli.mdx +++ b/content/docs/(reference)/reference/cli.mdx @@ -19,8 +19,9 @@ Run the command from the folder that holds `project.yaml`. It reads the project there and writes to `build` there. Four options adjust the run: - `--env dev` limits `compare` and `request` to one environment; repeat it - for more. `plan`, `apply` and `check` always work on every environment - together, and refuse it. + for more. `source` and `contract --from-metadata` take it once. + `plan`, `apply` and `check` always work on every environment together, and + refuse it. - `--definitions ` reads a project that is not the current folder. - `--out ` writes somewhere other than `build`. - `--no-diff` lists changed files without their line differences. @@ -62,6 +63,67 @@ combined with `--template` or `--standard`. See platform's. It always exits `0`. See [List them](/docs/how-to/choose-a-published-standard#list-them). +## Source metadata [#source-metadata] + +Ingestron never connects to a source. `ingestron source` takes one of two +commands, and each works on the project in `--definitions`. +[Describe a source](/docs/how-to/describe-a-source) uses both, then drafts +the contracts that are needed: + +- `ingestron source request` prints what to ask of whoever can reach each + source: for every connection a flow reads, the query to run or the call to + make, who can do it, the access it needs and where to save the result. It + writes nothing. `--connection ` limits it to one connection. The recipe + for each kind of source is in + [Source metadata recipes](/docs/reference/recipes). +- `ingestron source import ` reads ``, a path from + the current folder, and writes `sources/.yaml`, and adds that to + the imports of `project.yaml` when it is not there. It keeps names and + types, and never a value from a row. `--dry-run` prints what the file would + change and writes nothing, and `--object ` names the table a file of + one table describes, when its path does not name it. + +Both take `--env ` once. Without it they use the first environment +`project.yaml` declares, and `default` when it declares none. An import with +`--env` also records the environment as the document's `environment`; without +it none is recorded. + +`ingestron contract --from-metadata .` drafts a contract for each +table named from the imported metadata, and with `--needed`, for every table +that needs a contract and names none. It resolves for the environment `--env` +names, or the first one. It writes `contracts/.
.yaml`, +never replaces a file that is there, and prints the `contract:` line to add to +the flow. A draft says what to review: its types, which columns are required +and its key. + +The document an import writes is in [Documents](/docs/reference/documents#source-metadata), +and the codes `plan` reports about it are in +[Errors](/docs/reference/errors), as the ones that start `SOURCE_`. + +## Project guide [#project-guide] + +`ingestron init --assistant ` and `ingestron assistant add ` +write a project guide for coding assistants. Every name writes `AGENTS.md`, +which holds the guide, and `copilot` and `claude` write one more file that +points to it: + +| Name | Files | +| ------------------------------ | ---------------------------------------------- | +| `copilot` | `AGENTS.md`, `.github/copilot-instructions.md` | +| `claude` | `AGENTS.md`, `CLAUDE.md` | +| `codex`, `cursor` and `agents` | `AGENTS.md` | + +`none` is the default and writes nothing. Without `--assistant` and without +`--yes`, `init` asks last, in a terminal: an empty answer is `none`. Without a +terminal, or with `--yes`, it asks nothing and writes nothing. + +The guide is written from the project's own files, and `plan` and `apply` keep +its Ingestron section current. `check` never looks at it. +`assistant add` takes `--definitions`, the project's folder, and `--out`, its +output folder, and needs `project.yaml` there. It exits `1` when there is no +guide to write or a file could not be written. See +[Work with an AI assistant](/docs/tools/assistants). + ## Output `plan` and `apply` print one block for each output folder under `--out`, such @@ -70,11 +132,11 @@ as `databricks → build/databricks (dev, prod)`; see ## Exit codes -| Code | Means | -| ---- | ------------------------------------------------------------------------------------------------------------------------------- | -| `0` | Success. Warnings, such as a pack's notice, do not change it. | -| `1` | Problems in the definitions, or output that is not up to date or in conflict. `check` also fails while a value is `REPLACE_ME`. | -| `2` | A mistake in the command, such as an unknown option or `--env` given to `plan`, or a pack that cannot be loaded (`PACK_LOAD`). | +| Code | Means | +| ---- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `0` | Success. Warnings, such as a pack's notice, do not change it. A `source import --dry-run` that would write, and breaking changes an import reports, also exit `0`. | +| `1` | Problems in the definitions, or output that is not up to date or in conflict. `check` also fails while a value is `REPLACE_ME`. `source` also exits `1` on a refused file, an unknown connection or recipe, a recipe that breaks a rule (even one no connection uses), or definitions that cannot be loaded or have errors that stop the command; `contract --from-metadata` on an unknown flow table, a table with no entry in the source metadata, a table that already names a contract, or definitions that cannot be loaded or have errors that stop it. `contract --from-metadata` does not read recipes. | +| `2` | A mistake in the command, such as an unknown option or `--env` given to `plan`, or a pack that cannot be loaded (`PACK_LOAD`). `source` without `request` or `import`, `source import` without a connection and a file, or with more, `--env` more than once on `source` or on `contract --from-metadata`, `--from-metadata` with neither a table nor `--needed`, or with both, and a flag given to a command it does not belong to are mistakes too. | ## Editor schemas diff --git a/content/docs/(reference)/reference/documents.mdx b/content/docs/(reference)/reference/documents.mdx index 9bd78ba..9e59c40 100644 --- a/content/docs/(reference)/reference/documents.mdx +++ b/content/docs/(reference)/reference/documents.mdx @@ -1,6 +1,6 @@ --- title: Documents -description: Every field of project.yaml and the Standard, Platform, Connection, Flow and Transform documents. +description: Every field of project.yaml and the Standard, Platform, Connection, Source metadata, Flow, Transform and data contract documents. --- Each document starts with `apiVersion: ingestron.io/v1`, its `kind` and an @@ -63,6 +63,29 @@ See [Connectors](/docs/connectors) and [Reachability](/docs/start/choose-a-route A connection's `timezone` is the zone its timestamps are converted from; see [Cleanse data in the right layer](/docs/how-to/cleanse-data#add-the-columns-to-cleanse). +A connection's `metadata` says which [recipe](/docs/reference/recipes) gets +its source metadata: `from` names the recipe and `with` gives its inputs. A +connection with no `metadata` uses its connector's default recipe, if it has +one. + +## Source metadata [#source-metadata] + +A `SourceMetadata` document says what the tables of one connection look like +at the source: each table's columns, their types and its key. Nothing in it +is a promise, so it has no checks, descriptions or classifications. +`ingestron source import` writes it, as `sources/.yaml`, from a +file someone who can reach the source saved; see +[Describe a source](/docs/how-to/describe-a-source). A connection has one +document. + +A flow table that names no `contract` takes its columns and key from its +entry here. A table that does name one is compared with its entry, and +a change at the source is reported before the data stops landing; see +[`SOURCE_COLUMN_GONE`](/docs/reference/errors#SOURCE_COLUMN_GONE) and the +codes that start `SOURCE_`. + + + ## Flow diff --git a/content/docs/(reference)/reference/meta.json b/content/docs/(reference)/reference/meta.json index a887259..6b5f919 100644 --- a/content/docs/(reference)/reference/meta.json +++ b/content/docs/(reference)/reference/meta.json @@ -1,4 +1,4 @@ { "title": "Reference", - "pages": ["documents", "intent", "packs", "errors", "cli", "templates", "versions"] + "pages": ["documents", "intent", "packs", "errors", "cli", "recipes", "templates", "versions"] } diff --git a/content/docs/(reference)/reference/recipes.mdx b/content/docs/(reference)/reference/recipes.mdx new file mode 100644 index 0000000..ab90460 --- /dev/null +++ b/content/docs/(reference)/reference/recipes.mdx @@ -0,0 +1,39 @@ +--- +title: Source metadata recipes +description: How to get the tables, columns and types of each kind of source. Who to ask, what to run or call, what to save, and the documentation each recipe comes from. +--- + +Ingestron never connects to a source. Someone who can reach it runs a query, +calls an API or exports a file, and `ingestron source import` reads the file +into the connection's [source metadata](/docs/reference/documents#source-metadata). +A recipe says how to get that file for one kind of source. +`ingestron source request` prints each connection's recipe filled in for your +project, ready to send to whoever can reach the source; see +[Describe a source](/docs/how-to/describe-a-source). + + + +A connection with no `metadata` uses the recipe its connector is listed for +above; any other connection names its recipe in `metadata.from`, and gives the +recipe's inputs in `metadata.with`. Both are +[fields of a connection](/docs/reference/documents#connection-metadata). + +## What each recipe asks for [#recipes] + +In a query, a call or a path, the request fills in what is in braces: + + + +A placeholder the request cannot fill is left as written, and the request says +what to write in its place. + + + +## Use your own recipe [#your-own] + +A file `recipes/.yaml` in your project is a recipe too, and replaces the +installed recipe of the same id, so a team can correct a query without waiting +for a release. Every recipe in the project is read whole, so one that breaks a +rule stops `ingestron source request` and `ingestron source import`, whichever +connection uses it. VS Code shows it as a problem at its file; see +[Problems with a recipe](/docs/tools/vscode#recipe-problems). diff --git a/content/docs/(reference)/reference/templates.mdx b/content/docs/(reference)/reference/templates.mdx index 32ddde8..fae4e91 100644 --- a/content/docs/(reference)/reference/templates.mdx +++ b/content/docs/(reference)/reference/templates.mdx @@ -4,7 +4,7 @@ description: The built-in starters and how to create a project from one, the tem --- A template is a project folder with a `template.yaml`. `ingestron init` copies -the folder into a new project, writes its `README.md`, and asks nothing. A +the folder into a new project, writes its `README.md`, and asks nothing about its values. A starter is a template that ships with Ingestron: a complete project for one common situation, with `REPLACE_ME` where you write your own values. See [Share templates and standards](/docs/how-to/share-a-template) to make a template of your own. A @@ -49,7 +49,7 @@ Each question: | `ask` | What the value is | | `default` | The value written, unless `--set =` gives another | -The built-in starters have no questions: they write `REPLACE_ME`, with a +The built-in starters ask no question about their values: they write `REPLACE_ME`, with a comment, where a value is yours to give. Any other field is an error, so a misspelt one is found at once. Your editor diff --git a/content/docs/(reference)/reference/versions.mdx b/content/docs/(reference)/reference/versions.mdx index 65f5b2f..71c4d37 100644 --- a/content/docs/(reference)/reference/versions.mdx +++ b/content/docs/(reference)/reference/versions.mdx @@ -29,5 +29,5 @@ Connector maturity, such as Public Preview or Beta, is shown on each The [deployment templates](/docs/how-to/deploy-through-environments) pin the tools their stages run: `@microsoft/azure-data-factory-utilities` 1.0.3, the Databricks CLI v1.23.0, and sqlcmd (Go) v1.10.0. Their check stage installs -`ingestron@2.3.1`, and each GitHub action is pinned to a commit, with its +`ingestron@2.4.0`, and each GitHub action is pinned to a commit, with its version beside it. diff --git a/content/docs/(reference)/tools/assistants.mdx b/content/docs/(reference)/tools/assistants.mdx new file mode 100644 index 0000000..cc875bc --- /dev/null +++ b/content/docs/(reference)/tools/assistants.mdx @@ -0,0 +1,260 @@ +--- +title: Work with an AI assistant +description: Give your coding assistant a guide to your project, written from its own files and kept current by plan and apply. +--- + +A coding assistant does better work in an Ingestron project when it knows the +project's rules: change the definitions and never the generated files, plan +before applying, work inside the standard, and never write a secret. +Ingestron writes those rules, with the facts of your project, into +`AGENTS.md`, the guide file that Codex, Cursor and many other assistants read, +and points GitHub Copilot and Claude Code at it with one small file each. It +writes them on your computer from the project's own files. It calls no +service, reads no secret, and works with whichever assistant you already use. + +## Add the guide [#add] + + + + +Name the assistants when you create the project, with `--assistant`. From the +parent folder, run: + +```sh +ingestron init shop --template duckdb --assistant copilot,claude --yes +``` + +Each file it writes prints a line after the project's own: + +```text +Created shop from the template duckdb (12 files). +AGENTS.md: written +CLAUDE.md: written +.github/copilot-instructions.md: written +``` + +Without `--assistant` and `--yes`, `init` asks last, in a terminal: + +```text +AI assistant files (copilot, claude, codex, cursor, agents or none) [none]: +``` + +An empty answer writes none. Without a terminal, `init` asks nothing, and +writes the guide only when `--assistant` names the assistants. + +For a project you already have, run from its folder: + +```sh +ingestron assistant add agents +``` + +```text +AGENTS.md: written +``` + +`assistant add` takes the same names, separated by commas or spaces, and +`--definitions` when the project is in another folder. Run again, it prints +`AGENTS.md: up to date`. + + + + +**Ingestron: New project** asks, after what to start from and before the +folder, **New project: AI assistant files (optional)**. Tick any of **GitHub +Copilot**, **Claude Code**, **Codex**, **Cursor** and **Another tool that +reads AGENTS.md**, or press Enter with nothing ticked for none. + +For a project you already have, run **Ingestron: Add assistant files** from +the Command Palette, or right-click the project's folder, or one of its files, +in the Explorer. Tick one or more. The Ingestron output panel lists each file, +and a notification offers **Open AGENTS.md**. If a file could not be written, +a warning says so, and the output panel says why. + + + + +Commit the files with the project, so the whole team's assistants read the +same guide. Ingestron does not add them to `.gitignore`. + +## The files for each assistant [#files] + +Every name writes `AGENTS.md`, which holds the guide. Two write one more, +small file that sends their assistant to it: + +| Name | In VS Code | Files | Why | +| --------- | --------------------------------- | ---------------------------------------------- | ------------------------------------------------------------------------------------- | +| `copilot` | GitHub Copilot | `AGENTS.md`, `.github/copilot-instructions.md` | Some Copilot features read `.github/copilot-instructions.md` and not `AGENTS.md`. | +| `claude` | Claude Code | `AGENTS.md`, `CLAUDE.md` | `CLAUDE.md` imports `AGENTS.md`, so the guide is read whatever else the folder holds. | +| `codex` | Codex | `AGENTS.md` | Codex reads `AGENTS.md`. | +| `cursor` | Cursor | `AGENTS.md` | Cursor reads `AGENTS.md`. | +| `agents` | Another tool that reads AGENTS.md | `AGENTS.md` | Any other assistant that reads `AGENTS.md`. | + +`CLAUDE.md` holds two lines: + +{/* prettier-ignore */} +```markdown kind="plain" title="CLAUDE.md" + +@AGENTS.md +``` + +`.github/copilot-instructions.md` links to the guide and repeats three of its +rules. It holds no fact about the project, so it never goes out of date: + +{/* prettier-ignore */} +```markdown kind="plain" title=".github/copilot-instructions.md" +See [AGENTS.md](../AGENTS.md) for this Ingestron project. + +- Change the definitions, never the generated files: Ingestron reports a hand edit there as a conflict. +- After a change, follow AGENTS.md: run its plan command, fix every problem it reports, and show the user the plan before applying. Its check command must then pass. +- Never write a password, token, key or connection string in any file. Use `${secret:name}` for a credential. +``` + +When a file is already there, Ingestron adds to it and keeps what it holds: +the guide's section goes at the end of `AGENTS.md`, and the lines above at the +end of `CLAUDE.md` or `.github/copilot-instructions.md`, each only when the +file lacks it. Nothing else is written: no rules, prompts or skills folders. + +## What the guide says [#guide] + +The guide names no assistant, so one text serves them all. For the DuckDB +starter it starts: + +{/* prettier-ignore */} +```markdown kind="writes" title="AGENTS.md" + +# Working on this Ingestron project + +This folder holds Ingestron definitions: YAML that generates the files the data platform deploys. Ingestron runs on this computer; it never connects to a source or a cloud. + +## This project + +- Project `shop`. Environments: `dev` (`project.yaml`). +- Standard `defaults` in `standard.yaml`. Packs: `duckdb@2`. +- Definitions: the files `project.yaml` imports: `standard.yaml`, `platform.yaml`, `connections/*.yaml`, `flows/*.yaml`, `transforms/*.yaml`, `contracts/*.yaml`. +- Generated, one folder per pack: `build/duckdb/` (a folder per environment). +``` + +Then come: + +- **Rules:** change the definitions, never `build/`; run `ingestron plan`, fix + every problem, and show you the plan before `ingestron apply`; work inside + the standard, with the methods it allows and the settings it fixes, and + never edit it to clear a problem; ask you for each `REPLACE_ME` value; look + settings up in these docs or the editor schemas; never write a secret; how + this project's `${envvar:…}` values are supplied, and never to read or + commit `.env` files; and that environments differ only in values. +- **Source metadata,** when the project has a connection; see + [below](#source-metadata). +- **Commands:** `plan`, `apply`, `check`, `compare`, `request` and `contract`, + each with what it does, and the two `source` commands when the project has + a connection. + +Ingestron reads the facts from `project.yaml` and the files it imports, +without resolving them, so a project with problems still gets a guide. It +never reads a `.env` file, and no value of a secret or an environment +variable appears in any file it writes. With `--out`, the guide names that +folder in place of `build/`, and each `plan`, `apply` and `check` it writes +carries the same `--out`. + +## Keep it current [#keep-current] + +The guide is the section between the two marker lines. Ingestron owns it; the +rest of `AGENTS.md` is yours, for your own notes. The end marker records a +hash of the section as written, so Ingestron can tell whether anyone has +changed it since. + +When the facts change, such as an environment added to `project.yaml`, a pack +or a fixed setting in the standard, `ingestron plan` says so, after +everything else it prints: + +```text +AGENTS.md: the Ingestron section would change +``` + +`ingestron apply` rewrites the section, and only the section: + +```text +AGENTS.md: updated the Ingestron section +``` + +If someone edited the section by hand, `plan` and `apply` leave it as it is. +`plan` says: + +```text +AGENTS.md: the Ingestron section was edited by hand, so apply will not update it. Move your notes outside it, or delete both markers to keep it as your own. +``` + +and `apply`: + +```text +AGENTS.md: the Ingestron section was edited by hand, so it was not updated. Move your notes outside it, or delete both markers to keep it as your own. +``` + +Move your notes outside the markers, and `apply` keeps the section current +again. If the guide's own text was changed, delete the section, markers and +all, and run `ingestron assistant add agents` to write it afresh. To keep the +edited text as your own instead, delete both marker lines: `plan` and `apply` +then leave the file alone. + +The guide never changes the exit code of `plan` or `apply`, and +`ingestron check` never looks at it, so neither a build nor CI fails over it. +`CLAUDE.md` and `.github/copilot-instructions.md` hold no facts, so they are +never rewritten. + +## Source metadata [#source-metadata] + +When the project has a connection, the guide gains a **Source metadata** part. +It names the connections and tells the assistant: + +- each connection's tables are described in `sources/.yaml`, which + `ingestron source import` writes, and that file is never edited by hand; +- `ingestron source request` says what to run or export at each source; +- a flow table needs a contract only where a person must say something the + source cannot, and `ingestron contract --from-metadata --needed` drafts the + missing ones for you to review. + +It also tells the assistant how to turn material no reader takes, such as a +vendor's data dictionary, a spreadsheet, a PDF or a description in an email, +into a columns file, `sources/.assistant.csv`. The file has the +eight fields of the columns file, keeps every table and column the material +shows, writes a type the assistant is unsure of as `string`, never invents a +key and never copies a value from a row. The assistant checks the file with +`ingestron source import sources/.assistant.csv --dry-run` +before it says it is done. + +Then you import the file, as any other; in VS Code, a file saved under +`sources/` is imported as soon as it is saved. Once the guide has this part, +the source request offers the assistant's columns file as one of the ways to +describe a source. Ingestron trusts it no more than a sample; see +[A coding assistant's columns file](/docs/how-to/describe-a-source#assistant). + +## Keep the project at the root [#root] + +Ingestron writes the files in the project's folder, next to `project.yaml`. +Assistants look for them from where they start, and some read them only at +the root of the repository, or of the folder open in the editor. So the files +work best when the project's folder is that root. If the project is a folder +inside a larger repository, open that folder in the editor, or start the +assistant there. + +## What each line means [#lines] + +`init --assistant` and `assistant add` print one line for each file: + +| Line | Means | +| --------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| `: written` | The file was new, or empty. | +| `: up to date` | The file already had what would be written. | +| `AGENTS.md: added the Ingestron section` | `AGENTS.md` was yours, with no section; the section was added at the end. | +| `AGENTS.md: updated the Ingestron section` | The section was out of date, and was rewritten. | +| `CLAUDE.md: added @AGENTS.md` | `CLAUDE.md` was there without the import, which was added at the end. | +| `.github/copilot-instructions.md: added the link to AGENTS.md` | The file was there without the link; the lines above were added at the end. | +| `AGENTS.md: the Ingestron section was edited by hand, …` | Nothing was written to `AGENTS.md`; see [Keep it current](#keep-current). The other files still are. | +| `: not a regular file, so nothing was written to it.` | The path is a link, a folder or something else that is not a file. The other files are still written. | +| `: could not be read (), …` or `could not be written ().` | The file system refused, with its error code, such as `EACCES`. The original is left as it was. | +| `No guide: project.yaml cannot be read.` or `… has no id.` | Nothing was written: fix `project.yaml` first. | +| `No project.yaml in .` | `assistant add` found no project in the folder; run it there, or give `--definitions`. It exits `1`. | + +`assistant add` exits with `1` when there is no guide or a file was not +written, and with `0` otherwise, an edited section included. The names and +options are in [the command line reference](/docs/reference/cli#project-guide). diff --git a/content/docs/(reference)/tools/meta.json b/content/docs/(reference)/tools/meta.json index a612fab..6a368e8 100644 --- a/content/docs/(reference)/tools/meta.json +++ b/content/docs/(reference)/tools/meta.json @@ -1,4 +1,4 @@ { "title": "Tools", - "pages": ["vscode"] + "pages": ["vscode", "assistants"] } diff --git a/content/docs/(reference)/tools/vscode.mdx b/content/docs/(reference)/tools/vscode.mdx index 26f12e8..01bc9d2 100644 --- a/content/docs/(reference)/tools/vscode.mdx +++ b/content/docs/(reference)/tools/vscode.mdx @@ -1,6 +1,6 @@ --- title: VS Code -description: Write Ingestron projects in VS Code with problems as you type, navigation, completion and previews. +description: Write Ingestron projects in VS Code with problems as you type, navigation, completion, previews and source metadata. --- The Ingestron extension works on any folder with a `project.yaml`. It uses the @@ -101,7 +101,13 @@ When a problem has an obvious remedy, the light bulb beside it, or **⌘.** - each allowed value, and a missing field with its usual value; - for a method that cannot read a source, each method that can: for this flow, for every such source in the standard, or from a pack the standard - does not name yet, which adds the pack too. + does not name yet, which adds the pack too; +- for a table that needs a contract (`CONTRACT_NEEDED`) and that the source + metadata describes, **Draft a contract from the source metadata**, which + writes the draft `ingestron contract --from-metadata` would, and names it + in the table's `contract`. When that file is there already, the fix is + **Use contracts/…**, which only names it. See + [Describe a source](/docs/how-to/describe-a-source#needs-a-contract). A fix may change several files; one undo reverts it. You choose the fix: each one means something different, such as one flow or the whole standard. The @@ -192,6 +198,39 @@ environment. caption="A new, empty flow file filled from the skeleton: the id comes from the file name, and Tab moves to the source, offered from the project's connections." /> +### From the source metadata [#completion-from-source-metadata] + +Once a connection's [source metadata](/docs/how-to/describe-a-source) is +imported, completion offers what it lists: + +- **A flow table's `object`** offers the tables of the metadata of the flow's + connection, each with how many columns it has and when it was captured. +- **Column names** come from the table's metadata in an expectation's `check`, + in the table's `sequence` and `load.cursor`, in + `cleanse.deduplicate.order_by`, and as a new key under `cleanse.columns`, + where a column already written is not offered again. Outside a `check`, the + columns of the table's contract that the metadata does not list follow. + `sequence` and `load.cursor` written for the whole flow offer none, as they + apply to several tables. +- **A contract's property names** come from the metadata of the flow tables + that use the contract; a property inside another offers the columns inside + that column. +- **A connection's `metadata.from`** offers each recipe, with its title, and + a new key under `metadata.with` offers the recipe's inputs not yet written, + each with what it is. See [Source metadata recipes](/docs/reference/recipes). + +A column's name goes in as the text around it reads it, while the list shows +the name as the source has it. In a `check`, which is Spark SQL, a name that is +not a plain identifier, such as `Order Date`, or that is a reserved word, such +as `select`, goes in backticks, `` `Order Date` ``. In YAML, a name that YAML +would read as something else, such as `true` or `1`, goes in double quotes. + +YAML cannot start an unquoted value with a backtick. So when a `check:` is +still empty and the name needs backticks, completion opens a double quote +first, writing `` check: "`Order Date` ``, and you close the quote. A name +holding `: ` or ` #` cannot be part of an unquoted `check` at all; put the +check in double quotes before you choose such a name. + ## What each line becomes For the environment in the status bar, the editor shows beside each line what @@ -204,6 +243,10 @@ it resolves to: - after a value with `${…}`, the value it resolves to. Secrets and environment variables are never shown. +On the first line of a source metadata document, `sources/.yaml`, +it shows `Written by ingestron source import. Do not edit: import again to +replace it.` + Turn these off with VS Code's `editor.inlayHints.enabled` and `editor.codeLens` settings. @@ -218,6 +261,13 @@ setting the standard fixes, with the file and line. Hover a setting under `history`: `latest`, from the table's layer in the standard (`standard.yaml:18`). +Hover a flow table that names no contract, on its key or its `object`, to see +the columns the source metadata lists for it: each with the type the source +gives it, the key's columns marked, and the columns inside a column below it, +up to 50 lines. The hover ends with when and from which file they were +captured. A table whose `object` holds an environment variable shows none, +since which table it is depends on a value the editor does not show. + Hover a value that is still `REPLACE_ME` to see the field it fills, what the field is, where to find the value, and a suggestion of what it looks like, such as ``. A value in `project.yaml`'s `values` is explained @@ -240,10 +290,11 @@ The **Ingestron** tab in the secondary side bar, on the right, opens with a problems, previews and the platform request show. Click an environment to select it, as in the status bar. - **Check**, **Plan**, **Apply**, **Compare environments**, **Prepare platform - request** and **Show generated files**. Click one to run it for the project. - Check, Plan and Apply work on every environment together; Prepare platform - request and Show generated files use the selected one. Each item's tooltip - says which. + request**, **Request source metadata**, **Import source metadata…** and + **Show generated files**. Click one to run it for the project. Check, Plan + and Apply work on every environment together; Prepare platform request, + Request source metadata and Show generated files use the selected one. Each + item's tooltip says which. With several projects, each has a row, named by its `id` and showing its folder in the workspace, that holds the same lists. Every row starts folded except @@ -256,7 +307,7 @@ one to the file adds it to the list. @@ -325,6 +376,64 @@ pack needs on the platform and the SQL to run once, such as Databricks' `prerequisites/.sql`. Nothing is sent or saved until you do so. See [Ask the platform team](/docs/how-to/ask-the-platform-team). +## Describe a source [#source-metadata] + +Ingestron never connects to a source, so someone who can reach it sends the +names and types of its tables as a file. Two commands, and saving a file, +cover that; [Describe a source](/docs/how-to/describe-a-source) walks through +it. + +- **Ingestron: Request source metadata** writes what to ask for, for the + environment in the status bar: who to ask, the query to run or the call to + make, filled in for your project, and where to save the result. With more + than one connection, choose one, or **All connections**. The request opens + as an unsaved Markdown document, ready to send; nothing is written to the + project. +- **Ingestron: Import source metadata…** reads a file into + `sources/.yaml`. Run it on a project, and choose the file; or + right-click the file in the Explorer, such as a CSV someone sent. A file + outside every project is imported into the project of the file in the + editor. When the file's place does not name a connection, it uses the + project's only one, or asks which; and when the file describes one table + and does not say which, it asks for the table's name. + +A file saved or copied under a project's `sources/` folder, where the request +says, is imported once the save settles, without the command. A hidden file, +such as an editor's swap file, is left alone, and so is a file over 50 MB, +with a warning that names the command, which takes any size, and +`ingestron source import`. + +An import records the environment in the status bar, and is one edit of the +project's files, which one undo reverts. A notification gives the first line +of what changed, such as `sources/erp.yaml: 3 added, 0 changed, 0 removed.`, +with how many changes break something, and **Show all changes** for the rest. +Files imported together give one notification, such as +`Imported 5 files: 7 tables changed, 1 breaking.`, whose **Show all changes** +opens one document with a section for each file. A file the import refuses +gives a warning with the reasons, and nothing of it is written; several give +one warning, `3 files could not be imported.`, with **Show reasons**. + +### Problems with a recipe [#recipe-problems] + +A connection's `metadata` names the recipe that says how to get its metadata; +see [Source metadata recipes](/docs/reference/recipes). Three problems about +recipes are shown in the editor only. `ingestron source request` and +`ingestron source import` stop with the same message instead; `plan` and +`ingestron contract --from-metadata` do not read recipes. + +- **`SOURCE_RECIPE_UNKNOWN`**, at the connection's `metadata.from`: + `The connection "" names the recipe "", which does not exist. Recipes: .` + Name one of the recipes it lists, or add your own as `recipes/.yaml`; + see [Use your own recipe](/docs/reference/recipes#your-own). +- **`SOURCE_RECIPE_INPUT`**, at a key under the connection's `metadata.with`: + `The recipe "" takes no input "". Its inputs: .` Rename + the key to one of the inputs it lists, or remove it. +- **`SOURCE_RECIPE_BROKEN`**, on the first line of a project's own + `recipes/.yaml`: the recipe breaks a rule, and the message, which + starts with the file's name, says which. Correct the file as it says; see + [Use your own recipe](/docs/reference/recipes#your-own). While a recipe is broken, no other recipe problem + is shown, and completion offers no recipes. + ## Plan, apply and check **Ingestron: Plan**, **Ingestron: Apply** and **Ingestron: Check** run the @@ -338,26 +447,29 @@ Palette. Each acts on the project you started it from: - **Right-click** a project folder, or a YAML file in a project, in the Explorer for **Ingestron: Check**, **Plan**, **Apply**, **Compare - environments** and **Prepare platform request**. The files in `build/` do not - have the menu. + environments**, **Prepare platform request**, **Request source metadata** + and **Add assistant files**. A project folder also has **Import source + metadata…**, as does any file that is not one of a project's definitions. + The files in `build/` do not have the menu. - **Editor title buttons**: with a YAML file of a project, or a preview of generated files, open, **Plan** and **Apply** are buttons in the editor's - title bar. **Check**, **Compare environments** and **Prepare platform - request** are under the **…** menu beside them. + title bar. **Check**, **Compare environments**, **Prepare platform + request** and **Request source metadata** are under the **…** menu beside + them. - **The Project view** lists the same actions; see [Project view](#project-view). .yaml`, and that has no +table matching its `object`, or the project has none. + +**Fix:** run `ingestron source request`, save what it asks for and import it, +as [Describe a source](/docs/how-to/describe-a-source#request) shows. Check +that the table's `object` names the table as the metadata does. Or name a +contract with `contract`. + +### A table needs a contract [#contract-needed] + +**Symptom:** [`CONTRACT_NEEDED`](/docs/reference/errors#CONTRACT_NEEDED): + +```text +flows/orders.yaml:7:5 error CONTRACT_NEEDED + "orders" needs a contract: it has its own expectations. + Name a contract for the table with `contract`. +``` + +**Cause:** the table names no contract, but a person must say something about +it that the source cannot. The message gives the reason: + +- its layer's `does` lists `cleanse`, `check` or `quarantine`; +- access is managed for it: the standard governs its access, and the + platform manages grants or masks for its pack, unless its layer sets + `access: { contract: false }`; +- as here, it has its own `expectations`. + +A table that is an output of a data product needs a contract too, since the +product's output port names the contract its table uses. A port whose +contract no table uses is +[`PRODUCT_CONTRACT_UNUSED`](/docs/reference/errors#PRODUCT_CONTRACT_UNUSED) +instead. + +**Fix:** name a contract for it. When the source metadata has the table, the +hint gives the command that drafts one, +`ingestron contract --from-metadata .
`, and VS Code offers the +same as a quick fix; see +[When a table needs a contract](/docs/how-to/describe-a-source#needs-a-contract). + +### A change at the source fails the plan [#source-changed] + +**Symptom:** after an import, `plan` and `check` fail with a code that starts +`SOURCE_`, such as +[`SOURCE_COLUMN_GONE`](/docs/reference/errors#SOURCE_COLUMN_GONE): + +```text +flows/orders.yaml:6:43 error SOURCE_COLUMN_GONE + Column "ordered_at" of contract "shop.orders" is not in the source metadata for "orders.csv". + Remove the column from the contract, or import the source's metadata again if the column is still there. +``` + +**Cause:** the table's contract and its source metadata disagree in a way +that would break a load: a column gone +([`SOURCE_COLUMN_GONE`](/docs/reference/errors#SOURCE_COLUMN_GONE)), a type +that no longer fits +([`SOURCE_TYPE_CHANGED`](/docs/reference/errors#SOURCE_TYPE_CHANGED)), a +required column that can now be null +([`SOURCE_NULLABLE`](/docs/reference/errors#SOURCE_NULLABLE)), another key +([`SOURCE_KEY_CHANGED`](/docs/reference/errors#SOURCE_KEY_CHANGED)), or a table +a catalog import does not list +([`SOURCE_TABLE_GONE`](/docs/reference/errors#SOURCE_TABLE_GONE)). + +**Fix:** change the contract to what the source has now, or, if the import +was wrong or old, import the source's metadata again. To make these warnings +you can accept instead of errors, set `sources: { breaking: warn }`; see +[When the source changes](/docs/how-to/describe-a-source#changes). + ## Environment variables and `.env` ### A variable is not set diff --git a/content/docs/concepts/definitions.mdx b/content/docs/concepts/definitions.mdx index 520a646..841a10f 100644 --- a/content/docs/concepts/definitions.mdx +++ b/content/docs/concepts/definitions.mdx @@ -8,11 +8,14 @@ selects environments and lists the files to load. A definition only participates when an `imports` pattern includes it. @@ -22,16 +25,17 @@ to produce another table when the selected packs support it. ## What each file owns -| Document | Responsibility | -| -------------- | ------------------------------------------------------------------------------ | -| `project.yaml` | Project ID, imports, shared values and environment overrides | -| `Standard` | Packs, allowed methods, defaults, naming and fixed policy | -| `Platform` | Shared infrastructure and operational configuration supported by the packs | -| `Connection` | Source connector, reachability and access settings | -| `Flow` | Source objects, destination table definitions and ingestion choices | -| `DataContract` | Expected columns, logical types, keys and other supported contract information | -| `Transform` | Declared input tables and code that produces another table | -| `DataProduct` | Who owns a set of published tables, their purpose and support, in ODPS | +| Document | Responsibility | +| ---------------- | ------------------------------------------------------------------------------ | +| `project.yaml` | Project ID, imports, shared values and environment overrides | +| `Standard` | Packs, allowed methods, defaults, naming and fixed policy | +| `Platform` | Shared infrastructure and operational configuration supported by the packs | +| `Connection` | Source connector, reachability and access settings | +| `SourceMetadata` | What one connection's tables look like at the source, as an import wrote it | +| `Flow` | Source objects, destination table definitions and ingestion choices | +| `DataContract` | Expected columns, logical types, keys and other supported contract information | +| `Transform` | Declared input tables and code that produces another table | +| `DataProduct` | Who owns a set of published tables, their purpose and support, in ODPS | A data product is an Open Data Product Standard file, usually in `products/`, whose output ports name the contracts it publishes; see @@ -42,9 +46,15 @@ their method IDs. Capabilities still depend on the selected packs. ## Link definitions by ID -A flow's `source` names a Connection ID. Each table's `contract` names its data -contract; `object` identifies the source object. These are different identities. -A transform's `inputs` names tables produced by the project. +A flow's `source` names a Connection ID. Each table's `object` identifies the +source object, and its `contract`, when it has one, names its data contract. +These are different identities. A table without a `contract` takes its columns +from the connection's source metadata, `sources/.yaml`, by its +`object`; a contract is needed only where a person must say what the source +cannot, such as who may read the table. See +[Describe a source](/docs/how-to/describe-a-source). A transform's `inputs` +names tables produced by the project, and a transform always names its +contract. @@ -65,7 +75,7 @@ not rename the others. | -------------------------- | ------------------------------------------------------------------------------------------------------------ | | Project and definition IDs | `id` in the project or document | | Project table name | A flow table's key or a transform's `id` | -| Contract link | The table's `contract`, identifying a data contract | +| Contract link | The table's `contract`, when it has one, identifying a data contract | | Platform resource name | The pack's default patterns, or `naming` in the standard or platform; for ADF, the method's `resource_names` | [Name tables and resources](/docs/how-to/name-things-your-way) shows how the diff --git a/content/docs/concepts/generated-files.mdx b/content/docs/concepts/generated-files.mdx index b27edf4..20e66cc 100644 --- a/content/docs/concepts/generated-files.mdx +++ b/content/docs/concepts/generated-files.mdx @@ -66,7 +66,10 @@ holds a file that was not generated, or when a folder generated before is no longer generated. It writes nothing. `ingestron plan` shows the changes before `ingestron apply` writes them; -see [Command line](/docs/reference/cli). +see [Command line](/docs/reference/cli). Outside `build/`, `apply` keeps one +more thing current: the guide for coding assistants in `AGENTS.md`, when the +project has one, and only while nobody has edited it; see +[Keep it current](/docs/tools/assistants#keep-current). ## Nothing of Ingestron at run time diff --git a/content/docs/concepts/layers.mdx b/content/docs/concepts/layers.mdx index 4d30324..6a0d38c 100644 --- a/content/docs/concepts/layers.mdx +++ b/content/docs/concepts/layers.mdx @@ -86,6 +86,12 @@ layer, or an expectation's `drop` in a layer that does not `check`, is error that names the layers that do it. In a layer whose `does` leaves out `check`, every action is `warn`. +A flow table in a layer that does `cleanse`, `check` or `quarantine` needs a +contract, which says what to cleanse and check. In a `raw` layer, or one that +only deduplicates, a flow table can take its columns from the source's +metadata instead; see +[Describe a source](/docs/how-to/describe-a-source#needs-a-contract). + A layer without `does` asks nothing of its tables, so a project written before layers had duties generates the same files. Its flow tables still get the standard's `cleanse` defaults; a transform's table does not. Databricks checks diff --git a/content/docs/how-to/(build)/add-a-source.mdx b/content/docs/how-to/(build)/add-a-source.mdx index fd074e4..cbfea10 100644 --- a/content/docs/how-to/(build)/add-a-source.mdx +++ b/content/docs/how-to/(build)/add-a-source.mdx @@ -109,7 +109,10 @@ Each source is a connection, in `connections/`, and the flows that read it, in [YAML references and placeholders](/docs/concepts/yaml-references). 2. Write a flow whose `source` is the connection's `id`, with a table for each - object, as in `flows/orders.yaml` above. Give each table a contract. + object, as in `flows/orders.yaml` above. A table takes its columns from its + contract, or, without one, from the source's metadata, which you import from + a file someone who can reach the source saves; see + [Describe a source](/docs/how-to/describe-a-source). 3. Plan. The plan checks that a method of your standard's packs reads the connector, and says what is missing. For example, a flow that reads a diff --git a/content/docs/how-to/(build)/add-quality-checks.mdx b/content/docs/how-to/(build)/add-quality-checks.mdx index d92a482..7983dde 100644 --- a/content/docs/how-to/(build)/add-quality-checks.mdx +++ b/content/docs/how-to/(build)/add-quality-checks.mdx @@ -83,6 +83,12 @@ error, [`METHOD_CHECK`](/docs/reference/errors#METHOD_CHECK). In the example, `bronze` does `check` and `quarantine`, so every engine checks the readings there. +A flow table in a layer that does `cleanse`, `check` or `quarantine`, or with +`expectations` of its own, needs a contract: its rules are what the layer +applies. Elsewhere a table can take its columns from the source's metadata, +with no checks; see +[When a table needs a contract](/docs/how-to/describe-a-source#needs-a-contract). + ## What each engine does [#engines] Every engine writes the checks into the files it generates, as plain SQL. What diff --git a/content/docs/how-to/(build)/check-in-ci.mdx b/content/docs/how-to/(build)/check-in-ci.mdx index 9012170..1e72bc4 100644 --- a/content/docs/how-to/(build)/check-in-ci.mdx +++ b/content/docs/how-to/(build)/check-in-ci.mdx @@ -8,7 +8,7 @@ from the definitions project folder on every pull request: it fails when someone changed the definitions without generating, so the files you deploy are always what the definitions say. It needs only Node.js 22.13 or later: each template installs an exact version of the command from npm, -`npm install -g ingestron@2.3.1`, then runs it. Change the version when you +`npm install -g ingestron@2.4.0`, then runs it. Change the version when you upgrade, as [Upgrade Ingestron and its packs](/docs/how-to/upgrade#choose-the-version-you-run) shows, so a new release never fails a pull request that changed nothing. diff --git a/content/docs/how-to/(build)/deploy-through-environments.mdx b/content/docs/how-to/(build)/deploy-through-environments.mdx index 40019ed..d14b9aa 100644 --- a/content/docs/how-to/(build)/deploy-through-environments.mdx +++ b/content/docs/how-to/(build)/deploy-through-environments.mdx @@ -43,7 +43,7 @@ The tools are pinned: Microsoft's `@microsoft/azure-data-factory-utilities` 1.0.3, the Databricks CLI v1.23.0 and sqlcmd (Go) v1.10.0, each download checked by its SHA-256 where the template downloads it. Each GitHub action is pinned to a full commit, with its version beside it, and the check installs an -exact Ingestron, `ingestron@2.3.1`: change them when you upgrade. The ADF +exact Ingestron, `ingestron@2.4.0`: change them when you upgrade. The ADF build fetches its build tool from `adf.azure.com`, so the build stage needs network access there. diff --git a/content/docs/how-to/(build)/describe-a-source.mdx b/content/docs/how-to/(build)/describe-a-source.mdx new file mode 100644 index 0000000..d190b23 --- /dev/null +++ b/content/docs/how-to/(build)/describe-a-source.mdx @@ -0,0 +1,625 @@ +--- +title: Describe a source +description: Get each source table's columns, types and key into the project without connecting to the source, see what changed at the source, and draft the contracts that are needed. +--- + +A table takes its columns, their types and its key from one of two places: + +- **Source metadata:** what the source says its tables look like, kept in + `sources/.yaml`. It is an observation, never a promise: a table + that takes its columns from it gets no checks, descriptions or + classifications. +- **A contract:** what your team promises about the table; see + [Describe a table with a contract](/docs/how-to/write-a-contract). + +A raw copy of a source can do without a contract, since its columns are what +the source has, unless the platform manages who may read it. A contract is +needed only where a person must say what the source cannot; see +[When a table needs a contract](#needs-a-contract). Where a table has both, +Ingestron compares them, so a change at the source shows up in `plan` before +the data stops landing. + +Ingestron never connects to a source. Someone who can reach it runs a query, +calls an API or exports a file, and Ingestron reads that file. It keeps only +the names and types of the columns, and writes no value from a row. + +To follow along, create a project that loads two tables of a SQL Server ERP +with Databricks, and generate its files: + +```sh +ingestron init sales --connector sqlserver --connectivity public --via databricks --yes \ + --set source_uc_connection=erp_sqlserver --set customers_object=ERP.dbo.Customers \ + --set invoices_object=ERP.dbo.Invoices +cd sales +ingestron apply +``` + +Its connection is `erp`, and its flow, `flows/erp.yaml`, loads the tables +`customers` and `invoices`, each with a contract. + +## Ask for the metadata [#request] + +A connection's recipe says how to get its kind of source's metadata. A SQL +Server connection uses the `sqlserver` recipe, a query of the database's +catalog. The recipe needs the database's name, so give it in the connection. +Add this to `connections/erp.yaml`: + +```yaml +metadata: + from: sqlserver + with: { database: ERP } +``` + +`from` names the recipe, and `with` gives its inputs. +[Source metadata recipes](/docs/reference/recipes) lists every recipe, its +inputs and the connectors that use it by default. A connector with no recipe of +its own, or a connection that reads the files another tool lands, names the +recipe of the system the data comes from: a `files` connection whose files ADF +copies from Oracle names `oracle`, for example. + +Then write the request to send to whoever can reach the source: + + + + +From the project's folder, run: + +```sh +ingestron source request +``` + + + + +Run **Ingestron: Request source metadata** from the Command Palette, or +right-click the project's folder in the Explorer. With more than one +connection, choose one, or **All connections**. The request opens as an +unsaved Markdown document, ready to send. + + + + +It starts: + +```text +# Source request: sales (dev) + +Ingestron needs to know what each source table looks like: its columns, their types and its key. Only names and types are needed, never rows of real data. Save each file where this request says. + +## erp + +- Who: Someone who can run queries in the ERP database on the SQL Server, Azure SQL Database or Azure SQL Managed Instance that holds the tables +- Access: SQL Server shows a login only the metadata of objects it owns or has some permission on, so the login needs VIEW DEFINITION on the ERP database. Without it, the result leaves tables out and says nothing. +- Recipe: SQL Server (`sqlserver`) +``` + +For each connection a flow reads, the request gives the recipe's step: the +query to run or the URL to call, filled in for your project, how to save the +result and where. The query covers whole schemas, `dbo` here, not only the +tables the flows name, so a table you add later is already described. Under +**If that cannot be done** it lists the other ways to describe the source, +below. It ends with the command that imports each file. A connection whose +tables are all described already is listed with when each was captured, +instead of being asked again. + +## Save what comes back [#save] + +Each way of describing a source is saved under `sources/`, in a file whose name +says how to read it. Each way below is a fallback for the one above it, so +use the first that works for your source: + +| Way | Save it as | Recorded as | +| ----------------------------------------------- | ----------------------------------------------------------- | ----------- | +| [The recipe's query or call](#catalog) | where the request says | `catalog` | +| [A document the source publishes](#document) | where the request says, or `sources/.odcs.yaml` | `document` | +| [A sample of each table](#sample) | `sources//.` | `sample` | +| [The columns file](#columns-file) | `sources/.columns.csv` | `columns` | +| [A coding assistant's columns file](#assistant) | `sources/.assistant.csv` | `assistant` | + +Each table in the metadata records how it was described, so a reviewer can see +where its columns came from. + +### The recipe's query or call [#catalog] + +A catalog query lists every table and column the database holds, your own +custom ones included, with their types, whether each can be null, and the +primary key. An application's metadata API, such as Salesforce's `describe` or +Dataverse's `$metadata`, does the same for its objects, and a schema registry +for a Kafka topic. + +Here the ERP's database administrator runs the query and saves the result as +`sources/erp.catalog.csv`, one row for each column: + +```csv +table_catalog,table_schema,table_name,column_name,ordinal_position,data_type,is_nullable,primary_key_position +ERP,dbo,Customers,customer_id,1,int,NO,1 +ERP,dbo,Customers,name,2,nvarchar(200),YES, +ERP,dbo,Customers,country,3,nchar(2),YES, +ERP,dbo,Customers,email,4,nvarchar(320),YES, +ERP,dbo,Invoices,invoice_id,1,int,NO,1 +ERP,dbo,Invoices,customer_id,2,int,NO, +ERP,dbo,Invoices,amount,3,"decimal(18,2)",YES, +ERP,dbo,Invoices,status,4,nvarchar(20),YES, +ERP,dbo,Payments,payment_id,1,int,NO,1 +ERP,dbo,Payments,invoice_id,2,int,NO, +ERP,dbo,Payments,paid_on,3,date,NO, +ERP,dbo,Payments,amount,4,"decimal(18,2)",NO, +``` + +The recipe maps each source type, such as `nvarchar(200)`, to a logical type, +such as `string` with a `maxLength` of 200. A type no rule knows is read as +`string`, with a note naming it. To correct a recipe's query for your +source, keep your own copy as `recipes/.yaml` in the project; see +[Use your own recipe](/docs/reference/recipes#your-own). + +### A document the source publishes [#document] + +Many applications publish a description of their data. Ingestron reads OData +CSDL (an OData service's `$metadata`), OpenAPI, JSON Schema and Avro. The +`odata` and `openapi` recipes ask for the download, and the request says where +to save it. A data contract (ODCS) +from the source's own team is saved as `sources/.odcs.yaml`; each +of its schema entries is a table. + +### A sample of each table [#sample] + +Any source can export a few rows. Save each table's sample where the request +says, `sources//.`, such as +`sources/erp/ERP.dbo.Customers.csv`: the path names the table. A table whose +name ends with the extension already keeps it once: a flow that reads +`orders.csv` from `shop` is asked for `sources/shop/orders.csv`, and that file +is imported as `orders.csv`. Saved anywhere else, a sample is imported with +`--object`, which names its table. + +A sample's values are read only to find each column's type, and then dropped: +the metadata holds names and types, and no value. Even so, a CSV or JSON +sample should hold synthetic or non-sensitive rows. A Parquet file is read +from its schema alone, never its rows. A sample cannot show which columns form +the key, so its tables have none, and a CSV or JSON sample cannot show which +columns must have a value, so none of its columns is required. + +A JSON sample can hold records inside records, as an API response or a JSON +export does. Ingestron reads the nesting from the data. For example, this +sample, saved as `sources/erp/ERP.dbo.Orders.json`: + +```json +[ + { + "order_id": 1001, + "placed_at": "2026-11-02T09:15:00Z", + "customer": { "id": 7, "name": "Ana" }, + "lines": [ + { "sku": "A-1", "qty": 2 }, + { "sku": "B-4", "qty": 1, "gift": true } + ], + "totals_by_day": { "2026-11-02": 40.5 } + }, + { + "order_id": 1002, + "placed_at": "2026-11-03T10:00:00Z", + "customer": { "id": 9, "name": "Ben", "tier": "gold" }, + "lines": [{ "sku": "C-2", "qty": 5 }], + "totals_by_day": { "2026-11-03": 12 } + } +] +``` + +is imported as this table: + +{/* prettier-ignore */} +```yaml kind="writes" title="sources/erp.yaml" + - object: ERP.dbo.Orders + captured: { at: 2026-11-13T09:30:00Z, from: sample, recipe: sqlserver, file: sources/erp/ERP.dbo.Orders.json } + columns: + - { name: order_id, logicalType: integer, required: false } + - { name: placed_at, logicalType: timestamp, required: false } + - name: customer + logicalType: object + required: false + properties: + - { name: id, logicalType: integer, required: false } + - { name: name, logicalType: string, required: false } + - name: lines + logicalType: array + required: false + items: + name: items + logicalType: object + required: false + properties: + - { name: sku, logicalType: string, required: false } + - { name: qty, logicalType: integer, required: false } + - { name: totals_by_day, logicalType: object, required: false } +``` + +- **A record inside a record** is a column of `logicalType: object`, with its + keys as its fields, when the keys are names rather than data. With several + records in the sample, the nested record must be in at least two of them, + and at least one key must be in every one it is in; a nested record found + in only one record is an `object` with no fields. A key found in only one + is left out: one record cannot show that `tier` is a name and not a value. + A later sample that has it twice adds it. +- **Keys that look like data,** such as dates, numbers or UUIDs, keys with + characters other than letters, digits, spaces and `_ $ . -`, or more than 100 + different keys, make the column an `object` with no fields, as + `totals_by_day` is. Its keys are never written, since they are values. +- **A list** is a column of `logicalType: array`, whose elements are read + together as its `items`. So `gift`, found in one element only, is left out + of `lines`. + +### The columns file [#columns-file] + +When nothing above works, write the columns file: the same eight fields as +the catalog query's result, `table_catalog`, `table_schema`, `table_name`, +`column_name`, `ordinal_position`, `data_type`, `is_nullable` (`YES` or `NO`) +and `primary_key_position` (empty when the column is not in the key), one row +for each column. A script, a catalog export or a spreadsheet can make it. Save +it as `sources/.columns.csv`. A file with any other field is +refused, naming the fields it lacks, so a query that returned rows instead of +columns is never imported. + +### A coding assistant's columns file [#assistant] + +Some sources come with nothing a reader takes: a vendor's data dictionary, a +spreadsheet, a PDF, inconsistent API responses. A coding assistant can turn +those into a columns file. Give the project a guide for assistants: + +```sh +ingestron assistant add agents +``` + +It writes `AGENTS.md`, whose source metadata section tells an assistant to save +the columns file as `sources/.assistant.csv`, keep every table and +column the material shows, write a type it is unsure of as `string`, never +invent a key, never copy a value from a row, and check the file with +`ingestron source import --dry-run` before it says it is +done. The source request then offers this way too. Which files each +assistant needs, and how `plan` and `apply` keep the guide current, are in +[Work with an AI assistant](/docs/tools/assistants). + +Ingestron trusts an assistant's columns file no more than a sample: its tables +are compared by column names only, and a contract drafted from them carries no +key. + +## Import it [#import] + + + + +Check what the file would change, without writing anything: + +```sh +ingestron source import erp sources/erp.catalog.csv --dry-run +``` + +```text +sources/erp.yaml: 3 added, 0 changed, 0 removed. +Compatible: + ERP.dbo.Customers: added + ERP.dbo.Invoices: added + ERP.dbo.Payments: added +Nothing was written (--dry-run). +``` + +Then import it: + +```sh +ingestron source import erp sources/erp.catalog.csv +``` + +```text +sources/erp.yaml: 3 added, 0 changed, 0 removed. +Compatible: + ERP.dbo.Customers: added + ERP.dbo.Invoices: added + ERP.dbo.Payments: added +Wrote sources/erp.yaml. +Added sources/erp.yaml to the imports of project.yaml. +``` + +A file it cannot read is refused with the reasons, and nothing is written. + + + + +Save the file where the request says. A file saved under `sources/` is +imported as soon as it is saved, and a notification gives the first line of +what changed, such as `sources/erp.yaml: 3 added, 0 changed, 0 removed.`, with +**Show all changes** for the rest. Files saved together give one +notification. A file the import refuses gives a warning with the reasons, and +nothing is written. The import is an edit of `sources/.yaml` you +can undo. It records the environment selected in the status bar, else the +project's first, so the document it writes has `environment: dev`, which the +command writes only with `--env`. + +To import a file kept somewhere else, right-click it in the Explorer and choose +**Ingestron: Import source metadata…**, or run it from the Command Palette +and choose the file. When the file's path does not name a connection and the +project has more than one, it asks which; and it asks for the table when the +file describes one table and its path does not name it. + + + + +The import writes `sources/erp.yaml`: each table, where and when it was +captured, and its columns: + +{/* prettier-ignore */} +```yaml kind="writes" title="sources/erp.yaml" +# Written by ingestron source import. Do not edit: import again to replace it. +apiVersion: ingestron.io/v1 +kind: SourceMetadata +id: erp +connection: erp +names: { parts: 3, caseSensitive: false } +tables: + - object: ERP.dbo.Customers + captured: { at: 2026-11-13T09:30:00Z, from: catalog, recipe: sqlserver, file: sources/erp.catalog.csv } + columns: + - { name: customer_id, sourceType: int, logicalType: integer, required: true, primaryKeyPosition: 1 } + - { name: name, sourceType: nvarchar(200), logicalType: string, logicalTypeOptions: { maxLength: 200 }, required: false } + - { name: country, sourceType: nchar(2), logicalType: string, logicalTypeOptions: { maxLength: 2 }, required: false } + - { name: email, sourceType: nvarchar(320), logicalType: string, logicalTypeOptions: { maxLength: 320 }, required: false } + - object: ERP.dbo.Invoices + captured: { at: 2026-11-13T09:30:00Z, from: catalog, recipe: sqlserver, file: sources/erp.catalog.csv } + columns: + - { name: invoice_id, sourceType: int, logicalType: integer, required: true, primaryKeyPosition: 1 } + - { name: customer_id, sourceType: int, logicalType: integer, required: true } + - { name: amount, sourceType: "decimal(18,2)", logicalType: number, logicalTypeOptions: { precision: 18, scale: 2 }, required: false } + - { name: status, sourceType: nvarchar(20), logicalType: string, logicalTypeOptions: { maxLength: 20 }, required: false } + - object: ERP.dbo.Payments + captured: { at: 2026-11-13T09:30:00Z, from: catalog, recipe: sqlserver, file: sources/erp.catalog.csv } + columns: + - { name: payment_id, sourceType: int, logicalType: integer, required: true, primaryKeyPosition: 1 } + - { name: invoice_id, sourceType: int, logicalType: integer, required: true } + - { name: paid_on, sourceType: date, logicalType: date, required: true } + - { name: amount, sourceType: "decimal(18,2)", logicalType: number, logicalTypeOptions: { precision: 18, scale: 2 }, required: true } +``` + +Never edit it: import again to replace it. A catalog import replaces every +table an earlier catalog import wrote, so a table dropped at the source +disappears. Any other import adds its tables, or replaces the same ones. +[Source metadata](/docs/reference/documents#source-metadata) lists its fields. + +Ask for the metadata again, and the request lists what the flows read instead +of asking: + +```sh +ingestron source request +``` + +```text +## erp + +Every table its flows read is in sources/erp.yaml: + +- `ERP.dbo.Customers`: catalog, captured 2026-11-13 +- `ERP.dbo.Invoices`: catalog, captured 2026-11-13 +``` + +The metadata holds names and types only. Whether to keep the files you saved +under `sources/` in Git is your team's choice; keep a sample out of Git if its +rows are real. + +## Load a table without a contract [#without-a-contract] + +The catalog listed `Payments`, which no flow reads yet. Add it to +`flows/erp.yaml`, under `tables`, with no contract: + +{/* prettier-ignore */} +```yaml + payments: { object: ERP.dbo.Payments } +``` + +Plan: + +```sh +ingestron plan +``` + +The plan first notes, as information (`SOURCE_COLUMN_NEW`), that the +source's customers table has a column, `email`, which the customers contract +does not take. The table takes its columns and key from the metadata, and the +managed ingestion pipeline gets them. The plan ends: + +```text +databricks → build/databricks (dev, prod) +update resources/sales_erp_ingestion.pipeline.yml + @@ -42,2 +42,18 @@ + primary_keys: + - invoice_id + + - table: + + source_catalog: ERP + + source_schema: dbo + + source_table: Payments + + destination_catalog: ${var.catalog} + + destination_schema: bronze + + destination_table: payments + + table_configuration: + + scd_type: SCD_TYPE_1 + + include_columns: + + - payment_id + + - invoice_id + + - paid_on + + - amount + + primary_keys: + + - payment_id + +6 files: 1 update, 5 unchanged. Ready to apply. +``` + +Apply it. Such a table's columns can all be null, whatever the source says, +and get no checks: only a contract makes a promise. Its key is the source's +primary key, which `history: latest` and `full` use. + +A flow table's `object` is matched with the metadata's tables part by part, +from the right, with or without case as the recipe says. A part written as a +reference, such as `${values.erp_db}.dbo.Payments`, matches any database, so +`dev` and `prod` databases with different names share one metadata file. A table with +neither a contract nor metadata has no columns, and `plan` reports +[`TABLE_NO_COLUMNS`](/docs/reference/errors#TABLE_NO_COLUMNS), whose hint +names the source request. + +## When a table needs a contract [#needs-a-contract] + +A flow table needs a contract when a person must say something the source +cannot: + +- its [layer](/docs/concepts/layers) does `cleanse`, `check` or `quarantine`; +- it is an output of a [data product](/docs/how-to/define-a-data-product); +- the platform manages grants or masks for it, so a contract must say who may + read it and which columns are sensitive, unless its layer sets + `access: { contract: false }`, as the bronze layer of + [Streaming lakehouse on Databricks](/docs/how-to/kits/databricks-lakehouse-streaming) + does; see + [Grant access, mask columns and filter rows](/docs/how-to/grant-access); +- it has its own `expectations`. + +For example, turn on grants in `platform.yaml`: + +```yaml +databricks: + manage: { grants: true } +``` + +Plan again, and the payments table needs a contract: + +```text +flows/erp.yaml:15:13 error CONTRACT_NEEDED + "payments" needs a contract: access is managed for it on databricks. + Draft one from the source metadata with `ingestron contract --from-metadata erp.payments`, then name it with `contract`. +``` + +Draft the contract from the metadata. `ingestron contract --from-metadata` +drafts the tables you name, each as `.
`, such as `erp.payments`, +or, with `--needed`, every table that needs a contract and names none: + + + + +```sh +ingestron contract --from-metadata --needed +``` + +```text +Wrote contracts/erp.payments.yaml. +Add to erp.payments: contract: erp.payments +``` + + + + +Open the problem's light bulb in `flows/erp.yaml` and choose **Draft a +contract from the source metadata**. It writes the draft and adds the +`contract:` line. + + + + +The draft carries the source's types, which columns cannot be null, and its +key, for you to review. `status: draft` is plain ODCS and changes nothing in +Ingestron. A decimal's precision and scale are not carried, since ODCS 3.1 +has no field for them, so `amount` is a `number`; give it a `physicalType`, +such as `DECIMAL(18,2)`, if the table needs the exact type: + +```yaml kind="plain" title="contracts/erp.payments.yaml" +# Drafted from "ERP.dbo.Payments" in sources/erp.yaml (catalog, captured 2026-11-13). +# Review the types, required columns and key, then add descriptions, classifications and checks. +apiVersion: v3.1.0 +kind: DataContract +id: erp.payments +version: 0.1.0 +status: draft +schema: + - name: payments + logicalType: object + properties: + - name: payment_id + logicalType: integer + required: true + primaryKey: true + primaryKeyPosition: 1 + - name: invoice_id + logicalType: integer + required: true + - name: paid_on + logicalType: date + required: true + - name: amount + logicalType: number + required: true +``` + +A table drafted from a sample or an assistant's file has no key; add it. A +file that is already there is never replaced: the command skips it and says so. +The contract is yours now: add the roles, classifications and descriptions +that only your team can give; see +[Describe a table with a contract](/docs/how-to/write-a-contract). Name it in +the flow, as the command printed: + +{/* prettier-ignore */} +```yaml + payments: { object: ERP.dbo.Payments, contract: erp.payments } +``` + +## When the source changes [#changes] + +Import each new result the same way. Say the ERP team drops the invoices' +`status` and adds `due_on`. The next catalog result, saved over +`sources/erp.catalog.csv`, imports as: + +```text +sources/erp.yaml: 0 added, 1 changed, 0 removed. +Breaking: + ERP.dbo.Invoices: column "status" is gone +Compatible: + ERP.dbo.Invoices: column "due_on" is new +Wrote sources/erp.yaml. +``` + +A change is **breaking** when a table that reads the source could fail: a +column gone, a type that no longer fits, a required column that can now be +null, a different key, or, from a catalog, a table gone. It is +**compatible** when nothing that reads the source breaks: a new table or +column, or a type whose values all fit the type it is compared with, such as +`integer` where the contract, or the import before, says `number`. + +`plan` and `check` compare each table that has a contract with its metadata. +The contract for `invoices` still has `status`, so the plan fails: + +```text +flows/erp.yaml:13:15 error SOURCE_COLUMN_GONE + Column "status" of contract "erp.invoices" is not in the source metadata for "ERP.dbo.Invoices". + Remove the column from the contract, or import the source's metadata again if the column is still there. +``` + +A compatible difference is information, such as `SOURCE_COLUMN_NEW` for +`due_on`: the contract does not take a column until you add it. A table +without a contract is not compared, since its columns are the metadata's. A +table from a sample or an assistant's file is compared by column names only. A +table a catalog import does not list is reported only once the connection has +tables from a catalog, since only a catalog describes the whole source. Every +code that starts `SOURCE_` is in [Errors](/docs/reference/errors). + +A breaking difference fails `plan` and `check` by default: +`sources: { breaking: fail }`. To make each a warning instead, set `breaking` +to `warn`, in the standard or in `platform.yaml`, which wins and stays when +you move to a newer published standard; see +[`sources.breaking`](/docs/reference/documents#platform-sources-breaking): + +```yaml +sources: { breaking: warn } +``` + +A warning can then be accepted in `platform.yaml`, for one column of one +table, with the reason; see [Accept a warning](/docs/how-to/accept-a-warning): + +```yaml +accept: + - code: SOURCE_COLUMN_GONE + about: { table: invoices, feature: status } + reason: The ERP no longer has an invoice status; the contract keeps it until customer_balances stops reading it. +``` + +The plan then says `1 accepted warning (platform.yaml)`, and generates as +before. diff --git a/content/docs/how-to/(build)/grant-access.mdx b/content/docs/how-to/(build)/grant-access.mdx index 2bbd964..103b726 100644 --- a/content/docs/how-to/(build)/grant-access.mdx +++ b/content/docs/how-to/(build)/grant-access.mdx @@ -4,7 +4,10 @@ description: Say who may read each table, hide sensitive columns and filter rows --- Who may read a table belongs with its contract, and how sensitive columns -and rows are hidden belongs in the standard. Ingestron turns both into the +and rows are hidden belongs in the standard. So a flow table needs a contract +wherever the platform manages its grants or masks; see +[When a table needs a contract](/docs/how-to/describe-a-source#needs-a-contract). +Ingestron turns both into the `GRANT`, `REVOKE` and policy statements the generated files run, so access changes in a pull request, as the tables do. Each part is off until the platform turns it on, and an engine that cannot apply one says so. diff --git a/content/docs/how-to/(build)/meta.json b/content/docs/how-to/(build)/meta.json index 91f8ddb..995d388 100644 --- a/content/docs/how-to/(build)/meta.json +++ b/content/docs/how-to/(build)/meta.json @@ -5,6 +5,7 @@ "pages": [ "start-from-a-template", "add-a-source", + "describe-a-source", "write-a-contract", "write-a-transform", "keep-a-table-current", diff --git a/content/docs/how-to/(build)/start-from-a-template.mdx b/content/docs/how-to/(build)/start-from-a-template.mdx index 3042ae3..5014e50 100644 --- a/content/docs/how-to/(build)/start-from-a-template.mdx +++ b/content/docs/how-to/(build)/start-from-a-template.mdx @@ -48,9 +48,11 @@ ingestron init sales ``` Choose the source category, connector, network reachability and route. -When only one route is supported, Ingestron selects it. Nothing else is asked: -the project's id is the folder's name, its environments are `dev` and `prod`, -and its standard is the route's own. +When only one route is supported, Ingestron selects it. In a terminal it then +asks, last, which assistant files to write; press Enter for none, as +[the project guide](#project-guide) below explains. Nothing else is asked: the +project's id is the folder's name, its environments are `dev` and `prod`, and +its standard is the route's own. To choose them with flags (`--via` is `databricks`, `adf-databricks`, `adf-synapse`, `adf-duckdb` or `adf-landing`; see @@ -189,6 +191,10 @@ Then make the rest yours: contracts README.md names are examples. See [Describe a table with a contract](/docs/how-to/write-a-contract), and the **Read with …** section of the connector's page for its reading options. + A table that copies the source as it is can do without a contract, + [unless the platform manages its access](/docs/how-to/describe-a-source#needs-a-contract): import the source's + metadata instead, and draft the contracts that are needed from it; see + [Describe a source](/docs/how-to/describe-a-source). 3. Check the environments, platform and standard. Use the connector reference and [Azure Data Factory](/docs/platforms/adf) for ADF infrastructure, and [ADF resource names](/docs/how-to/name-things-your-way#adf-resource-names) @@ -207,6 +213,21 @@ for what those files are for. To grow the project later, see [Add a source and its tables](/docs/how-to/add-a-source). +## Add a guide for coding assistants [#project-guide] + +`ingestron init --assistant ` writes a project guide as the project is +created, and `ingestron assistant add ` adds it to a project you +already have. Without `--assistant`, `init` asks which assistant files to +write, last and only in a terminal; an empty answer is `none`, and `--yes` +skips the question. Without a terminal, nothing is asked +and nothing is written. The guide is `AGENTS.md`, written from the project's own files: +what the project is, what it may change and which commands to run, and, when +the project has connections, how to +[prepare a source's columns file](/docs/how-to/describe-a-source#assistant). +`ingestron plan` and `ingestron apply` keep its Ingestron section current. +[Work with an AI assistant](/docs/tools/assistants) lists the files each +assistant gets, and what the guide says. + ## Review and generate From the project folder, run `ingestron plan --no-diff`, or use diff --git a/content/docs/how-to/(build)/write-a-contract.mdx b/content/docs/how-to/(build)/write-a-contract.mdx index ec3c6fa..e860a4b 100644 --- a/content/docs/how-to/(build)/write-a-contract.mdx +++ b/content/docs/how-to/(build)/write-a-contract.mdx @@ -1,6 +1,6 @@ --- title: Describe a table with a contract -description: Write a table's columns, types, keys and rules in a data contract, or draft one from a sample file. +description: Write a table's columns, types, keys and rules in a data contract, or draft one from a sample file or from the source's metadata. --- A contract says what a table holds: its columns, their types, which columns @@ -9,6 +9,14 @@ the table's columns, to keep rows by their key and to check them. Contracts use the [Open Data Contract Standard](https://bitol-io.github.io/open-data-contract-standard/) (ODCS) v3.1.0 unchanged, so other tools can read them too. +A flow table needs a contract only where a person must say what the source +cannot: when its layer cleanses or checks rows, when it is a data product's +output, when the platform manages its grants or masks, or when it has its own +`expectations`. Any other flow table can take its columns and key from the +source's metadata instead; see +[Describe a source](/docs/how-to/describe-a-source#needs-a-contract). A +transform always names a contract. + Each contract is a file in `contracts/`. This one, from the [DuckDB starter](/docs/start/first-project), describes the orders table: @@ -166,3 +174,18 @@ is `status: draft` with neither. Review the types, then add `primaryKey`, `required` and descriptions, and link it from a table with `contract`. [Add a source and its tables](/docs/how-to/add-a-source#add-a-table-from-a-source-you-already-read) drafts one and links it. + +## Draft from source metadata [#draft-from-source-metadata] + +Once the project has the source's metadata, draft a flow table's contract +from it, with each column's type, whether it can be null and the source's key: + +```sh +ingestron contract --from-metadata erp.payments +``` + +`erp.payments` is the flow's id and the table's key. `--needed` instead +drafts every table that needs a contract and names none. Each draft is +`contracts/.
.yaml`, never replacing a file that is there. +[Describe a source](/docs/how-to/describe-a-source) gets the metadata and +drafts one, step by step. diff --git a/content/docs/how-to/(standards)/share-a-template.mdx b/content/docs/how-to/(standards)/share-a-template.mdx index d9d488d..1442aef 100644 --- a/content/docs/how-to/(standards)/share-a-template.mdx +++ b/content/docs/how-to/(standards)/share-a-template.mdx @@ -21,7 +21,7 @@ A template is an ordinary project folder with one extra file, -`init` asks nothing: the new project's `README.md` lists each `REPLACE_ME` +`init` asks nothing about those values: the new project's `README.md` lists each `REPLACE_ME` with its comment, `ingestron plan` warns about each, and `ingestron check` fails until each is written. `--set landing=` writes one as the project is created. diff --git a/content/docs/how-to/(standards)/upgrade.mdx b/content/docs/how-to/(standards)/upgrade.mdx index a806fe1..d1c80a9 100644 --- a/content/docs/how-to/(standards)/upgrade.mdx +++ b/content/docs/how-to/(standards)/upgrade.mdx @@ -1,6 +1,6 @@ --- title: Upgrade Ingestron and its packs -description: Choose the version every project runs, see what an upgrade changes with plan, move CI and your team together, and what changed from 1.x to 2.3. +description: Choose the version every project runs, see what an upgrade changes with plan, move CI and your team together, and what changed from 1.x to 2.4. --- The packs come inside the `ingestron` command: each release of the command @@ -46,7 +46,7 @@ choose to upgrade. The [CI templates](/docs/how-to/check-in-ci) and the [deployment templates](/docs/how-to/deploy-through-environments) do: ```sh -npm install -g ingestron@2.3.1 +npm install -g ingestron@2.4.0 ``` `npm ls -g ingestron` shows the version a computer has installed. @@ -95,7 +95,7 @@ next; see [Deploy through environments](/docs/how-to/deploy-through-environments ### The standard and the folders -1. Install `ingestron@2.3.1` and plan. Every pack is version 2, so a standard that +1. Install `ingestron@2.4.0` and plan. Every pack is version 2, so a standard that names `databricks@1`, `adf@1`, `synapse@1` or `duckdb@1` stops with `PACK_VERSION`, once for all the environments that have it: @@ -135,7 +135,7 @@ next; see [Deploy through environments](/docs/how-to/deploy-through-environments [`DEPLOY_VALUE`](/docs/reference/errors#DEPLOY_VALUE): write it in `project.yaml` for each environment. See [Supplied when deploying](/docs/concepts/yaml-references#supplied-when-deploying). -5. Change CI: the check templates install `ingestron@2.3.1`, and the deploy +5. Change CI: the check templates install `ingestron@2.4.0`, and the deploy pipelines deploy `build//` by target, parameter file or script, as [Deploy through environments](/docs/how-to/deploy-through-environments) shows. @@ -531,11 +531,55 @@ files are the same: such as `mode` and `sequence_by`, after the others, each with the intent that replaces it. +## Upgrade to 2.4 [#upgrade-to-2-4] + +2.4 generates the same files as 2.3 for a project that changes nothing. It +adds [source metadata](/docs/how-to/describe-a-source): what each source table +looks like, imported from a file its owners send, so a table that copies the +source as it is can do without a contract. It also adds +[a guide for coding assistants](/docs/tools/assistants). What you may notice: + +- **A flow table's `contract` is optional.** A table without one takes its + columns and key from the source metadata. A table that needs one, because + its layer cleanses, checks or quarantines, the platform manages its access + or it has its own `expectations`, is + [`CONTRACT_NEEDED`](/docs/reference/errors#CONTRACT_NEEDED); one with + neither a contract nor metadata is + [`TABLE_NO_COLUMNS`](/docs/reference/errors#TABLE_NO_COLUMNS). The editor + no longer marks a missing `contract` as a schema error. Every table of a + 2.3 project names a contract, so none of these is new to it. +- **`init` asks one more question,** in a terminal, last: which coding + assistants to write files for. An empty answer writes none, and + `--assistant` or `--yes` answers it. +- **Words.** The packs' setting descriptions and some messages say "the + table's columns" where they said "the contract's", since the columns can now + come from the source metadata. Databricks' `lakeflow_connect.columns` value + `contract` is now described as "Only the table's columns." +- **VS Code 2.3** adds **Request source metadata**, **Import source + metadata…** and **Add assistant files**, imports a file saved under + `sources/`, and completes recipes and column names; see + [VS Code](/docs/tools/vscode#source-metadata). + +For code that calls the packages: + +- `@ingestron/definitions` 1.14 adds the `SourceMetadata` document and its + types, `Contract.origin`, and the `sourceTables` option of resolving, which + lists each flow table with its entry in the metadata. +- `@ingestron/core` 1.2 adds the recipes (`sourceRecipes`, + `installedSourceRecipes`, `readSourceRecipe` and `sourceOrigin`), the + request (`sourceRequest`), the import (`importSource`) and drafting + (`draftContracts`). It reads the recipe catalogue from + `@ingestron/standards` 1.1. +- `@ingestron/authoring` 2.4 adds `importSource`, `sourceRequest` and + `sourceFileAt` to `ProjectSnapshot`, and `replaceFrom` to a `Suggestion`. + ## What changed in a release -Recent releases are commits whose message starts with the version, such as -`2.0.0:`. Read the history of the command and of each pack -your standard names: +Each release is merged into `main` from a branch named for it, so the history +shows it as a merge of `release/`, such as +`Merge pull request #33 from ingestron/release/2.3`. A patch release's commit +starts with its version, such as `2.3.1:`. Read the history of the command and +of each pack your standard names: - the command: [ingestron/ingestron](https://github.com/ingestron/ingestron/commits/main) - the packs: [pack-duckdb](https://github.com/ingestron/pack-duckdb/commits/main), diff --git a/content/docs/index.mdx b/content/docs/index.mdx index 69e9546..99bbc29 100644 --- a/content/docs/index.mdx +++ b/content/docs/index.mdx @@ -122,6 +122,18 @@ every one, and `ingestron check` fails the CI build. See start from a [published standard](/docs/how-to/choose-a-published-standard): one written for your platform, checked against its documentation and dated. +### Contracts only where you promise something + +Ingestron never connects to your sources. It writes a request saying what to +run or export at each one, and reads the names and types of the columns from +the file that comes back, never the rows. A table that copies a source as it +is takes its columns from there, so only the tables your team makes promises +about, such as who may read them, +[need a contract](/docs/how-to/describe-a-source#needs-a-contract), and +Ingestron drafts those. A change at the source +shows up in the plan before it breaks a load. See +[Describe a source](/docs/how-to/describe-a-source). + ### Reuse your team's work Start a project from a template your team keeps in Git with @@ -131,6 +143,14 @@ one YAML file, so you can keep them in any Git repository your team, or anyone else, can clone. The packs are open source under Apache-2.0. See [Share templates and standards](/docs/how-to/share-a-template). +### A guide for your coding assistant + +`ingestron init --assistant` writes `AGENTS.md`, which tells the coding +assistant you use how to work in the project: change the definitions and +never the generated files, show you the plan before applying, and stay +inside the standard. `plan` and `apply` keep it current as the project +changes. See [Work with an AI assistant](/docs/tools/assistants). + ## Where your data can go Ingestron generates for these routes today. The first is the starter above and diff --git a/content/docs/start/choose-a-route.mdx b/content/docs/start/choose-a-route.mdx index aeee1c4..42d126a 100644 --- a/content/docs/start/choose-a-route.mdx +++ b/content/docs/start/choose-a-route.mdx @@ -75,7 +75,9 @@ command that creates each. ingestron init sales --connector sqlserver --connectivity private --via adf-databricks ``` -Without `--yes`, `init` asks only for a choice the flags leave out. It writes +Without `--yes`, `init` asks for each choice the flags leave out, and, in a +terminal, last, which assistant files to write; `--assistant` answers that +question. It writes `REPLACE_ME` for each value it cannot know, such as the server, and its `README.md` lists them. Add `--standard ` to use your organisation's standard. For DuckDB on a laptop, use `--template duckdb` instead, as in diff --git a/content/docs/start/first-project.mdx b/content/docs/start/first-project.mdx index c96e980..b9ee220 100644 --- a/content/docs/start/first-project.mdx +++ b/content/docs/start/first-project.mdx @@ -225,6 +225,10 @@ table gets: +A table that copies a source as it is can do without a contract, and take its +columns from what the source says it has, unless the platform manages its +access; see [When a table needs a contract](/docs/how-to/describe-a-source#needs-a-contract). + The transform builds `customer_totals` from `orders` with SQL. Its own contract, `contracts/customer_totals.yaml`, types the result: diff --git a/content/docs/start/how-it-works.mdx b/content/docs/start/how-it-works.mdx index 4df28f0..f876c8a 100644 --- a/content/docs/start/how-it-works.mdx +++ b/content/docs/start/how-it-works.mdx @@ -28,8 +28,15 @@ deploying belongs to your platform's tools and your release pipeline. - **Connections:** where a source is and how it is reached. - **Flows:** which tables to read from a connection, and how to keep them. -- **Contracts:** each table's columns and types, in the Open Data Contract - Standard (ODCS). +- **Contracts:** what you promise about a table: its columns, types, keys and + checks, in the Open Data Contract Standard (ODCS). A table that copies a + source as it is can do without one, + [unless the platform manages its access](/docs/how-to/describe-a-source#needs-a-contract). +- **Source metadata,** which you import rather than write: what each source + table looks like, from a file that someone who can reach the source saves. + A table without a contract takes its columns from it, and a table with one + is compared with it, so a change at the source shows up before it breaks a + load; see [Describe a source](/docs/how-to/describe-a-source). - **Transforms:** SQL (or PySpark on Databricks) that builds a table from others. - **One standard:** the packs the project uses, and your organisation's rules. diff --git a/lib/connector-page.ts b/lib/connector-page.ts index 447f095..7027eeb 100644 --- a/lib/connector-page.ts +++ b/lib/connector-page.ts @@ -9,6 +9,7 @@ import type { TOCItemType } from "fumadocs-core/toc"; import { parse } from "yaml"; import { exampleFiles, listExamples, readExampleFile } from "./examples"; import { connectorEntries, packName, type ConnectorEntry } from "./reference"; +import { connectorRecipes } from "./recipes"; import { routeGuides } from "./routes"; /** What a part of the page shows after its words. */ @@ -320,7 +321,7 @@ function readSection(id: string, entry: ConnectorEntry): Section { id: `${pack}-tables`, title: "Choose the tables", before: [ - "The connection's `id` is the flow's `source`. Each entry under `tables` names a destination table and links its source `object` to a contract. The contract lists the columns and keys you expect. Replace the synthetic object and contract with yours before deploying.", + "The connection's `id` is the flow's `source`. Each entry under `tables` names a destination table and its source `object`, here with a contract that lists the columns and keys you expect. A table that copies the source as it is can leave out `contract` and take its columns from the [source metadata](/docs/how-to/describe-a-source) instead, [unless the platform manages its access](/docs/how-to/describe-a-source#needs-a-contract). Replace the synthetic object and contract with yours before deploying.", ], after: [], }, @@ -383,11 +384,23 @@ export function connectorSections(id: string): Section[] { id: "routes", title: "Routes", before: [ - `These are the routes a project can take for ${title}, by the pack that reads the source. To compare them, see [Choose where your data goes](/docs/start/choose-a-route). Create it gives the command that starts a project on each route; \`init\` asks only how the source is reached, and writes \`REPLACE_ME\` for each value it cannot know, as [Start from a connection](/docs/how-to/start-from-a-template#start-from-a-connection) describes.`, + `These are the routes a project can take for ${title}, by the pack that reads the source. To compare them, see [Choose where your data goes](/docs/start/choose-a-route). Create it gives the command that starts a project on each route; \`init\` asks which source and how it is reached, and, in a terminal, which assistant files to write, and writes \`REPLACE_ME\` for each value it cannot know, as [Start from a connection](/docs/how-to/start-from-a-template#start-from-a-connection) describes.`, ], items: [], }, ...entries.flatMap((entry) => [connectSection(entry), readSection(id, entry)]), + ...(connectorRecipes(id).length > 0 + ? [ + { + id: "source-metadata", + title: "Source metadata", + before: [ + `Ingestron never connects to ${title}. Someone who can reach it saves what its tables look like, their columns, types and key, and \`ingestron source import\` reads the file. A table that copies the source as it is can then do without a contract, [unless the platform manages its access](/docs/how-to/describe-a-source#needs-a-contract). \`ingestron source request\` prints this step for your project; see [Describe a source](/docs/how-to/describe-a-source).`, + ], + items: [], + }, + ] + : []), { id: "examples", title: "Examples", diff --git a/lib/recipes.ts b/lib/recipes.ts new file mode 100644 index 0000000..b81bef5 --- /dev/null +++ b/lib/recipes.ts @@ -0,0 +1,143 @@ +// Source metadata recipes, from the installed @ingestron/standards package: +// how to get one kind of source's tables, columns and types, so each recipe's +// section shows the recipe's own query or call, inputs and documentation. +import { join } from "node:path"; +import { installedSourceRecipes, sourceOrigin, type SourceRecipe } from "@ingestron/core"; + +/** The `recipes` folder of the installed @ingestron/standards package. */ +export const recipesFolder = join(process.cwd(), "node_modules/@ingestron/standards/recipes"); + +let found: SourceRecipe[] | undefined; + +/** Every installed recipe, by id in code point order. */ +export function recipes(): SourceRecipe[] { + found ??= installedSourceRecipes(recipesFolder); + return found; +} + +/** One installed recipe. */ +export function recipeById(id: string): SourceRecipe { + const recipe = recipes().find((item) => item.id === id); + if (!recipe) throw new Error(`No recipe "${id}" is installed.`); + return recipe; +} + +/** A recipe's anchor on the recipes page, such as `/docs/reference/recipes#sqlserver`. */ +export function recipePage(id: string): string { + return `/docs/reference/recipes#${id}`; +} + +/** + * How a connector comes to use a recipe: by default, because a connection names it, or because + * the recipe fits a source the connector reads. + */ +export interface ConnectorRecipe { + recipe: SourceRecipe; + /** + * `default`: a connection with no `metadata` uses it; `named`: the recipe has the connector's + * id and a connection writes `metadata.from`; `fits`: the same, for a recipe of another id. + */ + how: "default" | "named" | "fits"; +} + +/** + * Recipes that fit a connector's sources though they carry another id: the connectors read the + * kind of source the recipe asks about. This is the one place that says so; `sourceOrigin` knows + * only the defaults. + */ +export const recipesThatFit: Readonly> = { + "dynamics-365": ["dataverse"], + rest: ["openapi"], + // The ADF SAP ECC connector reads OData services that SAP Gateway exposes: its linked service + // takes the service URL (https://learn.microsoft.com/en-us/azure/data-factory/connector-sap-ecc, + // read 2026-11-13). + "sap-ecc": ["sap-odata"], +}; + +/** + * The recipes a connector's page shows: the one a connection with no `metadata` uses, a recipe + * that has the connector's id, and the recipes that fit its sources (written as `metadata.from`). + */ +export function connectorRecipes(connector: string): ConnectorRecipe[] { + const all = new Map(recipes().map((recipe) => [recipe.id, recipe])); + const byDefault = sourceOrigin({ id: "connection", connector }, all).recipe; + const named = all.get(connector); + return [ + ...(byDefault ? [{ recipe: byDefault, how: "default" as const }] : []), + ...(named && named !== byDefault ? [{ recipe: named, how: "named" as const }] : []), + ...(recipesThatFit[connector] ?? []).flatMap((id) => { + const recipe = all.get(id); + return recipe && recipe !== byDefault && recipe !== named + ? [{ recipe, how: "fits" as const }] + : []; + }), + ]; +} + +/** + * What the request fills in where a recipe writes `{name}`, besides the recipe's own inputs + * (specification, "Placeholders"). + */ +export const reservedPlaceholders: Readonly> = { + connection: "the connection's id", + object: + "one table's `object`, as its flow writes it; a step that holds it is written once for each table", + schemas: "the schemas the connection's flow tables read, each in single quotes", +}; + +/** + * Every `{name}` the installed recipes write in their text, with the recipes that write it. A + * name that is neither reserved nor an input of its recipe would be left as written by the + * request, so a test pins that there is none. + */ +export function placeholdersUsed(): Map { + const found = new Map(); + for (const recipe of recipes()) { + const { who, access, run, call, save, how } = recipe.ask; + const text = [who, access, run, call, save, how] + .filter((item) => item !== undefined) + .join("\n"); + for (const [, name] of text.matchAll(/\{([A-Za-z][A-Za-z0-9_]*)\}/g)) + found.set(name!, [...new Set([...(found.get(name!) ?? []), recipe.id])]); + } + return found; +} + +/** What a recipe's kind means, in words. */ +export const kindWords: Record = { + catalog: "A query or call on the source's own catalogue", + document: "A document the source publishes", + sample: "A file of what the source sends", +}; + +/** What each reader reads. */ +export const readerWords: Record = { + columns: "A columns file: one row for each column", + sample: "A sample file: CSV, JSON, JSON lines or Parquet", + odcs: "An ODCS data contract", + odata: "An OData metadata document", + openapi: "An OpenAPI description", + "json-schema": "A JSON Schema", + avro: "An Avro schema", + "salesforce-describe": "A Salesforce object description", +}; + +/** + * The paragraphs and indented commands of a recipe's `how`, in order. A block indented four + * spaces is a command; any other block is a paragraph. + */ +export function howBlocks(how: string): { command: boolean; text: string }[] { + return how + .trim() + .split(/\n\s*\n/) + .map((block) => { + const lines = block.split("\n"); + const command = lines.every((line) => line.startsWith(" ")); + return { + command, + text: command + ? lines.map((line) => line.slice(4)).join("\n") + : lines.map((line) => line.trim()).join(" "), + }; + }); +} diff --git a/lib/routes.ts b/lib/routes.ts index 3646825..bb24232 100644 --- a/lib/routes.ts +++ b/lib/routes.ts @@ -135,12 +135,12 @@ export function routeRow(id: string): RouteRow { return row; } -/** The command that creates a project from a connector on one route; `init` asks only how the source is reached. */ +/** The command that creates a project from a connector on one route; `init` asks which source and how it is reached, and in a terminal which assistant files to write. */ export function connectorCommand(connector: string, via: string): string { return `ingestron init sales --connector ${connector} --via ${via}`; } -/** The command that creates a project from a built-in starter; `init` asks nothing more. */ +/** The command that creates a project from a built-in starter; `init` asks only, in a terminal, which assistant files to write. */ export function starterCommand(template: string): string { return `ingestron init ${template === starter ? "shop" : "sales"} --template ${template}`; } diff --git a/package.json b/package.json index 3169caf..b3e63d9 100644 --- a/package.json +++ b/package.json @@ -23,15 +23,15 @@ "sync:vscode": "node scripts/sync-vscode-commands.mjs" }, "dependencies": { - "@ingestron/authoring": "^2.3.1", - "@ingestron/cli": "^2.3.1", - "@ingestron/core": "^1.1.1", - "@ingestron/definitions": "^1.13.0", - "@ingestron/pack-adf": "^2.3.0", - "@ingestron/pack-databricks": "^2.3.0", - "@ingestron/pack-duckdb": "^2.2.0", - "@ingestron/pack-synapse": "^2.2.0", - "@ingestron/standards": "^1.0.0", + "@ingestron/authoring": "^2.4.0", + "@ingestron/cli": "^2.4.0", + "@ingestron/core": "^1.2.0", + "@ingestron/definitions": "^1.14.0", + "@ingestron/pack-adf": "^2.4.0", + "@ingestron/pack-databricks": "^2.4.0", + "@ingestron/pack-duckdb": "^2.3.0", + "@ingestron/pack-synapse": "^2.3.0", + "@ingestron/standards": "^1.1.0", "fflate": "0.8.2", "fumadocs-core": "16.17.8", "fumadocs-mdx": "15.4.10", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 79426e5..cecf694 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,32 +9,32 @@ importers: .: dependencies: '@ingestron/authoring': - specifier: ^2.3.1 - version: 2.3.1 + specifier: ^2.4.0 + version: 2.4.0 '@ingestron/cli': - specifier: ^2.3.1 - version: 2.3.1 + specifier: ^2.4.0 + version: 2.4.0 '@ingestron/core': - specifier: ^1.1.1 - version: 1.1.1 + specifier: ^1.2.0 + version: 1.2.0 '@ingestron/definitions': - specifier: ^1.13.0 - version: 1.13.0 + specifier: ^1.14.0 + version: 1.14.0 '@ingestron/pack-adf': - specifier: ^2.3.0 - version: 2.3.0(@ingestron/definitions@1.13.0) + specifier: ^2.4.0 + version: 2.4.0(@ingestron/definitions@1.14.0) '@ingestron/pack-databricks': - specifier: ^2.3.0 - version: 2.3.0(@ingestron/definitions@1.13.0) + specifier: ^2.4.0 + version: 2.4.0(@ingestron/definitions@1.14.0) '@ingestron/pack-duckdb': - specifier: ^2.2.0 - version: 2.2.0(@ingestron/definitions@1.13.0) + specifier: ^2.3.0 + version: 2.3.0(@ingestron/definitions@1.14.0) '@ingestron/pack-synapse': - specifier: ^2.2.0 - version: 2.2.0(@ingestron/definitions@1.13.0) + specifier: ^2.3.0 + version: 2.3.0(@ingestron/definitions@1.14.0) '@ingestron/standards': - specifier: ^1.0.0 - version: 1.0.0 + specifier: ^1.1.0 + version: 1.1.0 fflate: specifier: 0.8.2 version: 0.8.2 @@ -789,12 +789,12 @@ packages: cpu: [x64] os: [win32] - '@ingestron/authoring@2.3.1': - resolution: {integrity: sha512-Z5yYloIYlyvbjGB4d961HVIdd8erWqjwYX/5of+bxu6jrWHJBdm3b/dVJV/H0l6bgydyVOpaP1NVFUCuJULDOQ==} + '@ingestron/authoring@2.4.0': + resolution: {integrity: sha512-iVKQ1PRhDsqel2rwsTJ4xB4st/BygjH3vksm5K1fZbvDFdK3IZ2h+SjnusXIeLM0ngA9f7x0EBuPINxwNNAvjQ==} engines: {node: '>=22.13.0'} - '@ingestron/cli@2.3.1': - resolution: {integrity: sha512-24CUipuWMChcmwF2Ez20vWEndyF6suIz1eRikesoSA1cTiPt8Zxn7V8axHi8ZbWGSHGLcya3N8B/viNCfUcovw==} + '@ingestron/cli@2.4.0': + resolution: {integrity: sha512-x0gDnj2D7wonhOyt2TSfK6Y+WIq+bHIMFi+5qoaB0fDMa5hXs9tr9e6S2QCZrNIOYjnaP4kVLscWE9a2BwN+RQ==} engines: {node: '>=22.13.0'} hasBin: true @@ -802,44 +802,44 @@ packages: resolution: {integrity: sha512-ybWLmq4xf4RARirY24MxqwzAyS0KDMLMuoWVW71zRxaCXzclDeuKjNx8UYt6Kfhcqwy6aXty2zhJQhj/7LMmbQ==} engines: {node: '>=22.13.0'} - '@ingestron/core@1.1.1': - resolution: {integrity: sha512-srMNFH2s0vKkMs6BsK9AcKgi3M12301QGIkwguOzJtQoRr2jWt7yR2Cy7JodkxNtZeVg1oy4zG86b3c3aZJGHg==} + '@ingestron/core@1.2.0': + resolution: {integrity: sha512-7Xrs3rrd8QuX8f4HGGLltXYAlblqe+rlp8gFLxRHKrKIe/p4kwpSHcGU0vb6Mt5muHKzI31LgU2TB+PbU7QmyA==} engines: {node: '>=22.13.0'} - '@ingestron/definitions@1.13.0': - resolution: {integrity: sha512-/hqZ10uvdlZtVWcyKEB1aPc9rx0WZhjJ3dmG8v3cbcgoAg1maDOT4z54fZpA+7eQY4Yxjj4XWBuwzSRtcjWACQ==} + '@ingestron/definitions@1.14.0': + resolution: {integrity: sha512-Js7tLXptes3tkeeR3rPX0fyBlOMiYQc/QduwBpnH+u9X3HAM+YMX/ZU3JjJ3OYL2YW8Rp+Sl4De+pg6IG4nKoA==} engines: {node: '>=22.13.0'} '@ingestron/generate@1.1.0': resolution: {integrity: sha512-Ouhps0spdV4ueubGYbx2ZGenTkAyA8YvuhuDpHi19uX33RfxlM5ILRSUwnZhg91lPcmJmxYrHnGyfuU98bp2Vw==} engines: {node: '>=22.13.0'} - '@ingestron/pack-adf@2.3.0': - resolution: {integrity: sha512-bMW32GInQ5L1KZb5F/tVDA0iJ/UtczH16n+pzOCFJoKILPr1uu2JIuGWRW4SWlERAHhcCq1/8B7SwBkeWz4CvQ==} + '@ingestron/pack-adf@2.4.0': + resolution: {integrity: sha512-VeQlcsDSXpp9sAXYDfIR8G49HqmrVdKJ+4PMjxVnakUipiL+md1h4FR+lh7pO0URBfRWp4zLYFb3sxv0XmqWdg==} engines: {node: '>=22.13.0'} peerDependencies: - '@ingestron/definitions': ^1.13.0 + '@ingestron/definitions': ^1.14.0 - '@ingestron/pack-databricks@2.3.0': - resolution: {integrity: sha512-LZ0+l/dGBWvxTKcOQgvvMWI10HNdWb2XAIab2CMlEMUrXt4ITMDMfsjCx8HfSdUbsWE/t6vnNozHOO+8ZBDczQ==} + '@ingestron/pack-databricks@2.4.0': + resolution: {integrity: sha512-cp3i2mWWFIWnLe5ZsWbGy0/hawwfajkxms2bdgurbK23j3v3VK9g7PaUmr5dgnXGebjNYhkYbIFAbsh9EZvtZw==} engines: {node: '>=22.13.0'} peerDependencies: - '@ingestron/definitions': ^1.13.0 + '@ingestron/definitions': ^1.14.0 - '@ingestron/pack-duckdb@2.2.0': - resolution: {integrity: sha512-x/C90ZTerzT2anRPRNlTy3hdmU+l+7cYCv5TaLoz/gbj5ZiiTXn75fxZE8WSV5tRaZC2qL+wpBvw+/YaU3vIkQ==} + '@ingestron/pack-duckdb@2.3.0': + resolution: {integrity: sha512-REdhncxcoECTr7MMqyO+nXwkoCcIuFEPcEgQGn8OqAQKi3nf4vanN9ll+C7WQEy1ddEa70s1c4R8wOISRZbYUg==} engines: {node: '>=22.13.0'} peerDependencies: - '@ingestron/definitions': ^1.13.0 + '@ingestron/definitions': ^1.14.0 - '@ingestron/pack-synapse@2.2.0': - resolution: {integrity: sha512-6Xrfg+BZ+oh1nEmMbhou0X6mc5cxtYP3DuEuLqJa8vPs8q+nFaCo6B4XERbclyb+lBIzuu0WVXfcrlUQbdAAgA==} + '@ingestron/pack-synapse@2.3.0': + resolution: {integrity: sha512-i+NnGY6MyfqPSPki8RmspNxT5QOE8oOcgJ353X0Fsk0hhz4dBoEoVBrIDK+MphdonEcXvv0J2GzOOneXAcuO3A==} engines: {node: '>=22.13.0'} peerDependencies: - '@ingestron/definitions': ^1.13.0 + '@ingestron/definitions': ^1.14.0 - '@ingestron/standards@1.0.0': - resolution: {integrity: sha512-QXO4JfaWKZ5fdqYo3uxaPTk+zRLbFLT+++6Tvj+7zFdVBxt7U/SJKC4B8hYgTVQ8k0XjeCHsv4j6ePImNp92MA==} + '@ingestron/standards@1.1.0': + resolution: {integrity: sha512-yYQk2WmuSENBD7p/ZGFjz5D4dSzwLM7iojz1+qm7sCZqI4ucSeA972j/mfE+J9C1FWKigXOONWU44XLdjFYDLg==} engines: {node: '>=22.13.0'} '@jridgewell/gen-mapping@0.3.13': @@ -2012,6 +2012,9 @@ packages: html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} + hyparquet@1.31.5: + resolution: {integrity: sha512-KPVfaxW8xOiO2/4fjeiBfFNEXhGVw9tfcb9ha2iNZdJ3J/yBjEwIeAnTyyuhllFFTb9YO9wg0DfRAOnm6KCvBg==} + inline-style-parser@0.2.7: resolution: {integrity: sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA==} @@ -2685,6 +2688,10 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + txml@6.0.3: + resolution: {integrity: sha512-LpgQIn4huTMpHdBqo9zUYj0lxksvh5epthFh/DoZ4FAd868kkuz31rAzrs41MOPneAWfuSCbBa0+lwxT4p/ZQg==} + engines: {node: '>=18.0.0'} + typescript@6.0.3: resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} engines: {node: '>=14.17'} @@ -3334,19 +3341,19 @@ snapshots: '@img/sharp-win32-x64@0.35.5': optional: true - '@ingestron/authoring@2.3.1': + '@ingestron/authoring@2.4.0': dependencies: '@ingestron/config': 1.7.0 - '@ingestron/core': 1.1.1 - '@ingestron/definitions': 1.13.0 + '@ingestron/core': 1.2.0 + '@ingestron/definitions': 1.14.0 '@ingestron/generate': 1.1.0 yaml: 2.9.1 - '@ingestron/cli@2.3.1': + '@ingestron/cli@2.4.0': dependencies: '@ingestron/config': 1.7.0 - '@ingestron/core': 1.1.1 - '@ingestron/definitions': 1.13.0 + '@ingestron/core': 1.2.0 + '@ingestron/definitions': 1.14.0 '@ingestron/generate': 1.1.0 yaml: 2.9.1 @@ -3355,19 +3362,21 @@ snapshots: ajv: 8.20.0 yaml: 2.9.1 - '@ingestron/core@1.1.1': + '@ingestron/core@1.2.0': dependencies: '@ingestron/config': 1.7.0 - '@ingestron/definitions': 1.13.0 + '@ingestron/definitions': 1.14.0 '@ingestron/generate': 1.1.0 - '@ingestron/pack-adf': 2.3.0(@ingestron/definitions@1.13.0) - '@ingestron/pack-databricks': 2.3.0(@ingestron/definitions@1.13.0) - '@ingestron/pack-duckdb': 2.2.0(@ingestron/definitions@1.13.0) - '@ingestron/pack-synapse': 2.2.0(@ingestron/definitions@1.13.0) - '@ingestron/standards': 1.0.0 + '@ingestron/pack-adf': 2.4.0(@ingestron/definitions@1.14.0) + '@ingestron/pack-databricks': 2.4.0(@ingestron/definitions@1.14.0) + '@ingestron/pack-duckdb': 2.3.0(@ingestron/definitions@1.14.0) + '@ingestron/pack-synapse': 2.3.0(@ingestron/definitions@1.14.0) + '@ingestron/standards': 1.1.0 + hyparquet: 1.31.5 + txml: 6.0.3 yaml: 2.9.1 - '@ingestron/definitions@1.13.0': + '@ingestron/definitions@1.14.0': dependencies: '@ingestron/config': 1.7.0 ajv: 8.20.0 @@ -3376,25 +3385,25 @@ snapshots: '@ingestron/generate@1.1.0': {} - '@ingestron/pack-adf@2.3.0(@ingestron/definitions@1.13.0)': + '@ingestron/pack-adf@2.4.0(@ingestron/definitions@1.14.0)': dependencies: - '@ingestron/definitions': 1.13.0 + '@ingestron/definitions': 1.14.0 - '@ingestron/pack-databricks@2.3.0(@ingestron/definitions@1.13.0)': + '@ingestron/pack-databricks@2.4.0(@ingestron/definitions@1.14.0)': dependencies: - '@ingestron/definitions': 1.13.0 + '@ingestron/definitions': 1.14.0 liquidjs: 10.30.0 yaml: 2.9.1 - '@ingestron/pack-duckdb@2.2.0(@ingestron/definitions@1.13.0)': + '@ingestron/pack-duckdb@2.3.0(@ingestron/definitions@1.14.0)': dependencies: - '@ingestron/definitions': 1.13.0 + '@ingestron/definitions': 1.14.0 - '@ingestron/pack-synapse@2.2.0(@ingestron/definitions@1.13.0)': + '@ingestron/pack-synapse@2.3.0(@ingestron/definitions@1.14.0)': dependencies: - '@ingestron/definitions': 1.13.0 + '@ingestron/definitions': 1.14.0 - '@ingestron/standards@1.0.0': {} + '@ingestron/standards@1.1.0': {} '@jridgewell/gen-mapping@0.3.13': dependencies: @@ -4525,6 +4534,8 @@ snapshots: html-void-elements@3.0.0: {} + hyparquet@1.31.5: {} + inline-style-parser@0.2.7: {} inline-style-parser@0.2.9: {} @@ -5517,6 +5528,8 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + txml@6.0.3: {} + typescript@6.0.3: {} undici-types@6.21.0: {} diff --git a/public/images/vscode/context-menu-dark.png b/public/images/vscode/context-menu-dark.png index 5046b05..7083808 100644 Binary files a/public/images/vscode/context-menu-dark.png and b/public/images/vscode/context-menu-dark.png differ diff --git a/public/images/vscode/context-menu-light.png b/public/images/vscode/context-menu-light.png index 9952886..ddfa4a0 100644 Binary files a/public/images/vscode/context-menu-light.png and b/public/images/vscode/context-menu-light.png differ diff --git a/public/images/vscode/project-view-dark.png b/public/images/vscode/project-view-dark.png index b42df1b..43c788c 100644 Binary files a/public/images/vscode/project-view-dark.png and b/public/images/vscode/project-view-dark.png differ diff --git a/public/images/vscode/project-view-light.png b/public/images/vscode/project-view-light.png index 2ace8cb..635604b 100644 Binary files a/public/images/vscode/project-view-light.png and b/public/images/vscode/project-view-light.png differ diff --git a/public/images/vscode/title-buttons-dark.png b/public/images/vscode/title-buttons-dark.png index f8372ca..2eec705 100644 Binary files a/public/images/vscode/title-buttons-dark.png and b/public/images/vscode/title-buttons-dark.png differ diff --git a/public/images/vscode/title-buttons-light.png b/public/images/vscode/title-buttons-light.png index ec44f47..7b980f8 100644 Binary files a/public/images/vscode/title-buttons-light.png and b/public/images/vscode/title-buttons-light.png differ diff --git a/scripts/ci-templates.test.mjs b/scripts/ci-templates.test.mjs index 05ce41f..74446f7 100644 --- a/scripts/ci-templates.test.mjs +++ b/scripts/ci-templates.test.mjs @@ -58,7 +58,7 @@ test("every YAML template parses, and the folder holds no other kind of file", ( * The ingestron bundle the templates install: an exact version, the one the * docs build with (test/standards-and-teams.test.ts checks the cli's version). */ -const bundle = "2.3.1"; +const bundle = "2.4.0"; test("each check installs an exact ingestron from npm, then runs ingestron check", () => { for (const file of [...checks, "github-actions/deploy.yml", "azure-devops/azure-pipelines.yml"]) { diff --git a/test/areas.test.ts b/test/areas.test.ts index 3cb0640..68fd1c8 100644 --- a/test/areas.test.ts +++ b/test/areas.test.ts @@ -29,6 +29,7 @@ const areas: Record = { "Build a project": [ "/docs/how-to/start-from-a-template", "/docs/how-to/add-a-source", + "/docs/how-to/describe-a-source", "/docs/how-to/write-a-contract", "/docs/how-to/write-a-transform", "/docs/how-to/keep-a-table-current", @@ -86,9 +87,11 @@ const areas: Record = { "/docs/reference/packs/duckdb", "/docs/reference/errors", "/docs/reference/cli", + "/docs/reference/recipes", "/docs/reference/templates", "/docs/reference/versions", "/docs/tools/vscode", + "/docs/tools/assistants", "/docs/troubleshooting", ], }; @@ -198,7 +201,7 @@ describe("page URLs", () => { const built = readFileSync(join(process.cwd(), "test/page-urls.txt"), "utf8") .trim() .split("\n"); - expect(built).toHaveLength(137); + expect(built).toHaveLength(140); expect( source .getPages() diff --git a/test/assistants.test.ts b/test/assistants.test.ts new file mode 100644 index 0000000..34cf8a7 --- /dev/null +++ b/test/assistants.test.ts @@ -0,0 +1,143 @@ +// "Work with a coding assistant", run as the page shows it: the project its +// `init` makes, the files each name writes, the start of the guide, and the +// lines `assistant add`, `plan` and `apply` print, compared with what the +// command prints and writes. +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { run } from "@ingestron/cli"; +import { describe, expect, it } from "vitest"; +import { cli, commands, fences, read, section } from "./pages.js"; + +const page = read("tools/assistants.mdx"); +const lines = commands(page); +const text = (cwd: string, file: string) => readFileSync(join(cwd, file), "utf8"); + +/** The arguments of a command the page shows, which must be there exactly as written. */ +function shown(line: string): string[] { + expect(lines, line).toContain(line); + return line.split(" ").slice(1); +} + +/** The DuckDB starter, made in a new folder by `init` with these arguments. */ +function starter(args: string[]): { parent: string; output: string } { + const parent = mkdtempSync(join(tmpdir(), "docs-assistants-")); + const made = cli(parent, args); + expect(made.code, made.output).toBe(0); + return { parent, output: made.output }; +} + +/** The files the page shows Ingestron writing, by the titles of its blocks. */ +const titles = [...page.matchAll(/^```markdown kind="(?:writes|plain)" title="([^"]+)"$/gm)].map( + (match) => match[1]!, +); +/** The small files that point an assistant at the guide: every file shown but the guide. */ +const pointers = titles.filter((file) => file !== "AGENTS.md"); +expect(pointers).toHaveLength(2); + +/** The `markdown` block whose title is `file`, as the page shows it. */ +function written(file: string): string { + const found = [ + ...page.matchAll(/^```markdown kind="(?:writes|plain)" title="([^"]+)"\n([\s\S]*?)^```$/gm), + ] + .filter((match) => match[1] === file) + .map((match) => match[2]!); + expect(found, file).toHaveLength(1); + return found[0]!; +} + +describe("tools/assistants", () => { + it("writes the files the page shows when init names the assistants", () => { + const init = lines.find((line) => line.startsWith("ingestron init "))!; + const { parent, output } = starter(shown(init)); + const add = section(page, "## Add the guide"); + expect(output).toContain(fences(add, "text")[0]!); + const cwd = join(parent, "shop"); + for (const file of pointers) expect(text(cwd, file)).toBe(written(file)); + expect(text(cwd, "AGENTS.md").startsWith(written("AGENTS.md"))).toBe(true); + // Each name in the table writes the files its row lists. + const rows = section(page, "## The files for each assistant") + .split("\n") + .filter((line) => /^\| `[a-z]+` /.test(line)) + .map((line) => line.split("|").map((cell) => cell.trim())); + // The names the command takes, in its order, as its own mistake message lists them. + const mistake = cli(cwd, ["assistant", "add", "none"]).output.split("\n")[0]!; + const names = /one or more of (.*)\.$/ + .exec(mistake)![1]! + .split(/, | or /) + .map((name) => `\`${name}\``); + expect(rows.map((row) => row[1])).toEqual(names); + for (const row of rows) { + const name = row[1]!.replace(/`/g, ""); + const { parent: folder } = starter([ + "init", + "p", + "--template", + "duckdb", + "--assistant", + name, + "--yes", + ]); + const files = row[3]!.split(", ").map((file) => file.replace(/`/g, "")); + for (const file of ["AGENTS.md", ...pointers]) { + let there = true; + try { + text(join(folder, "p"), file); + } catch { + there = false; + } + expect(there, `${name}: ${file}`).toBe(files.includes(file)); + } + } + }); + + it("asks the question the page shows, and adds the guide to a project", () => { + const add = section(page, "## Add the guide"); + const [, question, added] = fences(add, "text"); + // init asks it in a terminal: the last question, after the project is chosen. + const asked: string[] = []; + const parent = mkdtempSync(join(tmpdir(), "docs-assistants-")); + const io = { cwd: parent, write: () => undefined, ask: (q: string) => (asked.push(q), "") }; + expect(run(["init", "asked", "--template", "duckdb"], io)).toBe(0); + expect(asked.at(-1)!.trim()).toBe(question!.trim()); + const cwd = join(starter(["init", "shop", "--template", "duckdb", "--yes"]).parent, "shop"); + const first = cli(cwd, shown("ingestron assistant add agents")); + expect(first.code).toBe(0); + expect(first.output).toBe(added); + expect(cli(cwd, ["assistant", "add", "agents"]).output).toBe("AGENTS.md: up to date\n"); + }); + + it("prints the lines the page shows as plan and apply keep the section current", () => { + const keep = section(page, "## Keep it current"); + const [wouldChange, updated, planEdited, edited] = fences(keep, "text").map((block) => + block.trim(), + ); + const { parent } = starter(["init", "shop", "--template", "duckdb", "--yes"]); + const cwd = join(parent, "shop"); + cli(cwd, ["assistant", "add", "agents"]); + expect(cli(cwd, ["apply", "--no-diff"]).code).toBe(0); + // An environment added to project.yaml changes the facts. + writeFileSync( + join(cwd, "project.yaml"), + text(cwd, "project.yaml").replace(" dev: {}\n", " dev: {}\n prod: {}\n"), + ); + const plan = cli(cwd, ["plan", "--no-diff"]); + expect(plan.output.trimEnd().split("\n").at(-1)).toBe(wouldChange); + const apply = cli(cwd, ["apply", "--no-diff"]); + expect(apply.code).toBe(0); + expect(apply.output.trimEnd().split("\n").at(-1)).toBe(updated); + expect(text(cwd, "AGENTS.md")).toContain("Environments: `dev`, `prod`"); + // A section edited by hand is left, and said so; check never looks at it. + writeFileSync( + join(cwd, "AGENTS.md"), + text(cwd, "AGENTS.md").replace("3. Work inside the standard.", "3. Always work inside it."), + ); + const before = text(cwd, "AGENTS.md"); + expect(cli(cwd, ["plan", "--no-diff"]).output.trimEnd().split("\n").at(-1)).toBe(planEdited); + const again = cli(cwd, ["apply", "--no-diff"]); + expect(again.code).toBe(0); + expect(again.output.trimEnd().split("\n").at(-1)).toBe(edited); + expect(text(cwd, "AGENTS.md")).toBe(before); + expect(cli(cwd, ["check"]).code).toBe(0); + }); +}); diff --git a/test/block-kind.test.ts b/test/block-kind.test.ts index 96fea24..f3ec03f 100644 --- a/test/block-kind.test.ts +++ b/test/block-kind.test.ts @@ -116,8 +116,11 @@ describe("the kinds on the real pages", () => { expect(line.trimStart().startsWith("$ "), `${page}: ${line}`).toBe(false); }); - it("make every YAML, CSV and dotenv fence a file the reader writes", () => { - for (const fence of fences.filter((found) => ["yaml", "csv", "dotenv"].includes(found.lang))) + it("make every unmarked YAML, CSV and dotenv fence a file the reader writes", () => { + for (const fence of fences.filter( + (found) => + ["yaml", "csv", "dotenv"].includes(found.lang) && !/(?:^|\s)kind=/.test(found.meta), + )) expect(fence.kind, `${fence.page}: ${fence.lang}`).toBe("write"); }); @@ -132,9 +135,22 @@ describe("the kinds on the real pages", () => { }); it("carry a kind mark only where the rest cannot place the block", () => { - const marked = fences.filter((fence) => /(?:^|\s)kind=/.test(fence.meta)); + const all = fences.filter((fence) => /(?:^|\s)kind=/.test(fence.meta)); + // The files the project guide for coding assistants writes, outside build/: the guide, + // whose section Ingestron keeps, and two small files that are the reader's from then on. + const guide = all.filter((fence) => fence.page.endsWith("/assistants.mdx")); + expect( + guide.map((fence) => /^kind="([a-z]+)" title="[^"]+\.md"$/.exec(fence.meta)?.[1]), + ).toEqual(["plain", "plain", "writes"]); + const marked = all.filter((fence) => !guide.includes(fence)); + // A payload a server returns; the source metadata an import writes outside build/; and a + // contract Ingestron drafts that is the reader's to edit from then on, so neither a file + // the reader writes nor a generated one. expect(marked.map((fence) => [fence.page.split("/").pop(), fence.meta])).toEqual([ ["rest.mdx", 'kind="plain"'], + ["describe-a-source.mdx", 'kind="writes" title="sources/erp.yaml"'], + ["describe-a-source.mdx", 'kind="writes" title="sources/erp.yaml"'], + ["describe-a-source.mdx", 'kind="plain" title="contracts/erp.payments.yaml"'], ]); }); }); diff --git a/test/describe-a-source.test.ts b/test/describe-a-source.test.ts new file mode 100644 index 0000000..2410892 --- /dev/null +++ b/test/describe-a-source.test.ts @@ -0,0 +1,215 @@ +// "Describe a source", run as the page shows it: the project its `init` makes, +// each command it runs, each file it saves or adds to, and each output, draft +// and metadata it shows, compared with what the command prints and writes. +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ProjectSnapshot } from "@ingestron/authoring"; +import { installedSourceRecipes } from "@ingestron/core"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; +import { recipesFolder } from "../lib/recipes"; +import { cli, commands, fences, read, section } from "./pages.js"; + +const page = read("how-to/describe-a-source.mdx"); +const part = (heading: string) => section(page, heading); +const lines = commands(page); + +/** The arguments of a command the page shows, which must be there exactly as written. */ +function shown(line: string): string[] { + expect(lines, line).toContain(line); + return line.split(" ").slice(1); +} + +/** + * The page shows the request as Ingestron's own catalogue of recipes gives it. When the + * command these tests run has no catalogue installed, the project keeps the `sqlserver` + * recipe as its own, and the request then names the recipe's file, which this leaves out. + */ +const installed = installedSourceRecipes().some((recipe) => recipe.id === "sqlserver"); +const asInstalled = (text: string) => + text.replace(" (`sqlserver`, from `recipes/sqlserver.yaml`)", " (`sqlserver`)"); + +/** The time of each import, as the page writes it, for output that holds the time of the run. */ +const asOnPage = (text: string) => + text + .replace(/at: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z/g, "at: 2026-11-13T09:30:00Z") + .replace(/captured \d{4}-\d{2}-\d{2}/g, "captured 2026-11-13"); + +/** The `sales` project the page creates, generated, and with the recipe the request uses. */ +function sales(): string { + const parent = mkdtempSync(join(tmpdir(), "docs-describe-a-source-")); + const init = lines.find((line) => line.startsWith("ingestron init "))!; + const created = cli(parent, shown(init)); + expect(created.code, created.output).toBe(0); + expect(created.output).toContain("Nothing to replace: the project runs as written."); + const cwd = join(parent, lines.find((line) => line.startsWith("cd "))!.slice("cd ".length)); + expect(cli(cwd, shown("ingestron apply")).code).toBe(0); + if (!installed) { + mkdirSync(join(cwd, "recipes")); + copyFileSync(join(recipesFolder, "sqlserver.yaml"), join(cwd, "recipes/sqlserver.yaml")); + } + return cwd; +} + +const text = (cwd: string, file: string) => readFileSync(join(cwd, file), "utf8"); + +/** Adds the page's lines at the end of a file, as a reader does. */ +const append = (cwd: string, file: string, lines: string) => + writeFileSync(join(cwd, file), `${text(cwd, file)}${lines}`); + +/** Writes the page's table line under the flow's `tables`, after the tables `init` wrote, keeping the file's comments and lines. */ +function table(cwd: string, line: string): void { + const flow = text(cwd, "flows/erp.yaml"); + const payments = /^ {2}payments:.*\n/m; + writeFileSync( + join(cwd, "flows/erp.yaml"), + payments.test(flow) + ? flow.replace(payments, line) + : flow.replace(/^ingest:/m, `${line}ingest:`), + ); +} + +/** Runs a command the page shows and returns what it printed, after checking its exit code. */ +function ran(cwd: string, line: string, code = 0): string { + const result = cli(cwd, shown(line)); + expect(result.code, result.output).toBe(code); + return result.output; +} + +describe("how-to/describe-a-source", () => { + it("asks for the metadata, imports the catalog, loads a table without a contract and drafts the one needed", () => { + const cwd = sales(); + + // Ask for the metadata, with the recipe's input in the connection. + const ask = part("## Ask for the metadata [#request]"); + append(cwd, "connections/erp.yaml", fences(ask, "yaml")[0]!); + const request = ran(cwd, "ingestron source request"); + expect(asInstalled(request)).toContain(fences(ask, "text")[0]!.trim()); + expect(request).toContain("Save the result as `sources/erp.catalog.csv`."); + expect(request).toContain("s.name IN ('dbo')"); + expect(request).toContain("### If that cannot be done"); + + // Save the catalog's result, check it, then import it. + const catalog = part("### The recipe's query or call [#catalog]"); + expect(catalog.replace(/\s+/g, " ")).toContain("saves the result as `sources/erp.catalog.csv`"); + mkdirSync(join(cwd, "sources")); + writeFileSync(join(cwd, "sources/erp.catalog.csv"), fences(catalog, "csv")[0]!); + const imported = part("## Import it [#import]"); + const [dry, wrote] = fences(imported, "text"); + expect(ran(cwd, "ingestron source import erp sources/erp.catalog.csv --dry-run").trim()).toBe( + dry!.trim(), + ); + expect(ran(cwd, "ingestron source import erp sources/erp.catalog.csv").trim()).toBe( + wrote!.trim(), + ); + expect(parse(text(cwd, "project.yaml")).imports).toContain("sources/erp.yaml"); + expect(asOnPage(text(cwd, "sources/erp.yaml"))).toContain(fences(imported, "yaml")[0]!); + expect(asOnPage(ran(cwd, "ingestron source request"))).toContain( + fences(imported, "text")[2]!.trim(), + ); + + // A table without a contract takes its columns and key from the metadata. + const without = part("## Load a table without a contract [#without-a-contract]"); + table(cwd, fences(without, "yaml")[0]!); + expect(ran(cwd, "ingestron plan")).toContain(fences(without, "text")[0]!.trim()); + expect(cli(cwd, ["apply", "--no-diff"]).code).toBe(0); + + // Grants make it need a contract; the draft is the one the page shows. + const needs = part("## When a table needs a contract [#needs-a-contract]"); + const [grants, draft, named] = fences(needs, "yaml"); + append(cwd, "platform.yaml", grants!); + const [problem, drafted] = fences(needs, "text"); + const failed = cli(cwd, ["plan", "--no-diff"]); + expect(failed.code).toBe(1); + expect(failed.output).toContain(problem!.trim()); + // VS Code's fix for the problem, as the page names it. + const snapshot = ProjectSnapshot.open(cwd); + const needed = snapshot.problems("dev").find((found) => found.code === "CONTRACT_NEEDED")!; + const fix = snapshot + .quickFixes("dev", needed.file, needed.range) + .find((found) => found.problem?.code === "CONTRACT_NEEDED"); + expect(needs.replace(/\s+/g, " ")).toContain(`**${fix!.title}**`); + expect(ran(cwd, "ingestron contract --from-metadata --needed").trim()).toBe(drafted!.trim()); + expect(asOnPage(text(cwd, "contracts/erp.payments.yaml"))).toBe(draft); + table(cwd, named!); + expect(cli(cwd, ["plan", "--no-diff"]).code).toBe(0); + }, 120_000); + + it("reports what changed at the source, fails on a breaking change, and warns or accepts it as the platform says", () => { + const cwd = sales(); + append( + cwd, + "connections/erp.yaml", + fences(part("## Ask for the metadata [#request]"), "yaml")[0]!, + ); + mkdirSync(join(cwd, "sources")); + const csv = fences(part("### The recipe's query or call [#catalog]"), "csv")[0]!; + writeFileSync(join(cwd, "sources/erp.catalog.csv"), csv); + expect(cli(cwd, ["source", "import", "erp", "sources/erp.catalog.csv"]).code).toBe(0); + + // The ERP drops the invoices' status and adds due_on, as the page says. + const changes = part("## When the source changes [#changes]"); + expect(changes.replace(/\s+/g, " ")).toContain( + "drops the invoices' `status` and adds `due_on`", + ); + const status = "ERP,dbo,Invoices,status,4,nvarchar(20),YES,"; + expect(csv).toContain(status); + writeFileSync( + join(cwd, "sources/erp.catalog.csv"), + csv.replace(status, "ERP,dbo,Invoices,due_on,4,date,YES,"), + ); + const [reimported, gone] = fences(changes, "text"); + expect(cli(cwd, ["source", "import", "erp", "sources/erp.catalog.csv"]).output.trim()).toBe( + reimported!.trim(), + ); + for (const command of [["plan", "--no-diff"], ["check"]]) { + const failed = cli(cwd, command); + expect(failed.code).toBe(1); + expect(failed.output).toContain(gone!.trim()); + } + + const [warn, accept] = fences(changes, "yaml"); + append(cwd, "platform.yaml", warn!); + const warned = cli(cwd, ["plan", "--no-diff"]); + expect(warned.code).toBe(0); + expect(warned.output).toContain("warning SOURCE_COLUMN_GONE"); + append(cwd, "platform.yaml", accept!); + const accepted = cli(cwd, ["plan", "--no-diff"]); + expect(accepted.code).toBe(0); + expect(accepted.output).not.toContain("SOURCE_COLUMN_GONE"); + expect(accepted.output).toContain("1 accepted warning (platform.yaml)"); + expect(changes).toContain("`1 accepted warning (platform.yaml)`"); + // The reason's table reads the column the source dropped. + expect(text(cwd, "transforms/customer_balances.sql")).toContain("i.status"); + expect(accept).toContain("customer_balances"); + }, 120_000); + + it("reads a nested JSON sample into the columns the page shows, and keeps no value", () => { + const cwd = sales(); + const sample = part("### A sample of each table [#sample]"); + const file = "sources/erp/ERP.dbo.Orders.json"; + expect(sample).toContain(`saved as \`${file}\``); + mkdirSync(join(cwd, "sources/erp"), { recursive: true }); + const json = fences(sample, "json")[0]!; + writeFileSync(join(cwd, file), json); + expect(cli(cwd, ["source", "import", "erp", file]).code).toBe(0); + const metadata = asOnPage(text(cwd, "sources/erp.yaml")); + expect(metadata).toContain(fences(sample, "yaml")[0]!); + for (const value of ["Ana", "gold", "A-1", "2026-11-02", "40.5", "1001", "tier", "gift"]) + expect(metadata).not.toContain(value); + }, 60_000); + + it("offers an assistant's columns file once the project has a guide", () => { + const cwd = sales(); + const assistant = part("### A coding assistant's columns file [#assistant]"); + expect(cli(cwd, shown("ingestron assistant add agents")).code).toBe(0); + expect(assistant).toContain("`sources/.assistant.csv`"); + const guide = text(cwd, "AGENTS.md"); + expect(guide).toContain("sources/.assistant.csv"); + expect(guide).toContain("--dry-run"); + expect(cli(cwd, ["source", "request"]).output).toContain( + "saved as `sources/erp.assistant.csv`", + ); + }, 60_000); +}); diff --git a/test/page-urls.txt b/test/page-urls.txt index d44380a..b6998a8 100644 --- a/test/page-urls.txt +++ b/test/page-urls.txt @@ -96,6 +96,7 @@ /docs/how-to/configure-a-standard /docs/how-to/define-a-data-product /docs/how-to/deploy-through-environments +/docs/how-to/describe-a-source /docs/how-to/grant-access /docs/how-to/keep-a-table-current /docs/how-to/kits/databricks-lakehouse-streaming @@ -127,11 +128,13 @@ /docs/reference/packs/databricks /docs/reference/packs/duckdb /docs/reference/packs/synapse +/docs/reference/recipes /docs/reference/templates /docs/reference/versions /docs/start/choose-a-route /docs/start/first-project /docs/start/how-it-works /docs/start/install +/docs/tools/assistants /docs/tools/vscode /docs/troubleshooting diff --git a/test/platform-pages.test.ts b/test/platform-pages.test.ts index 7853f2f..651ede2 100644 --- a/test/platform-pages.test.ts +++ b/test/platform-pages.test.ts @@ -291,6 +291,6 @@ describe("reference/versions", () => { expect(tools).toContain(`\`@microsoft/azure-data-factory-utilities\` ${adf}`); expect(tools).toContain(`Databricks CLI ${databricks}`); expect(tools).toContain(`sqlcmd (Go) ${sqlcmd}`); - expect(tools).toContain("`ingestron@2.3.1`"); + expect(tools).toContain("`ingestron@2.4.0`"); }); }); diff --git a/test/recipes.test.ts b/test/recipes.test.ts new file mode 100644 index 0000000..2be2c89 --- /dev/null +++ b/test/recipes.test.ts @@ -0,0 +1,226 @@ +// Source metadata, as the site documents it: the recipes page lists exactly the +// recipes installed with Ingestron, each with its inputs, defaults and +// documentation; connector pages show the recipe their connections use; and the +// CLI, document and error references pick up the source metadata features from +// the code, with no copy written by hand. +import { readdirSync } from "node:fs"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it, vi } from "vitest"; +import { commands, flags } from "@ingestron/cli"; +import { installedSourceRecipes } from "@ingestron/core"; +import { codeHelp, loadingCodeHelp, documentReference } from "@ingestron/definitions"; +import { ConnectorPage } from "../components/reference/connector-page"; +import { CliCommands } from "../components/reference/cli-commands"; +import { ErrorCodes, LoadingCodes } from "../components/reference/error-codes"; +import { FieldTable } from "../components/reference/document-fields"; +import { RecipeCatalog, RecipeList, RecipePlaceholders } from "../components/reference/recipes"; +import { connectorIds } from "../lib/reference.js"; +import { + connectorRecipes, + howBlocks, + placeholdersUsed, + recipesFolder, + recipes, + recipesThatFit, + reservedPlaceholders, +} from "../lib/recipes.js"; +import { anchors, read } from "./pages.js"; + +// The code blocks render asynchronously in the site; here each shows its code. +vi.mock("fumadocs-ui/components/codeblock.rsc", () => ({ + ServerCodeBlock: ({ code }: { code: string }) => createElement("pre", null, code), +})); +vi.mock("@/components/example", () => ({ + Example: ({ name, file }: { name: string; file: string }) => + createElement("pre", { "data-example": `${name}/${file}` }), +})); +vi.setConfig({ testTimeout: 30_000 }); + +const html = (element: Parameters[0]) => renderToStaticMarkup(element); +/** What a page shows for text: the characters HTML escapes, escaped. */ +const escaped = (text: string) => + text + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); + +describe("the recipes page", () => { + it("lists exactly the recipes the catalogue installs", () => { + const catalogue = installedSourceRecipes(recipesFolder).map((recipe) => recipe.id); + const files = readdirSync(recipesFolder) + .filter((name) => name.endsWith(".yaml")) + .map((name) => name.replace(/\.yaml$/, "")) + .sort(); + expect(recipes().map((recipe) => recipe.id)).toEqual(catalogue); + expect(catalogue).toEqual(files); + expect(catalogue).toHaveLength(11); + expect(catalogue).not.toContain("servicenow"); + const shown = html(createElement(RecipeCatalog)); + expect([...shown.matchAll(/

match[1])).toEqual(catalogue); + const listed = html(createElement(RecipeList)); + expect( + [...listed.matchAll(/href="\/docs\/reference\/recipes#([^"]+)"/g)].map((match) => match[1]), + ).toEqual(catalogue); + }); + + it("shows each recipe's who, access, step, save path, inputs with defaults and documentation", () => { + const shown = html(createElement(RecipeCatalog)); + for (const recipe of recipes()) { + for (const text of [recipe.title, recipe.ask.who, recipe.ask.access, recipe.ask.save]) + expect(shown, recipe.id).toContain(escaped(text)); + const step = recipe.ask.run ?? recipe.ask.call; + if (step) expect(shown, recipe.id).toContain(escaped(step.trimEnd())); + for (const [name, input] of Object.entries(recipe.inputs)) { + expect(shown, `${recipe.id}: ${name}`).toContain(`${name}`); + expect(shown, `${recipe.id}: ${name}`).toContain(escaped(input.description)); + if (input.default !== undefined) expect(shown).toContain(escaped(input.default)); + } + for (const source of recipe.sources) { + expect(shown, recipe.id).toContain(`href="${escaped(source.url)}"`); + expect(shown, recipe.id).toContain(`(read ${source.read})`); + } + } + }); + + it("explains every placeholder the recipes write, from the catalogue", () => { + const used = placeholdersUsed(); + expect(used.size).toBeGreaterThan(3); + for (const [name, ids] of used) + for (const id of ids) + expect( + name in reservedPlaceholders || + name in recipes().find((recipe) => recipe.id === id)!.inputs, + `{${name}} in ${id}`, + ).toBe(true); + const shown = html(createElement(RecipePlaceholders)); + for (const name of Object.keys(reservedPlaceholders)) + expect(shown.includes(`{${name}}`), name).toBe(used.has(name)); + expect(read("reference/recipes.mdx")).toContain(""); + }); + + it('shows "None" for a recipe that is no connector\'s default', () => { + const listed = html(createElement(RecipeList)); + const row = (id: string) => listed.split("

").find((part) => part.includes(`#${id}"`))!; + expect(row("mysql")).toMatch(/
None<\/td><\/tr>/); + expect(row("sqlserver")).toContain('href="/docs/connectors/sqlserver"'); + }); + + it("splits a recipe's instructions into paragraphs and commands", () => { + const blocks = howBlocks("Save it.\nThen run:\n\n curl one\n curl two\n\nDone."); + expect(blocks).toEqual([ + { command: false, text: "Save it. Then run:" }, + { command: true, text: "curl one\ncurl two" }, + { command: false, text: "Done." }, + ]); + }); + + it("is a page of the Reference area, in the contents and in the URLs", () => { + const page = read("reference/recipes.mdx"); + expect(page).toContain(""); + expect(page).toContain(""); + expect(read("reference/meta.json")).toContain('"recipes"'); + // A link to a field's row, such as `#connection-metadata`, is checked with the documents. + for (const link of page.matchAll(/\]\((\/docs\/reference\/[a-z-]+)#([^)]+)\)/g)) { + if (link[2]!.startsWith("connection-")) continue; + const target = link[1]!.replace("/docs/", ""); + expect([...anchors(read(`${target}.mdx`)).keys()], link[0]).toContain(link[2]!); + } + }); +}); + +describe("connector pages", () => { + it("show the recipe a connector uses by default, and one that has its id", () => { + const byConnector = Object.fromEntries( + connectorIds().map((id) => [ + id, + connectorRecipes(id).map((item) => `${item.recipe.id}:${item.how}`), + ]), + ); + expect(byConnector.sqlserver).toEqual(["sqlserver:default"]); + expect(byConnector.salesforce).toEqual(["salesforce:default"]); + expect(byConnector.kafka).toEqual(["schema-registry:default"]); + expect(byConnector.files).toEqual(["sample:default"]); + expect(byConnector.mysql).toEqual(["mysql:named"]); + expect(byConnector.rest).toEqual(["openapi:fits"]); + expect(byConnector["dynamics-365"]).toEqual(["dataverse:fits"]); + expect(byConnector.odata).toEqual(["odata:named"]); + expect(byConnector["sap-ecc"]).toEqual(["sap-odata:fits"]); + expect(byConnector["sap-table"]).toEqual([]); + for (const id of ["sqlserver", "mysql", "files", "rest"]) { + const page = html(createElement(ConnectorPage, { id })); + expect(page, id).toContain('

{ + const ids = recipes().map((recipe) => recipe.id); + for (const [connector, fitting] of Object.entries(recipesThatFit)) { + expect(connectorIds(), connector).toContain(connector); + for (const id of fitting) expect(ids, `${connector}: ${id}`).toContain(id); + } + }); +}); + +describe("the references", () => { + it("list the source and assistant commands and their options from the CLI", () => { + const names = commands.map((command) => command.name); + for (const name of ["source", "assistant"]) expect(names).toContain(name); + const shown = html(createElement(CliCommands)); + for (const { name } of commands) expect(shown).toContain(`ingestron ${name}`); + const options = flags.map((flag) => flag.flag); + for (const flag of [ + "--from-metadata", + "--needed", + "--dry-run", + "--object", + "--connection", + "--assistant", + ]) + expect( + options.some((option) => option.startsWith(flag)), + flag, + ).toBe(true); + for (const { flag } of flags) expect(shown).toContain(`${escaped(flag)}`); + expect(read("reference/cli.mdx")).toContain("## Source metadata [#source-metadata]"); + }); + + it("give every field of a SourceMetadata document and a connection's metadata its anchor", () => { + const page = read("reference/documents.mdx"); + expect(page).toContain(''); + expect(page).toContain(''); + const fields = [ + ...documentReference("SourceMetadata"), + ...documentReference("Connection").filter((field) => field.path[0] === "metadata"), + ]; + expect(fields.length).toBeGreaterThan(20); + const shown = html(createElement(FieldTable, { label: "x", fields })); + for (const field of fields) { + const [path, anchor] = field.docs.split("#"); + expect(path).toBe("/docs/reference/documents"); + expect(shown).toContain(`id="${anchor}"`); + } + }); + + it("explain every code the definitions report, the source metadata ones included", () => { + const shown = html(createElement(ErrorCodes)) + html(createElement(LoadingCodes)); + const codes = Object.keys({ ...codeHelp, ...loadingCodeHelp }); + for (const code of codes) expect(shown, code).toContain(`id="${code}"`); + const source = codes.filter((code) => code.startsWith("SOURCE_")); + expect(source.length).toBeGreaterThanOrEqual(8); + for (const code of [...source, "CONTRACT_NEEDED", "TABLE_NO_COLUMNS"]) { + expect(codes).toContain(code); + const entry = shown.slice(shown.indexOf(`id="${code}"`)); + expect(entry.slice(0, entry.indexOf(""))).toContain("Fix:"); + } + }); +}); diff --git a/test/site-journey.test.ts b/test/site-journey.test.ts index 2647704..c694bb7 100644 --- a/test/site-journey.test.ts +++ b/test/site-journey.test.ts @@ -42,6 +42,7 @@ it("opens with one learning path, then areas by the reader's job, with no index expect(JSON.parse(read("how-to/(build)/meta.json")).pages).toEqual([ "start-from-a-template", "add-a-source", + "describe-a-source", "write-a-contract", "write-a-transform", "keep-a-table-current", @@ -100,10 +101,11 @@ it("opens with one learning path, then areas by the reader's job, with no index "packs", "errors", "cli", + "recipes", "templates", "versions", ]); - expect(JSON.parse(read("tools/meta.json")).pages).toEqual(["vscode"]); + expect(JSON.parse(read("tools/meta.json")).pages).toEqual(["vscode", "assistants"]); // The reference, tools and examples indexes are gone, and their URLs land on a page. for (const [section, first] of [ ["reference", "reference/documents"], diff --git a/test/standards-and-teams.test.ts b/test/standards-and-teams.test.ts index ec0e23f..4e307e2 100644 --- a/test/standards-and-teams.test.ts +++ b/test/standards-and-teams.test.ts @@ -190,7 +190,8 @@ describe("how-to/share-a-template", () => { .replace(standard![1]!, platformRepo) .split(" ") .slice(1); - // The page's command as written: init asks nothing. + // The page's command as written: init asks nothing about the template's values, only, + // last and in a terminal, which assistant files to write. const asked: string[] = []; let output = ""; const code = run(args, { @@ -200,7 +201,10 @@ describe("how-to/share-a-template", () => { }); const created = { code, output }; expect(created.code, created.output).toBe(0); - expect(asked, "init asks nothing").toEqual([]); + expect( + asked.filter((question) => !question.includes("assistant files")), + "init asks nothing else", + ).toEqual([]); // The template's value is still REPLACE_ME, and init says so. expect(created.output).toContain( "1 value is still REPLACE_ME: README.md lists each one and where to find it.", @@ -326,7 +330,7 @@ describe("how-to/upgrade", () => { it("pins the exact bundle these pages describe, as the templates install it", () => { // The ingestron bundle these pages describe: the installed cli's version. - const bundle = "2.3.1"; + const bundle = "2.4.0"; const choose = section(page, "## Choose the version you run"); const version = JSON.parse( readFileSync(join(process.cwd(), "node_modules/@ingestron/cli/package.json"), "utf8"), diff --git a/test/troubleshooting.test.ts b/test/troubleshooting.test.ts index 0d158f7..2cae4ed 100644 --- a/test/troubleshooting.test.ts +++ b/test/troubleshooting.test.ts @@ -376,6 +376,68 @@ describe("2.0", () => { }, 30_000); }); +describe("source metadata", () => { + it("quotes a table with no columns, and its fix: a contract, or metadata for the table", () => { + const project = copy("duckdb"); + edit(project, "flows/orders.yaml", (text) => text.replace(", contract: shop.orders", "")); + const failed = cli(project, ["plan", "--no-diff"]); + expect(failed.code).toBe(1); + quotes("A table has no columns", failed.output); + // The source's metadata for the table gives it its columns. + writeFileSync(join(project, "orders.csv"), readFileSync(join(project, "samples/orders.csv"))); + const imported = cli(project, [ + "source", + "import", + "shop", + "orders.csv", + "--object", + "orders.csv", + ]); + expect(imported.code, imported.output).toBe(0); + expect(cli(project, ["plan", "--no-diff"]).code).toBe(0); + }, 30_000); + + it("quotes a table that needs a contract, and naming one fixes it", () => { + const project = copy("duckdb"); + edit(project, "flows/orders.yaml", (text) => + text.replace( + " orders: { object: orders.csv, contract: shop.orders }", + " orders:\n object: orders.csv\n expectations:\n - { name: positive_amount, check: amount > 0 }", + ), + ); + const failed = cli(project, ["plan", "--no-diff"]); + expect(failed.code).toBe(1); + quotes("A table needs a contract [#contract-needed]", failed.output); + edit(project, "flows/orders.yaml", (text) => + text.replace( + " object: orders.csv\n", + " object: orders.csv\n contract: shop.orders\n", + ), + ); + expect(cli(project, ["plan", "--no-diff"]).output).not.toContain("CONTRACT_NEEDED"); + }, 30_000); + + it("quotes a column gone at the source, and a warning once breaking changes warn", () => { + const project = copy("duckdb"); + writeFileSync(join(project, "orders.csv"), "order_id,customer,amount\n1,Ana,10.5\n"); + expect( + cli(project, ["source", "import", "shop", "orders.csv", "--object", "orders.csv"]).code, + ).toBe(0); + for (const command of [["plan", "--no-diff"], ["check"]]) { + const failed = cli(project, command); + expect(failed.code).toBe(1); + quotes("A change at the source fails the plan [#source-changed]", failed.output); + } + edit(project, "platform.yaml", (text) => `${text}sources: { breaking: warn }\n`); + const warned = cli(project, ["plan", "--no-diff"]); + expect(warned.code).toBe(0); + expect(warned.output).toContain("warning SOURCE_COLUMN_GONE"); + expect(entry("A change at the source fails the plan [#source-changed]")).toContain( + "`sources: { breaking: warn }`", + ); + }, 30_000); +}); + describe("environment variables and .env", () => { const unset = () => { const project = copy("duckdb");