From 1af9940bbb9fb144c345eec15fc4e89be7b705f0 Mon Sep 17 00:00:00 2001 From: The Architect Date: Thu, 5 Nov 2026 17:20:10 +1300 Subject: [PATCH 01/39] CI templates: deploy one build through dev, test and prod GitHub Actions (deploy.yml and deploy-environment.yml) and Azure DevOps (azure-pipelines.yml and templates/deploy-environment.yml) check the generated files, build the Data Factory template once with Microsoft's package 1.0.3, and deploy the same files to each environment in turn, gated by that environment's approvals. Deploy steps use only the platforms' tools: the pre- and post-deployment script and an incremental ARM deployment, the Databricks CLI v1.23.0, and sqlcmd (Go) v1.10.0 checked by its SHA-256. The check templates install ingestron@2. Signed-off-by: The Architect --- ci-templates/azure-devops/azure-pipelines.yml | 68 ++++++ ci-templates/azure-devops/ingestron-check.yml | 2 +- .../templates/deploy-environment.yml | 146 +++++++++++ .../github-actions/deploy-environment.yml | 121 ++++++++++ ci-templates/github-actions/deploy.yml | 79 ++++++ .../github-actions/ingestron-check.yml | 2 +- scripts/ci-templates.test.mjs | 228 +++++++++++++++++- 7 files changed, 631 insertions(+), 15 deletions(-) create mode 100644 ci-templates/azure-devops/azure-pipelines.yml create mode 100644 ci-templates/azure-devops/templates/deploy-environment.yml create mode 100644 ci-templates/github-actions/deploy-environment.yml create mode 100644 ci-templates/github-actions/deploy.yml diff --git a/ci-templates/azure-devops/azure-pipelines.yml b/ci-templates/azure-devops/azure-pipelines.yml new file mode 100644 index 0000000..e28f868 --- /dev/null +++ b/ci-templates/azure-devops/azure-pipelines.yml @@ -0,0 +1,68 @@ +# Deploys the committed build/ folder: checks it, builds the Data Factory +# template once, then deploys the same files to dev, test and prod in turn. +# Copy this file and templates/deploy-environment.yml to your definitions +# repository, add this file as a pipeline, and delete the steps of each +# platform your project does not use. +# +# Set up once, as https://docs.ingestron.io/docs/how-to/deploy-through-environments +# explains: +# - Azure DevOps environments named dev, test and prod. Add approvals to test +# and prod, and a branch control check to prod that allows only main. +# - For each environment, an Azure Resource Manager service connection that +# uses workload identity federation, and a variable group named +# ingestron- with the variables the template reads. +# - The pipeline variable ADF_DEV_FACTORY_ID: the dev factory's resource ID, +# which names the template's resources. Nothing is deployed with it. +trigger: + branches: + include: [main] +pr: none + +stages: + - stage: Build + jobs: + - job: build + pool: + vmImage: ubuntu-latest + steps: + - checkout: self + - task: UseNode@1 + inputs: + version: 22.x + - script: npm install -g ingestron@2 + displayName: Install Ingestron + - script: ingestron check + displayName: ingestron check + # Data Factory: Microsoft's build validates the factory, then writes + # the ARM template and the pre- and post-deployment script to arm/. + # It downloads its build tool from adf.azure.com. + - script: | + npm install --prefix "$(Agent.TempDirectory)/adf" @microsoft/azure-data-factory-utilities@1.0.3 + adf="$(Agent.TempDirectory)/adf/node_modules/@microsoft/azure-data-factory-utilities/lib/index" + node "$adf" validate build/adf/factory "$(ADF_DEV_FACTORY_ID)" + node "$adf" export build/adf/factory "$(ADF_DEV_FACTORY_ID)" arm + displayName: "Data Factory: build the template" + # Every environment deploys these files, so prod gets the bytes dev + # got. Without Data Factory, copy build alone. + - script: cp -R arm build "$(Build.ArtifactStagingDirectory)/" + displayName: Collect the generated files + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(Build.ArtifactStagingDirectory) + artifact: deploy + + - template: templates/deploy-environment.yml + parameters: + environment: dev + serviceConnection: ingestron-dev + dependsOn: Build + - template: templates/deploy-environment.yml + parameters: + environment: test + serviceConnection: ingestron-test + dependsOn: Deploy_dev + - template: templates/deploy-environment.yml + parameters: + environment: prod + serviceConnection: ingestron-prod + dependsOn: Deploy_test diff --git a/ci-templates/azure-devops/ingestron-check.yml b/ci-templates/azure-devops/ingestron-check.yml index 2424a39..f2ad384 100644 --- a/ci-templates/azure-devops/ingestron-check.yml +++ b/ci-templates/azure-devops/ingestron-check.yml @@ -13,7 +13,7 @@ steps: - task: NodeTool@0 inputs: versionSpec: 22.x - - script: npm install -g ingestron@1 + - script: npm install -g ingestron@2 displayName: Install Ingestron - script: ingestron check displayName: ingestron check diff --git a/ci-templates/azure-devops/templates/deploy-environment.yml b/ci-templates/azure-devops/templates/deploy-environment.yml new file mode 100644 index 0000000..9cb0034 --- /dev/null +++ b/ci-templates/azure-devops/templates/deploy-environment.yml @@ -0,0 +1,146 @@ +# Deploys the files the Build stage published to one environment. Its +# deployment job targets the Azure DevOps environment of the same name, so that +# environment's approvals and checks gate it. +# +# Variables to set in the variable group ingestron-: +# - AZURE_SUBSCRIPTION_ID, AZURE_LOCATION: the subscription and region of the +# environment's resource group. +# - ADF_RESOURCE_GROUP, ADF_FACTORY_NAME: the environment's data factory. +# - SYNAPSE_SERVER: -ondemand.sql.azuresynapse.net. +# - DUCKDB_ACCOUNT, DUCKDB_FILE_SYSTEM, DUCKDB_PATH: where the DuckDB job reads +# run.sql, as the job's sqlUrl names it. +# Every step signs in as the environment's service connection. +parameters: + - name: environment + type: string + - name: serviceConnection + type: string + - name: dependsOn + type: string + +stages: + - stage: Deploy_${{ parameters.environment }} + displayName: Deploy ${{ parameters.environment }} + dependsOn: ${{ parameters.dependsOn }} + variables: + - group: ingestron-${{ parameters.environment }} + - name: files + value: $(Pipeline.Workspace)/deploy + jobs: + - deployment: deploy + displayName: Deploy ${{ parameters.environment }} + pool: + vmImage: ubuntu-latest + environment: ${{ parameters.environment }} + strategy: + runOnce: + deploy: + steps: + - download: current + artifact: deploy + + # Data Factory. One factory per project: the post-deployment + # script deletes every resource in the factory that the + # template does not hold. + - task: AzurePowerShell@5 + displayName: "Data Factory: stop the triggers" + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + azurePowerShellVersion: LatestVersion + pwsh: true + ScriptType: FilePath + ScriptPath: $(files)/arm/PrePostDeploymentScript.ps1 + ScriptArguments: >- + -ArmTemplate $(files)/arm/ARMTemplateForFactory.json + -ArmTemplateParameters $(files)/build/adf/parameters/${{ parameters.environment }}.json + -ResourceGroupName $(ADF_RESOURCE_GROUP) -DataFactoryName $(ADF_FACTORY_NAME) + -PreDeployment $true -DeleteDeployment $false + - task: AzureResourceManagerTemplateDeployment@3 + displayName: "Data Factory: deploy the template" + inputs: + deploymentScope: Resource Group + azureResourceManagerConnection: ${{ parameters.serviceConnection }} + subscriptionId: $(AZURE_SUBSCRIPTION_ID) + action: Create Or Update Resource Group + resourceGroupName: $(ADF_RESOURCE_GROUP) + location: $(AZURE_LOCATION) + templateLocation: Linked artifact + csmFile: $(files)/arm/ARMTemplateForFactory.json + csmParametersFile: $(files)/build/adf/parameters/${{ parameters.environment }}.json + # Add - for each ARM parameter that + # `ingestron request` lists under "Values to provide". + overrideParameters: -factoryName $(ADF_FACTORY_NAME) + deploymentMode: Incremental + - task: AzurePowerShell@5 + displayName: "Data Factory: clean up and start the triggers" + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + azurePowerShellVersion: LatestVersion + pwsh: true + ScriptType: FilePath + ScriptPath: $(files)/arm/PrePostDeploymentScript.ps1 + ScriptArguments: >- + -ArmTemplate $(files)/arm/ARMTemplateForFactory.json + -ArmTemplateParameters $(files)/build/adf/parameters/${{ parameters.environment }}.json + -ResourceGroupName $(ADF_RESOURCE_GROUP) -DataFactoryName $(ADF_FACTORY_NAME) + -PreDeployment $false -DeleteDeployment $true + + # Databricks: the bundle's target for this environment. A + # deploy that would delete a resource stops and asks; it is + # not approved here. + - bash: | + curl -fsSL -o "$(Agent.TempDirectory)/databricks.tar.gz" https://github.com/databricks/cli/releases/download/v1.23.0/databricks_cli_1.23.0_linux_amd64.tar.gz + echo "a5ec088f7f6cc5974327a42df99261e22fbc6f7ca9cddb945333c238dd9e2289 $(Agent.TempDirectory)/databricks.tar.gz" | sha256sum -c - + mkdir -p "$(Agent.TempDirectory)/databricks" && tar -xzf "$(Agent.TempDirectory)/databricks.tar.gz" -C "$(Agent.TempDirectory)/databricks" databricks + echo "##vso[task.prependpath]$(Agent.TempDirectory)/databricks" + displayName: "Databricks: install the CLI" + - task: AzureCLI@2 + displayName: "Databricks: deploy the bundle" + env: + DATABRICKS_AUTH_TYPE: azure-cli + # One line for each BUNDLE_VAR_ name that `ingestron request` lists: + # BUNDLE_VAR_landing_account: $(LANDING_ACCOUNT) + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + workingDirectory: $(files)/build/databricks + inlineScript: | + databricks bundle validate -t ${{ parameters.environment }} + databricks bundle deploy -t ${{ parameters.environment }} + + # Synapse serverless: sqlcmd (Go), checked by its SHA-256, + # signed in as the service connection. + - bash: | + curl -fsSL -o "$(Agent.TempDirectory)/sqlcmd.tar.bz2" https://github.com/microsoft/go-sqlcmd/releases/download/v1.10.0/sqlcmd-linux-amd64.tar.bz2 + echo "92516d98c63d99b0994de5b61350c91f6915f9b76f139a59039fbcb225c2e987 $(Agent.TempDirectory)/sqlcmd.tar.bz2" | sha256sum -c - + mkdir -p "$(Agent.TempDirectory)/sqlcmd" && tar -xjf "$(Agent.TempDirectory)/sqlcmd.tar.bz2" -C "$(Agent.TempDirectory)/sqlcmd" sqlcmd + echo "##vso[task.prependpath]$(Agent.TempDirectory)/sqlcmd" + displayName: "Synapse: install sqlcmd" + - task: AzureCLI@2 + displayName: "Synapse: create or alter the views" + # One line for each sqlcmd variable that `ingestron request` lists: + # env: + # SYNAPSE_SOURCE: $(SYNAPSE_SOURCE) + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + workingDirectory: $(files)/build/synapse + inlineScript: >- + sqlcmd -S "$(SYNAPSE_SERVER)" --authentication-method ActiveDirectoryDefault + -i environments/${{ parameters.environment }}.sql + + # DuckDB in Azure: copy the environment's SQL to where the job + # reads it. + - task: AzureCLI@2 + displayName: "DuckDB: upload the SQL" + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + workingDirectory: $(files) + inlineScript: >- + az storage fs directory upload --account-name "$(DUCKDB_ACCOUNT)" + --file-system "$(DUCKDB_FILE_SYSTEM)" --source "build/duckdb/${{ parameters.environment }}/*" + --destination-path "$(DUCKDB_PATH)" --recursive --auth-mode login diff --git a/ci-templates/github-actions/deploy-environment.yml b/ci-templates/github-actions/deploy-environment.yml new file mode 100644 index 0000000..02ebc6d --- /dev/null +++ b/ci-templates/github-actions/deploy-environment.yml @@ -0,0 +1,121 @@ +# Deploys the files deploy.yml built to one environment. It runs in the GitHub +# environment of the same name, so that environment's reviewers and branch rule +# gate it, and its variables and secrets are the ones used. +# +# Variables to set in each GitHub environment: +# - AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID: the identity this +# environment deploys as, with a federated credential for +# repo:/:environment:. +# - ADF_RESOURCE_GROUP, ADF_FACTORY_NAME: the environment's data factory. +# - DATABRICKS_CLIENT_ID: the service principal the bundle deploys as, with a +# federation policy for the same subject. +# - SYNAPSE_SERVER: -ondemand.sql.azuresynapse.net. +# - DUCKDB_ACCOUNT, DUCKDB_FILE_SYSTEM, DUCKDB_PATH: where the DuckDB job reads +# run.sql, as the job's sqlUrl names it. +name: deploy environment +on: + workflow_call: + inputs: + environment: + description: The environment to deploy, as project.yaml names it. + required: true + type: string +permissions: + contents: read + id-token: write +jobs: + deploy: + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + concurrency: + group: deploy-${{ inputs.environment }} + cancel-in-progress: false + env: + ENVIRONMENT: ${{ inputs.environment }} + steps: + - uses: actions/download-artifact@v4 + with: + name: deploy + - uses: azure/login@v2 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + enable-AzPSSession: true + + # Data Factory. One factory per project: the post-deployment script + # deletes every resource in the factory that the template does not hold. + - name: "Data Factory: stop the triggers" + uses: azure/powershell@v2 + env: + ADF_RESOURCE_GROUP: ${{ vars.ADF_RESOURCE_GROUP }} + ADF_FACTORY_NAME: ${{ vars.ADF_FACTORY_NAME }} + with: + azPSVersion: latest + inlineScript: | + ./arm/PrePostDeploymentScript.ps1 -ArmTemplate arm/ARMTemplateForFactory.json ` + -ArmTemplateParameters "build/adf/parameters/$env:ENVIRONMENT.json" ` + -ResourceGroupName $env:ADF_RESOURCE_GROUP -DataFactoryName $env:ADF_FACTORY_NAME ` + -PreDeployment $true -DeleteDeployment $false + - name: "Data Factory: deploy the template" + env: + ADF_RESOURCE_GROUP: ${{ vars.ADF_RESOURCE_GROUP }} + ADF_FACTORY_NAME: ${{ vars.ADF_FACTORY_NAME }} + # Add name=value after factoryName for each ARM parameter that + # `ingestron request` lists under "Values to provide". + run: | + az deployment group create --resource-group "$ADF_RESOURCE_GROUP" --mode Incremental \ + --template-file arm/ARMTemplateForFactory.json \ + --parameters "@build/adf/parameters/$ENVIRONMENT.json" factoryName="$ADF_FACTORY_NAME" + - name: "Data Factory: clean up and start the triggers" + uses: azure/powershell@v2 + env: + ADF_RESOURCE_GROUP: ${{ vars.ADF_RESOURCE_GROUP }} + ADF_FACTORY_NAME: ${{ vars.ADF_FACTORY_NAME }} + with: + azPSVersion: latest + inlineScript: | + ./arm/PrePostDeploymentScript.ps1 -ArmTemplate arm/ARMTemplateForFactory.json ` + -ArmTemplateParameters "build/adf/parameters/$env:ENVIRONMENT.json" ` + -ResourceGroupName $env:ADF_RESOURCE_GROUP -DataFactoryName $env:ADF_FACTORY_NAME ` + -PreDeployment $false -DeleteDeployment $true + + # Databricks: the bundle's target for this environment. A deploy that + # would delete a resource stops and asks; it is not approved here. + - uses: databricks/setup-cli@v1.23.0 + - name: "Databricks: deploy the bundle" + working-directory: build/databricks + env: + DATABRICKS_AUTH_TYPE: github-oidc + DATABRICKS_CLIENT_ID: ${{ vars.DATABRICKS_CLIENT_ID }} + # One line for each BUNDLE_VAR_ name that `ingestron request` lists: + # BUNDLE_VAR_landing_account: ${{ vars.LANDING_ACCOUNT }} + run: | + databricks bundle validate -t "$ENVIRONMENT" + databricks bundle deploy -t "$ENVIRONMENT" + + # Synapse serverless: sqlcmd (Go), checked by its SHA-256, signed in as + # the identity azure/login used. + - name: "Synapse: install sqlcmd" + run: | + curl -fsSL -o "$RUNNER_TEMP/sqlcmd.tar.bz2" https://github.com/microsoft/go-sqlcmd/releases/download/v1.10.0/sqlcmd-linux-amd64.tar.bz2 + echo "92516d98c63d99b0994de5b61350c91f6915f9b76f139a59039fbcb225c2e987 $RUNNER_TEMP/sqlcmd.tar.bz2" | sha256sum -c - + mkdir -p "$RUNNER_TEMP/sqlcmd" && tar -xjf "$RUNNER_TEMP/sqlcmd.tar.bz2" -C "$RUNNER_TEMP/sqlcmd" sqlcmd + echo "$RUNNER_TEMP/sqlcmd" >> "$GITHUB_PATH" + - name: "Synapse: create or alter the views" + working-directory: build/synapse + env: + SYNAPSE_SERVER: ${{ vars.SYNAPSE_SERVER }} + # One line for each sqlcmd variable that `ingestron request` lists: + # SYNAPSE_SOURCE: ${{ vars.SYNAPSE_SOURCE }} + run: sqlcmd -S "$SYNAPSE_SERVER" --authentication-method ActiveDirectoryDefault -i "environments/$ENVIRONMENT.sql" + + # DuckDB in Azure: copy the environment's SQL to where the job reads it. + - name: "DuckDB: upload the SQL" + env: + DUCKDB_ACCOUNT: ${{ vars.DUCKDB_ACCOUNT }} + DUCKDB_FILE_SYSTEM: ${{ vars.DUCKDB_FILE_SYSTEM }} + DUCKDB_PATH: ${{ vars.DUCKDB_PATH }} + run: | + az storage fs directory upload --account-name "$DUCKDB_ACCOUNT" --file-system "$DUCKDB_FILE_SYSTEM" \ + --source "build/duckdb/$ENVIRONMENT/*" --destination-path "$DUCKDB_PATH" --recursive --auth-mode login diff --git a/ci-templates/github-actions/deploy.yml b/ci-templates/github-actions/deploy.yml new file mode 100644 index 0000000..36083d2 --- /dev/null +++ b/ci-templates/github-actions/deploy.yml @@ -0,0 +1,79 @@ +# Deploys the committed build/ folder: checks it, builds the Data Factory +# template once, then deploys the same files to dev, test and prod in turn. +# Copy this file and deploy-environment.yml to .github/workflows/ in your +# definitions repository, and delete the steps of each platform your project +# does not use. +# +# Set up once, as https://docs.ingestron.io/docs/how-to/deploy-through-environments +# explains: +# - GitHub environments named dev, test and prod. Give test and prod required +# reviewers, and let only the main branch deploy to prod. +# - In each environment, the variables deploy-environment.yml reads. +# - The repository variable ADF_DEV_FACTORY_ID: the dev factory's resource ID, +# which names the template's resources. Nothing is deployed with it. +name: deploy +on: + push: + branches: [main] +permissions: + contents: read + id-token: write +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: npm install -g ingestron@2 + - run: ingestron check + + build: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + # Data Factory: Microsoft's build validates the factory, then writes the + # ARM template and the pre- and post-deployment script to arm/. It + # downloads its build tool from adf.azure.com. + - name: "Data Factory: build the template" + env: + ADF_DEV_FACTORY_ID: ${{ vars.ADF_DEV_FACTORY_ID }} + run: | + npm install --prefix "$RUNNER_TEMP/adf" @microsoft/azure-data-factory-utilities@1.0.3 + adf="$RUNNER_TEMP/adf/node_modules/@microsoft/azure-data-factory-utilities/lib/index" + node "$adf" validate build/adf/factory "$ADF_DEV_FACTORY_ID" + node "$adf" export build/adf/factory "$ADF_DEV_FACTORY_ID" arm + # Every environment deploys these files, so prod gets the bytes dev got. + - uses: actions/upload-artifact@v4 + with: + name: deploy + path: | + arm/ + build/ + if-no-files-found: error + + dev: + needs: build + uses: ./.github/workflows/deploy-environment.yml + with: + environment: dev + secrets: inherit + + test: + needs: dev + uses: ./.github/workflows/deploy-environment.yml + with: + environment: test + secrets: inherit + + prod: + needs: test + uses: ./.github/workflows/deploy-environment.yml + with: + environment: prod + secrets: inherit diff --git a/ci-templates/github-actions/ingestron-check.yml b/ci-templates/github-actions/ingestron-check.yml index 74d84f2..ecb4450 100644 --- a/ci-templates/github-actions/ingestron-check.yml +++ b/ci-templates/github-actions/ingestron-check.yml @@ -15,5 +15,5 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22 - - run: npm install -g ingestron@1 + - run: npm install -g ingestron@2 - run: ingestron check diff --git a/scripts/ci-templates.test.mjs b/scripts/ci-templates.test.mjs index dac5392..caf61cc 100644 --- a/scripts/ci-templates.test.mjs +++ b/scripts/ci-templates.test.mjs @@ -1,28 +1,230 @@ -// The CI templates run the published CLI, not this repository's build. +// The CI templates: each parses, runs the published CLI rather than this +// repository's build, promotes one build through dev, test and prod, and names +// every tool it downloads by an exact version, checked where it can be. import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; -import { join } from "node:path"; +import { readdirSync, readFileSync } from "node:fs"; +import { join, relative } from "node:path"; import test from "node:test"; +import { parse } from "yaml"; -const read = (file) => readFileSync(join(import.meta.dirname, "../ci-templates", file), "utf8"); -const templates = ["github-actions/ingestron-check.yml", "azure-devops/ingestron-check.yml"]; +const folder = join(import.meta.dirname, "../ci-templates"); +const read = (file) => readFileSync(join(folder, file), "utf8"); +const yaml = (file) => parse(read(file)); +const checks = ["github-actions/ingestron-check.yml", "azure-devops/ingestron-check.yml"]; +const deploys = [ + "github-actions/deploy.yml", + "github-actions/deploy-environment.yml", + "azure-devops/azure-pipelines.yml", + "azure-devops/templates/deploy-environment.yml", +]; -test("each template installs ingestron from npm, then runs ingestron check", () => { - for (const file of templates) { +/** Every step of a GitHub workflow or Azure DevOps pipeline, wherever it is nested. */ +function steps(node) { + if (Array.isArray(node)) return node.flatMap(steps); + if (!node || typeof node !== "object") return []; + return Object.entries(node).flatMap(([key, value]) => + key === "steps" && Array.isArray(value) ? [...value, ...steps(value)] : steps(value), + ); +} + +/** The text a step runs: a GitHub `run`, an Azure DevOps `script` or `bash`, or an inline script. */ +const scripts = (file) => + steps(yaml(file)) + .map( + (step) => + step.run ?? + step.script ?? + step.bash ?? + step.with?.inlineScript ?? + step.inputs?.inlineScript ?? + step.inputs?.ScriptArguments ?? + "", + ) + .join("\n"); + +test("every YAML template parses, and the folder holds no other kind of file", () => { + const files = (path) => + readdirSync(path, { withFileTypes: true }).flatMap((entry) => + entry.isDirectory() + ? files(join(path, entry.name)) + : [relative(folder, join(path, entry.name))], + ); + const all = files(folder).sort(); + assert.deepEqual(all, [...checks, ...deploys, "azure/duckdb-job.bicep"].sort()); + for (const file of [...checks, ...deploys]) assert.equal(typeof yaml(file), "object", file); +}); + +test("each check template installs ingestron 2 from npm, then runs ingestron check", () => { + for (const file of [...checks, "github-actions/deploy.yml", "azure-devops/azure-pipelines.yml"]) { const text = read(file); assert.match( text, - /(run|script): npm install -g ingestron@1\n[\s\S]*(run|script): ingestron check\n/, + /(run|script): npm install -g ingestron@2\n[\s\S]*(run|script): ingestron check\n/, + file, + ); + assert.doesNotMatch(text, /npx|pnpm|node packages\//, file); + } +}); + +test("each check template installs Node 22", () => { + assert.match(read(checks[0]), /actions\/setup-node@v4[\s\S]*node-version: 22/); + assert.match(read(checks[1]), /NodeTool@0[\s\S]*versionSpec: 22\.x/); +}); + +test("GitHub Actions: one build, deployed to dev, then test, then prod, each in its environment", () => { + const deploy = yaml("github-actions/deploy.yml"); + assert.deepEqual(deploy.on, { push: { branches: ["main"] } }); + assert.deepEqual(deploy.permissions, { contents: "read", "id-token": "write" }); + assert.equal(deploy.jobs.build.needs, "check"); + const order = [ + ["dev", "build"], + ["test", "dev"], + ["prod", "test"], + ]; + for (const [environment, after] of order) { + const job = deploy.jobs[environment]; + assert.equal(job.needs, after, environment); + assert.equal(job.uses, "./.github/workflows/deploy-environment.yml", environment); + assert.deepEqual(job.with, { environment }, environment); + } + const upload = steps(deploy).find((step) => step.uses?.startsWith("actions/upload-artifact@")); + assert.match(upload.with.path, /arm\/\nbuild\//); + + const environment = yaml("github-actions/deploy-environment.yml"); + assert.ok(environment.on.workflow_call.inputs.environment.required); + const job = environment.jobs.deploy; + assert.equal(job.environment, "${{ inputs.environment }}"); + assert.equal(job.concurrency.group, "deploy-${{ inputs.environment }}"); + assert.equal(job.concurrency["cancel-in-progress"], false); + const uses = steps(environment).map((step) => step.uses); + assert.ok(uses.includes("actions/download-artifact@v4")); + assert.ok(uses.includes("azure/login@v2")); + assert.ok(!steps(environment).some((step) => step.uses?.startsWith("actions/checkout"))); +}); + +test("Azure DevOps: one Build stage, deployed to dev, then test, then prod, each in its environment", () => { + const pipeline = yaml("azure-devops/azure-pipelines.yml"); + assert.deepEqual(pipeline.trigger, { branches: { include: ["main"] } }); + assert.equal(pipeline.pr, "none"); + const [build, ...stages] = pipeline.stages; + assert.equal(build.stage, "Build"); + assert.ok(steps(build).some((step) => step.task === "PublishPipelineArtifact@1")); + assert.deepEqual( + stages.map((stage) => [stage.parameters.environment, stage.parameters.dependsOn]), + [ + ["dev", "Build"], + ["test", "Deploy_dev"], + ["prod", "Deploy_test"], + ], + ); + for (const stage of stages) assert.equal(stage.template, "templates/deploy-environment.yml"); + + const template = yaml("azure-devops/templates/deploy-environment.yml"); + const [stage] = template.stages; + assert.equal(stage.stage, "Deploy_${{ parameters.environment }}"); + const [deployment] = stage.jobs; + assert.equal(deployment.environment, "${{ parameters.environment }}"); + const tasks = steps(template).map((step) => step.task); + for (const task of [ + "AzurePowerShell@5", + "AzureResourceManagerTemplateDeployment@3", + "AzureCLI@2", + ]) + assert.ok(tasks.includes(task), task); + const arm = steps(template).find( + (step) => step.task === "AzureResourceManagerTemplateDeployment@3", + ); + assert.equal(arm.inputs.deploymentMode, "Incremental"); + for (const script of steps(template).filter((step) => step.task === "AzurePowerShell@5")) + assert.equal(script.inputs.pwsh, true); +}); + +test("Data Factory: built once with Microsoft's package 1.0.3, deployed incrementally with each environment's parameters", () => { + for (const file of ["github-actions/deploy.yml", "azure-devops/azure-pipelines.yml"]) { + const text = scripts(file); + assert.match( + text, + /npm install --prefix \S+ @microsoft\/azure-data-factory-utilities@1\.0\.3\n/, + ); + assert.match(text, /node "\$adf" validate build\/adf\/factory /, file); + assert.match(text, /node "\$adf" export build\/adf\/factory \S+ arm\n/, file); + } + const github = scripts("github-actions/deploy-environment.yml"); + assert.match(github, /--mode Incremental/); + assert.match(github, /--parameters "@build\/adf\/parameters\/\$ENVIRONMENT\.json" factoryName=/); + for (const file of [ + "github-actions/deploy-environment.yml", + "azure-devops/templates/deploy-environment.yml", + ]) { + const text = scripts(file); + // The pre- and post-deployment script read each environment's parameters, or prod's triggers stay stopped. + assert.equal( + text.match(/-ArmTemplateParameters \S*build\/adf\/parameters\//g)?.length, + 2, + file, + ); + assert.match(text, /-PreDeployment \$true -DeleteDeployment \$false/, file); + assert.match(text, /-PreDeployment \$false -DeleteDeployment \$true/, file); + } +}); + +test("Databricks: CLI v1.23.0, checked by its SHA-256 where it is downloaded, deploying each environment's target", () => { + const github = yaml("github-actions/deploy-environment.yml"); + assert.ok(steps(github).some((step) => step.uses === "databricks/setup-cli@v1.23.0")); + const deploy = steps(github).find((step) => step.name === "Databricks: deploy the bundle"); + assert.equal(deploy["working-directory"], "build/databricks"); + assert.equal(deploy.env.DATABRICKS_AUTH_TYPE, "github-oidc"); + assert.match( + deploy.run, + /databricks bundle validate -t "\$ENVIRONMENT"\ndatabricks bundle deploy -t "\$ENVIRONMENT"/, + ); + + const devops = scripts("azure-devops/templates/deploy-environment.yml"); + assert.match( + devops, + /releases\/download\/v1\.23\.0\/databricks_cli_1\.23\.0_linux_amd64\.tar\.gz\n.*echo "a5ec088f7f6cc5974327a42df99261e22fbc6f7ca9cddb945333c238dd9e2289 .*" \| sha256sum -c -/, + ); + assert.match(devops, /databricks bundle deploy -t \$\{\{ parameters\.environment \}\}/); + const cli = steps(yaml("azure-devops/templates/deploy-environment.yml")).find( + (step) => step.displayName === "Databricks: deploy the bundle", + ); + assert.equal(cli.env.DATABRICKS_AUTH_TYPE, "azure-cli"); +}); + +test("Synapse: sqlcmd (Go) v1.10.0, checked by its SHA-256, run from build/synapse", () => { + for (const file of [ + "github-actions/deploy-environment.yml", + "azure-devops/templates/deploy-environment.yml", + ]) { + const text = scripts(file); + assert.match( + text, + /go-sqlcmd\/releases\/download\/v1\.10\.0\/sqlcmd-linux-amd64\.tar\.bz2\necho "92516d98c63d99b0994de5b61350c91f6915f9b76f139a59039fbcb225c2e987 .*" \| sha256sum -c -/, + file, + ); + assert.match( + text, + /sqlcmd -S "\S+" --authentication-method ActiveDirectoryDefault\s+-i "?environments\//, file, ); - assert.doesNotMatch(text, /npx/, file); - assert.doesNotMatch(text, /pnpm|node packages\//, file); + assert.doesNotMatch(text, / -[xX] /, file); } + const step = steps(yaml("github-actions/deploy-environment.yml")).find( + (found) => found.name === "Synapse: create or alter the views", + ); + assert.equal(step["working-directory"], "build/synapse"); + const task = steps(yaml("azure-devops/templates/deploy-environment.yml")).find( + (found) => found.displayName === "Synapse: create or alter the views", + ); + assert.equal(task.inputs.workingDirectory, "$(files)/build/synapse"); }); -test("each template installs Node 22", () => { - assert.match(read(templates[0]), /actions\/setup-node@v4[\s\S]*node-version: 22/); - assert.match(read(templates[1]), /NodeTool@0[\s\S]*versionSpec: 22\.x/); +test("no deploy step installs or runs Ingestron", () => { + for (const file of [ + "github-actions/deploy-environment.yml", + "azure-devops/templates/deploy-environment.yml", + ]) + assert.doesNotMatch(read(file), /ingestron (check|apply|plan)|npm install -g ingestron/, file); }); test("the DuckDB job template pulls the public runner image with no registry", () => { From fe4ae58852c2940fe253199f18b5039589d09963 Mon Sep 17 00:00:00 2001 From: The Architect Date: Thu, 5 Nov 2026 17:20:40 +1300 Subject: [PATCH 02/39] Examples: every pack at version 2, and names the same in every environment Each standard names databricks@2, adf@2, synapse@2 or duckdb@2. One bundle or factory now serves every environment, so connection-first-adf names its ADF pipeline and trigger without the environment, and neither it nor standards-and-conventions names a pipeline per environment. Signed-off-by: The Architect --- examples/change-data/standard.yaml | 2 +- examples/connection-first-adf/flows/postgresql.yaml | 4 ++-- examples/connection-first-adf/standard.yaml | 5 ++--- examples/connection-first-files/standard.yaml | 2 +- examples/data-products/standard.yaml | 2 +- examples/duckdb-local/standard.yaml | 2 +- examples/environments/standard.yaml | 2 +- examples/existing-tables/standard.yaml | 2 +- examples/landing-only/standard.yaml | 2 +- examples/layers-and-transforms/standard.yaml | 2 +- examples/northwind/standard.yaml | 2 +- examples/platform-setup/standard.yaml | 2 +- examples/quality/standard.yaml | 2 +- examples/standards-and-conventions/standard.yaml | 3 +-- examples/synapse-views/standard.yaml | 2 +- 15 files changed, 17 insertions(+), 19 deletions(-) diff --git a/examples/change-data/standard.yaml b/examples/change-data/standard.yaml index f5fdbc9..9217762 100644 --- a/examples/change-data/standard.yaml +++ b/examples/change-data/standard.yaml @@ -1,4 +1,4 @@ apiVersion: ingestron.io/v1 kind: Standard id: crm -packs: [databricks@1] +packs: [databricks@2] diff --git a/examples/connection-first-adf/flows/postgresql.yaml b/examples/connection-first-adf/flows/postgresql.yaml index 70ca651..714dae6 100644 --- a/examples/connection-first-adf/flows/postgresql.yaml +++ b/examples/connection-first-adf/flows/postgresql.yaml @@ -6,8 +6,8 @@ ingest: uses: adf-table with: resource_names: - pipeline: pl_${values.resource_prefix}_orders_${environment.name} - trigger: tr_${values.resource_prefix}_orders_${environment.name} + pipeline: pl_${values.resource_prefix}_orders + trigger: tr_${values.resource_prefix}_orders copy_activity: copy_orders tables: postgresql_records: diff --git a/examples/connection-first-adf/standard.yaml b/examples/connection-first-adf/standard.yaml index 5443328..6aecf3e 100644 --- a/examples/connection-first-adf/standard.yaml +++ b/examples/connection-first-adf/standard.yaml @@ -2,8 +2,8 @@ apiVersion: ingestron.io/v1 kind: Standard id: native packs: - - databricks@1 - - adf@1 + - databricks@2 + - adf@2 ingest: settings: adf-table: @@ -16,7 +16,6 @@ ingest: naming: catalog: sales_{{ environment }} schema: "{{ layer }}" - pipeline: "{{ project }}_{{ environment }}" store: allowed: - streaming-table diff --git a/examples/connection-first-files/standard.yaml b/examples/connection-first-files/standard.yaml index 2f3df24..42fe711 100644 --- a/examples/connection-first-files/standard.yaml +++ b/examples/connection-first-files/standard.yaml @@ -2,4 +2,4 @@ apiVersion: ingestron.io/v1 kind: Standard id: defaults packs: - - databricks@1 + - databricks@2 diff --git a/examples/data-products/standard.yaml b/examples/data-products/standard.yaml index bec0a9c..9845c18 100644 --- a/examples/data-products/standard.yaml +++ b/examples/data-products/standard.yaml @@ -1,7 +1,7 @@ apiVersion: ingestron.io/v1 kind: Standard id: shop -packs: [duckdb@1, adf@1] # DuckDB first: the first pack names and types the tables; ADF only copies +packs: [duckdb@2, adf@2] # DuckDB first: the first pack names and types the tables; ADF only copies ingest: default: adf-copy # ADF copies from the source; DuckDB builds the tables settings: diff --git a/examples/duckdb-local/standard.yaml b/examples/duckdb-local/standard.yaml index 979676c..736186d 100644 --- a/examples/duckdb-local/standard.yaml +++ b/examples/duckdb-local/standard.yaml @@ -1,7 +1,7 @@ apiVersion: ingestron.io/v1 kind: Standard id: sales -packs: [duckdb@1, adf@1] # DuckDB first: the first pack names and types the tables; ADF only copies +packs: [duckdb@2, adf@2] # DuckDB first: the first pack names and types the tables; ADF only copies ingest: settings: adf-copy: diff --git a/examples/environments/standard.yaml b/examples/environments/standard.yaml index e8a9fc4..2a80468 100644 --- a/examples/environments/standard.yaml +++ b/examples/environments/standard.yaml @@ -1,6 +1,6 @@ apiVersion: ingestron.io/v1 kind: Standard id: finance -packs: [databricks@1] +packs: [databricks@2] naming: catalog: finance_{{ environment }} diff --git a/examples/existing-tables/standard.yaml b/examples/existing-tables/standard.yaml index 93dea68..358bea4 100644 --- a/examples/existing-tables/standard.yaml +++ b/examples/existing-tables/standard.yaml @@ -1,4 +1,4 @@ apiVersion: ingestron.io/v1 kind: Standard id: sales -packs: [databricks@1] +packs: [databricks@2] diff --git a/examples/landing-only/standard.yaml b/examples/landing-only/standard.yaml index be4d35b..7921d71 100644 --- a/examples/landing-only/standard.yaml +++ b/examples/landing-only/standard.yaml @@ -1,7 +1,7 @@ apiVersion: ingestron.io/v1 kind: Standard id: sales -packs: [adf@1] +packs: [adf@2] store: default: none # flows end at the files ADF lands ingest: diff --git a/examples/layers-and-transforms/standard.yaml b/examples/layers-and-transforms/standard.yaml index 5ce0d4a..969fa7e 100644 --- a/examples/layers-and-transforms/standard.yaml +++ b/examples/layers-and-transforms/standard.yaml @@ -1,7 +1,7 @@ apiVersion: ingestron.io/v1 kind: Standard id: sales -packs: [databricks@1] +packs: [databricks@2] store: when: # Gold tables are small and read whole: no clustering, longer retention. diff --git a/examples/northwind/standard.yaml b/examples/northwind/standard.yaml index 1f7ad20..5817e95 100644 --- a/examples/northwind/standard.yaml +++ b/examples/northwind/standard.yaml @@ -1,7 +1,7 @@ apiVersion: ingestron.io/v1 kind: Standard id: northwind -packs: [databricks@1, adf@1] # Databricks names, types and keeps the tables; ADF copies +packs: [databricks@2, adf@2] # Databricks names, types and keeps the tables; ADF copies ingest: default: adf-copy settings: diff --git a/examples/platform-setup/standard.yaml b/examples/platform-setup/standard.yaml index 20272dd..0fdb7fb 100644 --- a/examples/platform-setup/standard.yaml +++ b/examples/platform-setup/standard.yaml @@ -1,6 +1,6 @@ apiVersion: ingestron.io/v1 kind: Standard id: hr -packs: [databricks@1] +packs: [databricks@2] naming: catalog: hr_{{ environment }} diff --git a/examples/quality/standard.yaml b/examples/quality/standard.yaml index 1641c69..90dfe0e 100644 --- a/examples/quality/standard.yaml +++ b/examples/quality/standard.yaml @@ -1,7 +1,7 @@ apiVersion: ingestron.io/v1 kind: Standard id: iot -packs: [databricks@1] +packs: [databricks@2] quality: required: drop # a row missing a required value is dropped and counted default: warn # own checks only count failures unless they say otherwise diff --git a/examples/standards-and-conventions/standard.yaml b/examples/standards-and-conventions/standard.yaml index 5332984..de7e160 100644 --- a/examples/standards-and-conventions/standard.yaml +++ b/examples/standards-and-conventions/standard.yaml @@ -2,12 +2,11 @@ apiVersion: ingestron.io/v1 kind: Standard id: acme description: Acme's data platform conventions. -packs: [databricks@1] +packs: [databricks@2] naming: catalog: acme schema: "{{ environment }}_{{ layer }}" table: "{% if source %}{{ source }}__{% endif %}{{ table }}" - pipeline: "{{ project }}_{{ environment }}" layers: [raw, curated, published] types: number: DECIMAL(38,6) diff --git a/examples/synapse-views/standard.yaml b/examples/synapse-views/standard.yaml index 4836e8e..a880779 100644 --- a/examples/synapse-views/standard.yaml +++ b/examples/synapse-views/standard.yaml @@ -1,7 +1,7 @@ apiVersion: ingestron.io/v1 kind: Standard id: sales -packs: [synapse@1, adf@1] # Synapse first: the first pack names and types the views; ADF only copies +packs: [synapse@2, adf@2] # Synapse first: the first pack names and types the views; ADF only copies store: default: synapse-view # each table is a view over the files ADF lands ingest: From 1cd663d37b4c278d192931bce5bffd493123bfbf Mon Sep 17 00:00:00 2001 From: The Architect Date: Thu, 5 Nov 2026 17:21:51 +1300 Subject: [PATCH 03/39] Example READMEs: the 2.0 folders, build// Each README names the folder ingestron 2.0 writes: one bundle in build/databricks/, one factory in build/adf/ with a parameter file per environment, build/synapse/ with a script per environment, and build/duckdb// and build/products//. Signed-off-by: The Architect --- examples/change-data/README.md | 4 ++-- examples/connection-first-adf/README.md | 4 ++-- examples/connection-first-files/README.md | 2 +- examples/data-products/README.md | 4 ++-- examples/duckdb-local/README.md | 21 +++++++++--------- examples/environments/README.md | 23 +++++++++++--------- examples/existing-tables/README.md | 7 +++--- examples/landing-only/README.md | 4 ++-- examples/layers-and-transforms/README.md | 6 ++--- examples/northwind/README.md | 10 ++++----- examples/platform-setup/README.md | 15 +++++++------ examples/quality/README.md | 2 +- examples/standards-and-conventions/README.md | 3 +-- examples/synapse-views/README.md | 17 +++++++++------ 14 files changed, 65 insertions(+), 57 deletions(-) diff --git a/examples/change-data/README.md b/examples/change-data/README.md index 0a9e476..df37345 100644 --- a/examples/change-data/README.md +++ b/examples/change-data/README.md @@ -2,10 +2,10 @@ The source files hold one row per change: a key, a time and an operation. -- `accounts` uses `mode: upsert`: [the table](build/dev/src/crm/bronze/accounts.sql) +- `accounts` uses `mode: upsert`: [the table](build/databricks/src/crm/bronze/accounts.sql) keeps only the current row per key (`STORED AS SCD TYPE 1`); rows whose `operation` is `DELETE` remove the key. -- `contacts` uses `mode: history`: [the table](build/dev/src/crm/bronze/contacts.sql) +- `contacts` uses `mode: history`: [the table](build/databricks/src/crm/bronze/contacts.sql) keeps every version (`STORED AS SCD TYPE 2`), keyed by two columns in `primaryKeyPosition` order. diff --git a/examples/connection-first-adf/README.md b/examples/connection-first-adf/README.md index 9be6d24..7069c9f 100644 --- a/examples/connection-first-adf/README.md +++ b/examples/connection-first-adf/README.md @@ -2,8 +2,8 @@ What `ingestron init sales --connector postgresql --connectivity private --via adf` creates: one PostgreSQL connection that ADF copies through a self-hosted -integration runtime, one standard listing both packs, `packs: [databricks@1, -adf@1]`, and the ADF resource names set in the platform. +integration runtime, one standard listing both packs, `packs: [databricks@2, +adf@2]`, and the ADF resource names set in the platform. | File | What it shows | | ----------------------------- | ------------------------------------------------------------------------ | diff --git a/examples/connection-first-files/README.md b/examples/connection-first-files/README.md index 50bdc3a..84ad7c6 100644 --- a/examples/connection-first-files/README.md +++ b/examples/connection-first-files/README.md @@ -7,7 +7,7 @@ only the Databricks pack, so every other choice is a pack default. | File | What it shows | | ----------------------- | ------------------------------------------------------- | | `project.yaml` | The landing folder as a value | -| `standard.yaml` | `packs: [databricks@1]` | +| `standard.yaml` | `packs: [databricks@2]` | | `connections/shop.yaml` | The files source and its folder | | `flows/*.yaml` | The tables read from the folder, each with its contract | | `contracts/*.yaml` | ODCS contracts: the columns and their types | diff --git a/examples/data-products/README.md b/examples/data-products/README.md index 0d0a27b..25346a1 100644 --- a/examples/data-products/README.md +++ b/examples/data-products/README.md @@ -1,7 +1,7 @@ # Define data products The web shop's orders and customers are copied from Azure SQL by ADF, and -DuckDB builds the tables (`packs: [duckdb@1, adf@1]`), as in the DuckDB +DuckDB builds the tables (`packs: [duckdb@2, adf@2]`), as in the DuckDB example. On top, two **data products**, each an Open Data Product Standard file in `products/`: @@ -24,7 +24,7 @@ contracts/customers.yaml:13:17 error COLUMN_UNCLASSIFIED Column "email" of the contract "shop.customers" has no classification; "customers" publishes it, and the standard requires one. ``` -`ingestron apply` writes, beside the pipelines, `build//products/`: +`ingestron apply` writes, beside the pipelines, `build/products//`: a `README.md` listing the products, and a passport per product, `customers.md` and `sales.md`. A passport says, in plain words, who owns the product and what it is for, which table it is in this environment and where, its columns and diff --git a/examples/duckdb-local/README.md b/examples/duckdb-local/README.md index 3a13b28..a3c0c4d 100644 --- a/examples/duckdb-local/README.md +++ b/examples/duckdb-local/README.md @@ -1,7 +1,7 @@ # Run locally with DuckDB ADF copies the ERP's invoices into storage, and DuckDB loads them into a table, -runs a transform and publishes both as Parquet: `packs: [duckdb@1, adf@1]` in +runs a transform and publishes both as Parquet: `packs: [duckdb@2, adf@2]` in `standard.yaml`. Each load copies the whole table into one file, `erp/erp.dbo.Invoices/erp.dbo.Invoices.parquet`, replacing the last one (`layout: snapshot`), so the table is always the latest load. @@ -10,27 +10,28 @@ The `local` environment names `runs-on: duckdb`, so it runs the same tables on a laptop: ADF's copy becomes `duckdb-landed`, which reads a load's files from `./samples//`, and tables are published to the `./published` folder. -`ingestron apply` writes, for each environment under `build//`: +`ingestron apply` writes: -- `adf/` (`dev` only): the copy pipeline, a parent pipeline that runs it and - then the DuckDB job, the parent's trigger, linked services and datasets; -- `duckdb/run.sql`: every table in order, the same SQL in both environments - apart from where the files are. +- `build/adf/`, for `dev` only: the factory's copy pipeline, a parent pipeline + that runs it and then the DuckDB job, the parent's trigger, linked services + and datasets, and `parameters/dev.json`; +- `build/duckdb//run.sql`: every table in order, the same SQL in + both environments apart from where the files are. In `dev`, ADF's `after_copy` starts the Container Apps job `sales-dev-duckdb` after each schedule's copies: the parent pipeline, `pl_sales_every_24_hour`, runs the copies, starts the job and waits for it. The job runs `ghcr.io/ingestron/duckdb-runner:1`, a public image, which reads the -`run.sql` your deploy step copies from `build/dev/duckdb/` to storage, rebuilds -every table from the landed files and publishes them. `ingestron request --env -dev` lists the job and the access it needs. `start` here is an example: when +`run.sql` your deploy step copies from `build/duckdb/dev/` to storage, rebuilds +every table from the landed files and publishes them. `ingestron request --env dev` +lists the job and the access it needs. `start` here is an example: when you turn `after_copy` on, set it to the next window. To run locally, download a load's file from `erp/erp.dbo.Invoices/` in the `landing` file system to `samples/erp.dbo.Invoices/`. Then, from this folder: ```sh -duckdb -bail ingestron.duckdb -c ".read build/local/duckdb/run.sql" +duckdb -bail ingestron.duckdb -c ".read build/duckdb/local/run.sql" ``` Failures inside ADF (connectors, credentials, networking) only happen in ADF. diff --git a/examples/environments/README.md b/examples/environments/README.md index 59b3400..6ddd257 100644 --- a/examples/environments/README.md +++ b/examples/environments/README.md @@ -3,16 +3,19 @@ One set of definitions, three environments. `project.yaml` gives values per environment; the platform and connection read them. -Compare the three folders: +`ingestron apply` writes one bundle, `build/databricks/`, with a target for +each environment. Its [`databricks.yml`](build/databricks/databricks.yml) +shows what differs: -| | [dev](build/dev/) | [test](build/test/) | [prod](build/prod/) | -| ------------------------------------- | ----------------- | ------------------- | ------------------- | -| Catalog (`finance_{{ environment }}`) | `finance_dev` | `finance_test` | `finance_prod` | -| Workspace | shared | shared | its own | -| Landing path | `dev_landing` | `test_landing` | `prod_landing` | -| Bundle mode | development | development | production | -| Daily refresh job | paused | paused | running | +| | `dev` | `test` | `prod` | +| ------------------------------------- | ------------- | -------------- | -------------- | +| Catalog (`finance_{{ environment }}`) | `finance_dev` | `finance_test` | `finance_prod` | +| Workspace | shared | shared | its own | +| Landing path | `dev_landing` | `test_landing` | `prod_landing` | +| Bundle mode | development | production | production | +| Daily refresh job | paused | paused | running | -Mode and pause follow the environment's name by default (`prod` or -`production` is production); a platform can set `mode` and +Mode and pause follow the environment's name by default: `dev` is in +development mode, every other environment in production mode, and schedules +run only in `prod` or `production`. A platform can set `mode` and `schedule.paused` itself, from values if they differ per environment. diff --git a/examples/existing-tables/README.md b/examples/existing-tables/README.md index 7ec0c34..5b71dab 100644 --- a/examples/existing-tables/README.md +++ b/examples/existing-tables/README.md @@ -4,9 +4,10 @@ - `store_sales` joins it with the stores table another team owns. Its code names that table as `${values.stores}`, and `project.yaml` gives the name for each environment: `dev_reference.core.stores` in dev and - `prod_reference.core.stores` in prod. Compare - [dev](build/dev/src/sales/silver/store_sales.sql) with - [prod](build/prod/src/sales/silver/store_sales.sql). + `prod_reference.core.stores` in prod. In + [the generated table](build/databricks/src/sales/silver/store_sales.sql) + the part that differs is `${ingestron.store_sales_value}`, which each target + of `databricks.yml` sets. Ingestron never creates or changes the stores table, and does not list it in `inputs`: `inputs` are tables the project builds. diff --git a/examples/landing-only/README.md b/examples/landing-only/README.md index 0308687..f74c4c8 100644 --- a/examples/landing-only/README.md +++ b/examples/landing-only/README.md @@ -10,8 +10,8 @@ ADF copies the ERP's tables into storage, and the flows end there: | `customers` | `/erp.dbo.Customers/erp.dbo.Customers.parquet` | | `invoices` | `/erp.dbo.Invoices/erp.dbo.Invoices.parquet` | -`ingestron apply` writes the ADF resources under `build//`: -the SQL Server linked service through the `shir-erp` integration runtime, the +`ingestron apply` writes the factory to `build/adf/`, with a parameter file +for each environment: the SQL Server linked service through the `shir-erp` integration runtime, the pipeline with a copy per table, and a daily trigger. Nothing reads the files before the reports do, so nothing checks rows: neither the contract's quality rules nor its required columns. diff --git a/examples/layers-and-transforms/README.md b/examples/layers-and-transforms/README.md index 22578de..ccf0e1b 100644 --- a/examples/layers-and-transforms/README.md +++ b/examples/layers-and-transforms/README.md @@ -2,12 +2,12 @@ - Two flow tables land in **bronze** (the first layer). - `store_sales` joins them in **silver** (a transform's default layer): - [a materialised view](build/dev/src/sales/silver/store_sales.sql) over + [a materialised view](build/databricks/src/sales/silver/store_sales.sql) over `transforms/store_sales.sql`, where `{{ tables.sales }}` became - `` `dev`.`bronze`.`sales` ``. + `` `bronze`.`sales` ``, in the pipeline's catalog. - `region_sales` reads `store_sales` and says `layer: gold`. The standard's rule for `layer: gold` turns clustering off and adds a - retention property: compare [gold](build/dev/src/sales/gold/region_sales.sql) + retention property: compare [gold](build/databricks/src/sales/gold/region_sales.sql) with silver. Transforms are ordered by their inputs, not by file name, and one pipeline diff --git a/examples/northwind/README.md b/examples/northwind/README.md index 1a56243..2cc049b 100644 --- a/examples/northwind/README.md +++ b/examples/northwind/README.md @@ -2,7 +2,7 @@ ADF copies four tables from Northwind, Microsoft's sample database, into landing storage, and Databricks reads the landed files into tables: `packs: -[databricks@1, adf@1]` in `standard.yaml`. The platform pages' screenshots +[databricks@2, adf@2]` in `standard.yaml`. The platform pages' screenshots of Azure Data Factory, and of Databricks on the ADF route, show this project, deployed. @@ -16,14 +16,14 @@ The standard sets how the team builds its ADF pipelines, under - `folder` and `annotations`: the factory folder `ingestron/northwind`, and a `northwind` annotation on the pipeline. -`ingestron apply` writes, under `build/dev/`: +`ingestron apply` writes: -- `adf/`: the pipeline `pl_northwind_northwind`, its trigger every 24 hours +- `build/adf/`: the pipeline `pl_northwind_northwind`, its trigger every 24 hours (stopped outside production), the linked services and the datasets; -- `databricks/`: the bundle, with a pipeline that reads each table's files +- `build/databricks/`: the bundle, with a pipeline that reads each table's files into `bronze` and builds `silver.daily_sales` from `transforms/daily_sales.yaml`; -- `products/`: a passport for each data product in `products/`. +- `build/products/dev/`: a passport for each data product in `products/`. The server in `connections/northwind.yaml`, and the storage, Key Vault, runtime and workspace in `project.yaml`, are synthetic: replace them with your diff --git a/examples/platform-setup/README.md b/examples/platform-setup/README.md index e16bb57..4c4ec6a 100644 --- a/examples/platform-setup/README.md +++ b/examples/platform-setup/README.md @@ -2,13 +2,14 @@ `platform.yaml` holds what a platform team sets up for a project: -| Setting | Generated in [prod](build/prod/) | -| --------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | -| `workspace.host`, `run_as` | [databricks.yml](build/prod/databricks.yml) target | -| `permissions` | the target's permissions, applied to every resource | -| `compute.budget_policy_id`, `tags` | [the pipeline](build/prod/resources/hr.pipeline.yml) | -| `schedule`, `notifications` | [the refresh job](build/prod/resources/hr_refresh.job.yml) | -| `manage.schemas: true`, `grants.bronze` | [schemas.yml](build/prod/resources/schemas.yml); the pipeline refers to its schema so the bundle creates it first | +| Setting | Generated in `build/databricks/` | +| --------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| `workspace.host`, `run_as` | each target of [databricks.yml](build/databricks/databricks.yml) | +| `permissions` | each target's permissions, applied to every resource | +| `compute.budget_policy_id`, `tags` | [the pipeline](build/databricks/resources/hr.pipeline.yml) | +| `schedule`, `notifications` | [the refresh job](build/databricks/resources/hr_refresh.job.yml) | +| `manage.schemas: true`, `grants.bronze` | [schemas.yml](build/databricks/resources/schemas.yml); the pipeline refers to its schema so the bundle creates it first | +| `manage.schemas: true` | [prerequisites/prod.sql](build/databricks/prerequisites/prod.sql): the grants an administrator runs once in prod | Without `manage.schemas`, no schema is generated, because schemas are often shared infrastructure that one project must not change. diff --git a/examples/quality/README.md b/examples/quality/README.md index 8da7c2f..366b93d 100644 --- a/examples/quality/README.md +++ b/examples/quality/README.md @@ -1,7 +1,7 @@ # Quality The standard sets `quality.required: drop` and `quality.default: warn`. -In [the generated table](build/dev/src/iot/bronze/readings.sql): +In [the generated table](build/databricks/src/iot/bronze/readings.sql): | Check | From | Action | | ---------------------------------------- | -------------------------------- | ---------------------------------------------- | diff --git a/examples/standards-and-conventions/README.md b/examples/standards-and-conventions/README.md index 4a93c08..a7b3ca7 100644 --- a/examples/standards-and-conventions/README.md +++ b/examples/standards-and-conventions/README.md @@ -2,11 +2,10 @@ Acme's standard, shared by all its projects: -| Setting | Effect in [prod](build/prod/) | +| Setting | Effect in prod | | -------------------------------------------------------------- | ------------------------------------------------------------------ | | `catalog: acme`, `schema: "{{ environment }}_{{ layer }}"` | one catalog; schema `prod_raw` | | `table: "{% if source %}{{ source }}__{% endif %}{{ table }}"` | `ads__campaigns`, `web__visits` | -| `pipeline: "{{ project }}_{{ environment }}"` | pipeline `marketing_prod` | | `layers: [raw, curated, published]` | flow tables land in `raw` | | `types: { number: DECIMAL(38,6) }` | `spend` hinted as `DECIMAL(38,6)` | | `ingest.settings.files.schema_evolution: rescue` | `schemaEvolutionMode => 'rescue'` | diff --git a/examples/synapse-views/README.md b/examples/synapse-views/README.md index 36c1946..6c6c3a7 100644 --- a/examples/synapse-views/README.md +++ b/examples/synapse-views/README.md @@ -1,19 +1,22 @@ # Serve files as SQL ADF copies the ERP's tables into storage, one file per table, and Synapse -serverless SQL shows them to reports as views: `packs: [synapse@1, adf@1]` and +serverless SQL shows them to reports as views: `packs: [synapse@2, adf@2]` and `store: { default: synapse-view }` in `standard.yaml`. Synapse is listed first even though ADF runs first: the first pack names and types the views; ADF only copies. -`ingestron apply` writes, for each environment under `build//`: +`ingestron apply` writes: -- `adf/`: the copy pipeline, its trigger, linked services and datasets; -- `synapse/views.sql`: a view per table over its file, `[bronze].[customers]` - and `[bronze].[invoices]`, and `[silver].[customer_totals]` over the - invoices view, from `transforms/customer_totals.yaml`. +- `build/adf/`: the copy pipeline, its trigger, linked services and datasets, + and a parameter file for each environment; +- `build/synapse/views.sql`: a view per table over its file, + `[bronze].[customers]` and `[bronze].[invoices]`, and + `[silver].[customer_totals]` over the invoices view, from + `transforms/customer_totals.yaml`, with `environments/.sql` + setting each environment's database. -Run `views.sql` against the serverless database `sales_` after +Run `sqlcmd -i environments/.sql` from `build/synapse/` after each change to the definitions; loads need nothing more. `ingestron request` lists what the platform team creates: the database, a master key if it has none, the external data source `landing` with its credential, Storage Blob Data From a928abc745558634daef86e5c8ffe08402c27dc0 Mon Sep 17 00:00:00 2001 From: The Architect Date: Thu, 5 Nov 2026 17:26:32 +1300 Subject: [PATCH 04/39] Depend on ingestron 2.0: cli 2.0.0, authoring 2.0.0, definitions 1.10.0 and the four packs 2.0.0 Signed-off-by: The Architect --- package.json | 14 +++--- pnpm-lock.yaml | 119 +++++++++++++++++++++++-------------------------- 2 files changed, 62 insertions(+), 71 deletions(-) diff --git a/package.json b/package.json index 9f29437..02ff510 100644 --- a/package.json +++ b/package.json @@ -23,13 +23,13 @@ "sync:vscode": "node scripts/sync-vscode-commands.mjs" }, "dependencies": { - "@ingestron/authoring": "^1.7.0", - "@ingestron/cli": "^1.10.2", - "@ingestron/definitions": "^1.7.1", - "@ingestron/pack-adf": "^1.5.0", - "@ingestron/pack-databricks": "^1.1.0", - "@ingestron/pack-duckdb": "^1.3.1", - "@ingestron/pack-synapse": "^1.2.0", + "@ingestron/authoring": "^2.0.0", + "@ingestron/cli": "^2.0.0", + "@ingestron/definitions": "^1.10.0", + "@ingestron/pack-adf": "^2.0.0", + "@ingestron/pack-databricks": "^2.0.0", + "@ingestron/pack-duckdb": "^2.0.0", + "@ingestron/pack-synapse": "^2.0.0", "fflate": "0.8.2", "fumadocs-core": "16.17.8", "fumadocs-mdx": "15.4.10", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c04e507..9520a12 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,26 +9,26 @@ importers: .: dependencies: '@ingestron/authoring': - specifier: ^1.7.0 - version: 1.7.0 + specifier: ^2.0.0 + version: 2.0.0 '@ingestron/cli': - specifier: ^1.10.2 - version: 1.10.2 + specifier: ^2.0.0 + version: 2.0.0 '@ingestron/definitions': - specifier: ^1.7.1 - version: 1.7.1 + specifier: ^1.10.0 + version: 1.10.0 '@ingestron/pack-adf': - specifier: ^1.5.0 - version: 1.5.0(@ingestron/definitions@1.7.1) + specifier: ^2.0.0 + version: 2.0.0(@ingestron/definitions@1.10.0) '@ingestron/pack-databricks': - specifier: ^1.1.0 - version: 1.1.0(@ingestron/definitions@1.7.1) + specifier: ^2.0.0 + version: 2.0.0(@ingestron/definitions@1.10.0) '@ingestron/pack-duckdb': - specifier: ^1.3.1 - version: 1.3.1(@ingestron/definitions@1.7.1) + specifier: ^2.0.0 + version: 2.0.0(@ingestron/definitions@1.10.0) '@ingestron/pack-synapse': - specifier: ^1.2.0 - version: 1.2.0(@ingestron/definitions@1.7.1) + specifier: ^2.0.0 + version: 2.0.0(@ingestron/definitions@1.10.0) fflate: specifier: 0.8.2 version: 0.8.2 @@ -783,54 +783,50 @@ packages: cpu: [x64] os: [win32] - '@ingestron/authoring@1.7.0': - resolution: {integrity: sha512-T7I954MuZdDK8tSZuxw572mRN2jG27hpayd9Cvjwi5GzxeqRshWRQEpIUr7y8mXsZrTvJ6RUa8SsURyFNn5W7A==} + '@ingestron/authoring@2.0.0': + resolution: {integrity: sha512-atfEkCMQIxjE+349so3TU6djsKxhYbWTI1T4T6EUKJ/yawin1eGlfo5eku2mA1qr8n897kLBdpZmMdS4Wl5gnA==} engines: {node: '>=22.13.0'} - '@ingestron/cli@1.10.2': - resolution: {integrity: sha512-rItdOoqYoiOrCmzLaQcCFpVsbQ6d4Iymqt4uFYeMeNJa+qpC649uZrkaC6W1HgH7E3DA8eQBX7CHiEFYP7TsdQ==} + '@ingestron/cli@2.0.0': + resolution: {integrity: sha512-z0ArNi1lcbVJSK9dQrnQEl/IrjiGp/udqnM/WwY7VmyHLh9pMxekqR5Y9mte9FYqRpF5n4+IU6+MUdU3F1uobA==} engines: {node: '>=22.13.0'} hasBin: true - '@ingestron/config@1.3.0': - resolution: {integrity: sha512-F0Ws3jBubriVzBUUTc3cGBxkqqUxNns+YYAw/P9o0vH0JN06lcoz3ecmTEIxY/DMteK85FtNZCsxe8LPg5BNiw==} + '@ingestron/config@1.5.0': + resolution: {integrity: sha512-944AP9ehRUE4PL2GfUc8qEsadugYwWgCV3E9KNEcYRwGZYEU+iR8bgLUxjlETphV/dv2EkdP0B6x2Pq1c6BaRw==} engines: {node: '>=22.13.0'} - '@ingestron/config@1.4.0': - resolution: {integrity: sha512-QxxKUs/1MRquAzU3tptMtdkDm3Xqu97bSkQ89lXJLHpXF43t/v1pQtKSVp+tfsTIlh4Iae+g6qVgEKSYFOqr+w==} - engines: {node: '>=22.13.0'} - - '@ingestron/definitions@1.7.1': - resolution: {integrity: sha512-lKg6qnPjAkiT85dCXRLBlB67tB9r2QYlQ16aL7kgV+rFGE0mPk185wDx5f2xQckM0qbM0O5AQsN9ZXLJ4g9mvA==} + '@ingestron/definitions@1.10.0': + resolution: {integrity: sha512-i/CK6PFAaMVzYyVOMbTrdt052+/0tEcAVGPj0nTr7VqbmugVP8Ai1G9LYoqMnxxKyvU4rx0h945aKba5uHAPyw==} engines: {node: '>=22.13.0'} '@ingestron/generate@1.0.1': resolution: {integrity: sha512-xJGEAYV1dtoy0bajHwthzo7XPx1GQvcRgfz3FFK9578OOQolUF2W+Uu3VXxM+3hoZ1CDPhA+hcHlq6GZo3oPoQ==} engines: {node: '>=22.13.0'} - '@ingestron/pack-adf@1.5.0': - resolution: {integrity: sha512-CyVopN5S4os8crgNEl7SOU+wyOtQRMjayNZjewvhbHYUnR4NU4A2p8fQYz9wvxcVemsDaOFsiKs8tHHjYgbNXg==} + '@ingestron/pack-adf@2.0.0': + resolution: {integrity: sha512-eHNxOoxrIewDX90c3bkaZuN7kxb8x2zKdns3Z0amZrY0HP4QB6NCyKF/1tsc+ENyAn+DVYAM3ftucFDtJHxVkQ==} engines: {node: '>=22.13.0'} peerDependencies: - '@ingestron/definitions': ^1.1.0 + '@ingestron/definitions': ^1.9.0 - '@ingestron/pack-databricks@1.1.0': - resolution: {integrity: sha512-KHdr1NkcmKoCIJkW38NhJ8p/wYUi4gzwcny5QEQ4UYlQtWZJrcBySXc6sWo/+9qBRzosrYT/aEZ7Wq6YqtIk3g==} + '@ingestron/pack-databricks@2.0.0': + resolution: {integrity: sha512-5K9qDbLKkVGTmdmjPsq+Zm0jZVYBZpxQzg5jUH3XNomlNPaPNPzYIZV34RIshYgI1P7Wly7/sh4HvUTjBWIn9A==} engines: {node: '>=22.13.0'} peerDependencies: - '@ingestron/definitions': ^1.2.0 + '@ingestron/definitions': ^1.9.0 - '@ingestron/pack-duckdb@1.3.1': - resolution: {integrity: sha512-cg0kgmr/lP18gmy0997Ql1wzIXs6hkgvksd2feFFZHf9oj00+Z+3bhEWeK4fHbKAVqm4EYo5B11xYq0KgRMONw==} + '@ingestron/pack-duckdb@2.0.0': + resolution: {integrity: sha512-GVQpprSyw7+vKdSF0xkj/uX6Z/1Cblco/qIgnGLvUXaRnYUE6UaMj4Q3qFP0wiA1DxguJIYw002kYm6SF5HIWA==} engines: {node: '>=22.13.0'} peerDependencies: '@ingestron/definitions': ^1.3.1 - '@ingestron/pack-synapse@1.2.0': - resolution: {integrity: sha512-ZATvMdaC0tvj3cSLqaySHfUcHHtzREzOwFiYPAZEDnQID9rWdbC9H7TjmnNjhe88im1QY7ngiJT1tvjJ7I/Whw==} + '@ingestron/pack-synapse@2.0.0': + resolution: {integrity: sha512-R15xhzEt4ptoww23WKVw2JGDmurL4RbEHmv1hEtVefj/cStqicA5HnddGx0LU8ZA7Y+SVpLCW44OlyLenUh8Eg==} engines: {node: '>=22.13.0'} peerDependencies: - '@ingestron/definitions': ^1.2.0 + '@ingestron/definitions': ^1.9.0 '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -3324,61 +3320,56 @@ snapshots: '@img/sharp-win32-x64@0.35.5': optional: true - '@ingestron/authoring@1.7.0': + '@ingestron/authoring@2.0.0': dependencies: - '@ingestron/cli': 1.10.2 - '@ingestron/config': 1.3.0 - '@ingestron/definitions': 1.7.1 + '@ingestron/cli': 2.0.0 + '@ingestron/config': 1.5.0 + '@ingestron/definitions': 1.10.0 '@ingestron/generate': 1.0.1 yaml: 2.9.1 - '@ingestron/cli@1.10.2': + '@ingestron/cli@2.0.0': dependencies: - '@ingestron/config': 1.4.0 - '@ingestron/definitions': 1.7.1 + '@ingestron/config': 1.5.0 + '@ingestron/definitions': 1.10.0 '@ingestron/generate': 1.0.1 - '@ingestron/pack-adf': 1.5.0(@ingestron/definitions@1.7.1) - '@ingestron/pack-databricks': 1.1.0(@ingestron/definitions@1.7.1) - '@ingestron/pack-duckdb': 1.3.1(@ingestron/definitions@1.7.1) - '@ingestron/pack-synapse': 1.2.0(@ingestron/definitions@1.7.1) - yaml: 2.9.1 - - '@ingestron/config@1.3.0': - dependencies: - ajv: 8.20.0 + '@ingestron/pack-adf': 2.0.0(@ingestron/definitions@1.10.0) + '@ingestron/pack-databricks': 2.0.0(@ingestron/definitions@1.10.0) + '@ingestron/pack-duckdb': 2.0.0(@ingestron/definitions@1.10.0) + '@ingestron/pack-synapse': 2.0.0(@ingestron/definitions@1.10.0) yaml: 2.9.1 - '@ingestron/config@1.4.0': + '@ingestron/config@1.5.0': dependencies: ajv: 8.20.0 yaml: 2.9.1 - '@ingestron/definitions@1.7.1': + '@ingestron/definitions@1.10.0': dependencies: - '@ingestron/config': 1.4.0 + '@ingestron/config': 1.5.0 ajv: 8.20.0 liquidjs: 10.30.0 yaml: 2.9.1 '@ingestron/generate@1.0.1': {} - '@ingestron/pack-adf@1.5.0(@ingestron/definitions@1.7.1)': + '@ingestron/pack-adf@2.0.0(@ingestron/definitions@1.10.0)': dependencies: - '@ingestron/definitions': 1.7.1 + '@ingestron/definitions': 1.10.0 - '@ingestron/pack-databricks@1.1.0(@ingestron/definitions@1.7.1)': + '@ingestron/pack-databricks@2.0.0(@ingestron/definitions@1.10.0)': dependencies: - '@ingestron/definitions': 1.7.1 + '@ingestron/definitions': 1.10.0 liquidjs: 10.30.0 yaml: 2.9.1 - '@ingestron/pack-duckdb@1.3.1(@ingestron/definitions@1.7.1)': + '@ingestron/pack-duckdb@2.0.0(@ingestron/definitions@1.10.0)': dependencies: - '@ingestron/definitions': 1.7.1 + '@ingestron/definitions': 1.10.0 - '@ingestron/pack-synapse@1.2.0(@ingestron/definitions@1.7.1)': + '@ingestron/pack-synapse@2.0.0(@ingestron/definitions@1.10.0)': dependencies: - '@ingestron/definitions': 1.7.1 + '@ingestron/definitions': 1.10.0 '@jridgewell/gen-mapping@0.3.13': dependencies: From b1c1b1b90640a1709d8a56357a4491fadacfb83d Mon Sep 17 00:00:00 2001 From: The Architect Date: Thu, 5 Nov 2026 17:26:56 +1300 Subject: [PATCH 05/39] First run: the 2.0 folder, build/duckdb/dev/, and the starter's sample names The first project, the home page and Add a source show what ingestron 2.0 prints and where it writes. The starter's sample customer is Ana. The home page says each platform gets one build for every environment. Signed-off-by: The Architect --- content/docs/how-to/(build)/add-a-source.mdx | 12 ++-- content/docs/index.mdx | 22 +++++-- content/docs/start/first-project.mdx | 65 ++++++++++---------- test/first-project.test.ts | 10 +-- 4 files changed, 62 insertions(+), 47 deletions(-) diff --git a/content/docs/how-to/(build)/add-a-source.mdx b/content/docs/how-to/(build)/add-a-source.mdx index acb0610..610ab7d 100644 --- a/content/docs/how-to/(build)/add-a-source.mdx +++ b/content/docs/how-to/(build)/add-a-source.mdx @@ -26,7 +26,7 @@ and apply it again first. ```csv customer_id,name,region - 1,Aroha,North + 1,Ana,North 2,Ben,South 3,Chen,North ``` @@ -67,9 +67,9 @@ and apply it again first. ``` ```text - dev → build/dev - update run.sql - create tables/bronze.customers.sql + duckdb → build/duckdb (dev) + update dev/run.sql + create dev/tables/bronze.customers.sql 5 files: 1 create, 1 update, 3 unchanged. Ready to apply. ``` @@ -128,7 +128,9 @@ from its `project.yaml`, renaming any value your project already has. ## Start beside what you already run `plan` and `apply` write only to the output folder, `build/` by default, so a -project owns only the files it generated. A first project can hold one source +project owns only the files it generated. Each platform's files are in a +folder of their own, such as `build/databricks/`; see +[Generated files and ownership](/docs/concepts/generated-files). A first project can hold one source while the rest of your platform stays as it is: - **Keep the project in a folder of its own,** in your repository or in a new diff --git a/content/docs/index.mdx b/content/docs/index.mdx index 3585085..fd04f70 100644 --- a/content/docs/index.mdx +++ b/content/docs/index.mdx @@ -52,7 +52,7 @@ ingestron init shop --template duckdb --yes cd shop ingestron plan ingestron apply -duckdb -bail ingestron.duckdb -c ".read build/dev/run.sql" +duckdb -bail ingestron.duckdb -c ".read build/duckdb/dev/run.sql" duckdb ingestron.duckdb -c "SELECT * FROM silver.customer_totals ORDER BY customer" ``` @@ -64,7 +64,7 @@ that Ingestron generated: │ customer │ orders │ total │ │ varchar │ int64 │ double │ ├──────────┼────────┼────────┤ -│ Aroha │ 2 │ 65.75 │ +│ Ana │ 2 │ 65.75 │ │ Ben │ 2 │ 42.0 │ │ Chen │ 1 │ 8.75 │ └──────────┴────────┴────────┘ @@ -80,17 +80,28 @@ each step and each file, then changes the project and breaks it on purpose. This is the starter's transform as Ingestron writes it for DuckDB: your SQL, with each column cast to the type its contract gives. - + The Databricks pack writes the `minimal` example project's orders table as a streaming table in a Databricks pipeline, typed and checked from its contract: - + The bundle around it is YAML, Azure Data Factory gets pipeline JSON, and Synapse gets view SQL. [How Ingestron works](/docs/start/how-it-works) follows a project from the YAML to the platform. +### One build for every environment + +Each platform gets one set of files for every environment, laid out the way +your platform's CI/CD expects: a Databricks bundle with a target per +environment, a Data Factory factory with a parameter file per environment, and +one Synapse script with each environment's values. Only what differs between +environments varies, so a reviewer sees exactly what changes between dev and +prod, and prod deploys the files that test ran. +[Deploy through environments](/docs/how-to/deploy-through-environments) gives +the GitHub Actions and Azure DevOps pipelines. + ### No dependency once it's deployed The generated files need only your platform, so if you stop using Ingestron, @@ -131,7 +142,8 @@ each one and what runs after each load. - It doesn't connect to your sources or your cloud. Its only network call is to fetch a template or standard from a Git URL you give it. -- It doesn't deploy anything. Your platform's tools do. +- It doesn't deploy anything. Your platform's tools do, from your CI/CD + pipeline. - It doesn't read secret values. A `${secret:…}` reference carries only the secret's name. - It doesn't send telemetry, from the command line or from VS Code. diff --git a/content/docs/start/first-project.mdx b/content/docs/start/first-project.mdx index 6339634..3ca25ec 100644 --- a/content/docs/start/first-project.mdx +++ b/content/docs/start/first-project.mdx @@ -81,17 +81,18 @@ Run **Ingestron: Plan**. The output appears in the **Ingestron** output panel. You should see: ```text -dev → build/dev -create README.md -create run.sql -create tables/bronze.orders.sql -create tables/silver.customer_totals.sql +duckdb → build/duckdb (dev) +create dev/README.md +create dev/run.sql +create dev/tables/bronze.orders.sql +create dev/tables/silver.customer_totals.sql 4 files: 4 create. Ready to apply. ``` -`dev` is the project's one environment, and its files go to `build/dev/`. Plan -writes nothing: it shows what `apply` would write. +Each platform's files go to one folder, here `build/duckdb/`, with a folder +for each environment that DuckDB runs: `dev` is the project's one environment. +Plan writes nothing: it shows what `apply` would write. ## 3. Apply @@ -105,7 +106,7 @@ ingestron apply -Run **Ingestron: Apply**. The Explorer then shows the files in `build/dev/`, +Run **Ingestron: Apply**. The Explorer then shows the files in `build/duckdb/dev/`, and each table in a flow is labelled with the methods that build it, the file generated for it and its full name: @@ -120,25 +121,25 @@ file generated for it and its full name: ```text -dev → build/dev -create README.md -create run.sql -create tables/bronze.orders.sql -create tables/silver.customer_totals.sql +duckdb → build/duckdb (dev) +create dev/README.md +create dev/run.sql +create dev/tables/bronze.orders.sql +create dev/tables/silver.customer_totals.sql 4 files: 4 create. Ready to apply. Applied. ``` -`build/dev/` now holds the four files, and `.generated.json`, which records -each file as it was generated. +`build/duckdb/dev/` now holds the four files, and `build/duckdb/` holds +`.generated.json`, which records each file as it was generated. ## 4. Build the tables Run DuckDB in the `shop` folder (in VS Code, open a terminal there): ```sh -duckdb -bail ingestron.duckdb -c ".read build/dev/run.sql" +duckdb -bail ingestron.duckdb -c ".read build/duckdb/dev/run.sql" duckdb ingestron.duckdb -c "SELECT * FROM silver.customer_totals ORDER BY customer" ``` @@ -150,7 +151,7 @@ second shows the totals it built from the five sample orders: │ customer │ orders │ total │ │ varchar │ int64 │ double │ ├──────────┼────────┼────────┤ -│ Aroha │ 2 │ 65.75 │ +│ Ana │ 2 │ 65.75 │ │ Ben │ 2 │ 42.0 │ │ Chen │ 1 │ 8.75 │ └──────────┴────────┴────────┘ @@ -211,7 +212,7 @@ From those files Ingestron wrote `run.sql`. It is plain DuckDB SQL: every column cast to its contract's type, both tables built in one transaction, and published only once both are built. Nothing in it calls Ingestron. - + The [definition model](/docs/concepts/definitions) explains how the files link to each other. @@ -230,9 +231,9 @@ Then replace `samples/orders.csv` with orders that have a channel: ```csv order_id,customer,amount,ordered_at,channel -1,Aroha,25.50,2026-10-01,web +1,Ana,25.50,2026-10-01,web 2,Ben,12.00,2026-10-01,store -3,Aroha,40.25,2026-10-02,web +3,Ana,40.25,2026-10-02,web 4,Chen,8.75,2026-10-03,web 5,Ben,30.00,2026-10-03,store ``` @@ -247,8 +248,8 @@ Plan shows each changed line: the `channel` column is read, cast and published. A reviewer sees the same change in a pull request. ```text -dev → build/dev -update run.sql +duckdb → build/duckdb (dev) +update dev/run.sql @@ -5,5 +5,5 @@ CREATE SCHEMA IF NOT EXISTS "bronze"; @@ -262,7 +263,7 @@ update run.sql -COPY (SELECT "order_id", "customer", "amount", "ordered_at" FROM "bronze"."orders") TO './published/bronze.orders.parquet' (FORMAT parquet); +COPY (SELECT "order_id", "customer", "amount", "ordered_at", "channel" FROM "bronze"."orders") TO './published/bronze.orders.parquet' (FORMAT parquet); COPY (SELECT "customer", "orders", "total" FROM "silver"."customer_totals") TO './published/silver.customer_totals.parquet' (FORMAT parquet); -update tables/bronze.orders.sql +update dev/tables/bronze.orders.sql @@ -1,3 +1,3 @@ CREATE SCHEMA IF NOT EXISTS "bronze"; -CREATE OR REPLACE TABLE "bronze"."orders" AS SELECT CAST("order_id" AS BIGINT) AS "order_id", CAST("customer" AS VARCHAR) AS "customer", CAST("amount" AS DOUBLE) AS "amount", CAST("ordered_at" AS DATE) AS "ordered_at" FROM read_csv('./samples/orders.csv', header = true, delim = ','); @@ -278,7 +279,7 @@ nothing until the contract names it. Apply, and build the tables again: ```sh ingestron apply -duckdb -bail ingestron.duckdb -c ".read build/dev/run.sql" +duckdb -bail ingestron.duckdb -c ".read build/duckdb/dev/run.sql" duckdb ingestron.duckdb -c "SELECT * FROM bronze.orders ORDER BY order_id" ``` @@ -287,9 +288,9 @@ duckdb ingestron.duckdb -c "SELECT * FROM bronze.orders ORDER BY order_id" │ order_id │ customer │ amount │ ordered_at │ channel │ │ int64 │ varchar │ double │ date │ varchar │ ├──────────┼──────────┼────────┼────────────┼─────────┤ -│ 1 │ Aroha │ 25.5 │ 2026-10-01 │ web │ +│ 1 │ Ana │ 25.5 │ 2026-10-01 │ web │ │ 2 │ Ben │ 12.0 │ 2026-10-01 │ store │ -│ 3 │ Aroha │ 40.25 │ 2026-10-02 │ web │ +│ 3 │ Ana │ 40.25 │ 2026-10-02 │ web │ │ 4 │ Chen │ 8.75 │ 2026-10-03 │ web │ │ 5 │ Ben │ 30.0 │ 2026-10-03 │ store │ └──────────┴──────────┴────────┴────────────┴─────────┘ @@ -301,7 +302,7 @@ Two mistakes show how Ingestron protects your work. ### Edit a generated file -Add the line `-- my change` at the end of `build/dev/run.sql`, save it, and +Add the line `-- my change` at the end of `build/duckdb/dev/run.sql`, save it, and plan: ```sh @@ -309,8 +310,8 @@ ingestron plan ``` ```text -dev → build/dev -conflict run.sql +duckdb → build/duckdb (dev) +conflict dev/run.sql Changed by hand since it was generated. Move the change into the definitions, or delete the file to generate it again. 4 files: 3 unchanged, 1 conflict. Resolve the conflicts before applying. @@ -321,13 +322,13 @@ never overwritten. A change belongs in the definitions, where the next plan shows it to a reviewer. To undo it, delete the file and apply: ```sh -rm build/dev/run.sql +rm build/duckdb/dev/run.sql ingestron apply ``` ```text -dev → build/dev -create run.sql +duckdb → build/duckdb (dev) +create dev/run.sql 4 files: 1 create, 3 unchanged. Ready to apply. Applied. diff --git a/test/first-project.test.ts b/test/first-project.test.ts index 230f50e..0847560 100644 --- a/test/first-project.test.ts +++ b/test/first-project.test.ts @@ -178,7 +178,7 @@ describe("Run your first project on your laptop", () => { const { code, output } = shell.run(fence(apply(), "sh")); expect(code, output).toBe(0); expect(trimmed(output)).toBe(trimmed(fence(apply(), "text"))); - expect(existsSync(shell.file("build/dev/run.sql"))).toBe(true); + expect(existsSync(shell.file("build/duckdb/dev/run.sql"))).toBe(true); }); it(`builds the tables and shows the box the page shows${withDuckdb}`, ({ skip }) => { @@ -224,11 +224,11 @@ describe("Run your first project on your laptop", () => { }); it("reports a hand-edited file as a conflict, overwrites nothing, and writes it again once deleted", () => { - const line = /[Aa]dd the line `([^`]+)` at the end of `build\/dev\/run\.sql`/.exec( + const line = /[Aa]dd the line `([^`]+)` at the end of `build\/duckdb\/dev\/run\.sql`/.exec( prose(conflict()), )?.[1]; expect(line, "the line the page adds").toBeDefined(); - appendFileSync(shell.file("build/dev/run.sql"), `${line}\n`); + appendFileSync(shell.file("build/duckdb/dev/run.sql"), `${line}\n`); const [planned, undo] = fences(conflict(), "sh"); const [conflicted, rewritten] = fences(conflict(), "text"); let result = shell.run(planned!); @@ -236,11 +236,11 @@ describe("Run your first project on your laptop", () => { expect(trimmed(result.output)).toBe(trimmed(conflicted!)); result = shell.run("ingestron apply"); expect(result.code).toBe(1); - expect(readFileSync(shell.file("build/dev/run.sql"), "utf8")).toContain(line); + expect(readFileSync(shell.file("build/duckdb/dev/run.sql"), "utf8")).toContain(line); result = shell.run(undo!); expect(result.code, result.output).toBe(0); expect(trimmed(result.output)).toBe(trimmed(rewritten!)); - expect(readFileSync(shell.file("build/dev/run.sql"), "utf8")).not.toContain(line); + expect(readFileSync(shell.file("build/duckdb/dev/run.sql"), "utf8")).not.toContain(line); }); it("reports BROKEN_LINK with its fix for the page's broken flow, and plans nothing once undone", () => { From 6fdff53bba1fc2ae7bbeeffda90bfd6c6193bc8f Mon Sep 17 00:00:00 2001 From: The Architect Date: Thu, 5 Nov 2026 17:26:56 +1300 Subject: [PATCH 06/39] Concepts: one set of files per platform, and values supplied when deploying MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generated files and ownership holds the layout, build//, in one place. Environments and YAML references say that an ${envvar:…} value in a Databricks, Data Factory or Synapse project is supplied by the deploying pipeline, under the name ingestron request gives. Add an environment generates every environment together and shows the new target. Signed-off-by: The Architect --- .../concepts/environments-and-platform.mdx | 35 +++++--- content/docs/concepts/generated-files.mdx | 67 +++++++++++--- content/docs/concepts/yaml-references.mdx | 89 +++++++++++++------ .../how-to/(build)/add-an-environment.mdx | 43 ++++++--- .../(standards)/ask-the-platform-team.mdx | 37 +++++--- test/build-a-project.test.ts | 12 +-- test/reader-journeys.test.ts | 21 ++--- 7 files changed, 218 insertions(+), 86 deletions(-) diff --git a/content/docs/concepts/environments-and-platform.mdx b/content/docs/concepts/environments-and-platform.mdx index e8e6691..100a715 100644 --- a/content/docs/concepts/environments-and-platform.mdx +++ b/content/docs/concepts/environments-and-platform.mdx @@ -5,7 +5,8 @@ description: Reuse one project's definitions with different values and shared in An environment gives the same project another set of values, such as `dev`, `test` and `prod`. It does not change the tools or create anything in the -cloud. Plan and generate each environment on its own or all together. +cloud. Every environment is generated together, into one set of files per +platform that a deployment pipeline promotes from one environment to the next. ## Values that differ @@ -43,14 +44,28 @@ specific to each platform, and not every platform uses each one. See and [ADF runtime and Key Vault](/docs/platforms/adf#runtime-and-key-vault) for their effects. -## Separate generated output +## One set of files for every environment [#one-set-of-files] -The default output is `build//`. With several packs, each pack -gets its own subfolder. Ingestron fills in your values when it generates, but -some files still hold secret references, expressions or parameters that the -platform fills in when it runs. Generating never reads a secret or tests the -deployed infrastructure. + -Use [Add an environment](/docs/how-to/add-an-environment) to create another environment, -and compare the resolved choices before deployment. [Generated files and ownership](/docs/concepts/generated-files) -explains how Ingestron protects each output folder. +Ingestron fills in each environment's values when it generates, then writes +each platform's files once for every environment: a value that differs between +environments becomes a parameter of the bundle, the factory or the Synapse +script, set for each environment, and everything else is written once. So +environments may differ only in values; a table or resource present in some +environments only is +[`ENVIRONMENTS_DIFFER`](/docs/reference/errors#ENVIRONMENTS_DIFFER). +[Generated files and ownership](/docs/concepts/generated-files#where-the-files-go) +shows where each platform's files go. + +Some values are not known until deployment, such as a storage account each +stage of your pipeline supplies. Write them as `${envvar:NAME}`: for +Databricks, Data Factory and Synapse they become values the deploying pipeline +supplies, not values read when generating; see +[Read an environment variable](/docs/concepts/yaml-references#read-an-environment-variable). +Generating never reads a secret or tests the deployed infrastructure. + +Use [Add an environment](/docs/how-to/add-an-environment) to create another +environment and compare the resolved choices, and +[Deploy through environments](/docs/how-to/deploy-through-environments) to +deploy them in turn. diff --git a/content/docs/concepts/generated-files.mdx b/content/docs/concepts/generated-files.mdx index 301d1ee..d6714a7 100644 --- a/content/docs/concepts/generated-files.mdx +++ b/content/docs/concepts/generated-files.mdx @@ -1,17 +1,60 @@ --- title: Generated files and ownership -description: Where Ingestron writes, how it protects your edits, and how to check output in CI. +description: Where Ingestron writes each platform's files, how it protects your edits, and how to check output in CI. --- -Ingestron writes one folder per environment, `build//` by -default. When a standard uses several packs, each pack gets its own folder -inside it, such as `build/dev/databricks/` and `build/dev/adf/`. +Ingestron writes to an output folder, `build/` by default, with one folder for +each platform. Each holds that platform's files for every environment, laid +out as its own CI/CD tools expect. + +## Where the files go [#where-the-files-go] + +A project's files are in the same place whether its standard lists one pack or +several, so adding a second pack never moves the first one's files: + +```text kind="plain" +build/ + databricks/ one bundle: databricks.yml with a target per environment, + resources/, src//, tasks/ and prerequisites/.sql + adf/ one factory: factory/ (the folder Microsoft's build reads) + and parameters/.json + synapse/ views.sql and environments/.sql + duckdb/ a folder per environment: /run.sql and tables/ + products/ a folder per environment of data product passports +``` + +- **Databricks, Azure Data Factory and Synapse** get one set of files for every + environment. What is the same in every environment is written once, as it + is; a value that differs is a bundle variable, an ARM template parameter or + a SQLCMD variable, set in each environment's target, parameter file or + script. A reviewer reads the resources once, and the per-environment values + in one place. +- **DuckDB** has no deployment of its own to promote, so each environment it + runs gets its own folder, whose SQL differs only in where the files are. +- **Passports** describe one environment's tables, so each environment has its + own. + +A platform gets a folder only when some environment uses it. An environment +that runs on DuckDB alone, such as a laptop environment with +`runs-on: duckdb`, is not a target of the Databricks bundle or the factory. + +Environments may differ only in values, such as names, paths and hosts. A +table built in some environments only, or a resource name that differs, is +[`ENVIRONMENTS_DIFFER`](/docs/reference/errors#ENVIRONMENTS_DIFFER). Each +platform page's [What Ingestron writes](/docs/platforms/databricks#what-you-get) +section shows its files, and +[Deploy through environments](/docs/how-to/deploy-through-environments) deploys +them, one environment after another. + +## What protects your edits [#what-protects-your-edits] Generated SQL, Python and YAML files start with a comment saying where they came from and not to edit them. JSON files, such as the Azure Data Factory -resources, cannot hold comments. Each folder also holds `.generated.json`, -which records every generated file, whatever its format. With it Ingestron -knows: +resources, cannot hold comments. Each platform's folder also holds +`.generated.json`, which records every generated file, whatever its format. +For Data Factory it sits beside `factory/`, not in it, because Microsoft warns +that unrelated files in a factory's folder can stop it loading. With it +Ingestron knows: - which files it may update, and which it should delete when a table goes away; @@ -47,9 +90,11 @@ What names Ingestron in the files does nothing when they run: If you stop using Ingestron, keep the files and edit them yourself; you can delete each folder's `.generated.json`. While a project still generates a folder, keep its `.generated.json`: without it, `ingestron plan` reports every -file that differs from what it would generate as a conflict. When a project stops generating a folder, for -example after you remove an environment, `ingestron check` says so; delete that -folder's `.generated.json` to keep its files as your own, as +file that differs from what it would generate as a conflict. When a project +stops generating a folder, for example after you remove the last table of a +platform, or after upgrading from the folder per environment of version 1, +`ingestron check` says so; delete that folder's `.generated.json` to keep its +files as your own, as [Check generated files in CI](/docs/how-to/check-in-ci) shows. The one exception is [Run DuckDB in Azure](/docs/platforms/duckdb-in-azure). @@ -62,7 +107,7 @@ it, as that page shows. `ingestron apply` writes one output folder at a time. A folder with a conflict is left as it was, while the others are still written, and the command ends with a line such as -`Applied: dev. Not applied: prod.` and exits with `1`. +`Applied: databricks. Not applied: adf.` and exits with `1`. If writing a file fails, Ingestron puts back the files it already changed in that folder and says `Nothing was changed.` If it cannot put one back, or diff --git a/content/docs/concepts/yaml-references.mdx b/content/docs/concepts/yaml-references.mdx index d28b92a..0e75b3e 100644 --- a/content/docs/concepts/yaml-references.mdx +++ b/content/docs/concepts/yaml-references.mdx @@ -10,20 +10,20 @@ Quotation marks do not disable references: a quoted YAML string is still resolve ## Choose the right syntax -| Write | Meaning | Where to use it | -| ------------------------------------ | ---------------------------------------------------- | -------------------------------------------------------------- | -| `${values.landing}` | A project value after environment overrides | Definition values | -| `${values.cluster.workers}` | A nested value | Definition values | -| `${values.hosts.0}` | The first item in a value list | Definition values | -| `${environment.name}` | The selected environment, such as `dev` | Values and definitions | -| `${project.id}` or `${project.name}` | The project ID or optional display name | Values and definitions | -| `${file:query.sql}` | Contents of a project file | Whole value, or embedded when the result is scalar | -| `${file:shared.yaml#/columns/0}` | A value within a YAML file | Whole value or embedded scalar; the fragment is a JSON Pointer | -| `${secret:source-password}` | A reference to a secret, never the secret's contents | A whole value accepted by a connector or method | -| `${envvar:SOURCE_HOST}` | Text from an environment variable or an `.env` file | Non-secret values for one person or machine; see below | -| `{{ environment }}` or `{{ table }}` | A naming pattern evaluated with naming context | Standard or platform `naming` only | -| `{{ tables.orders }}` | A declared input's generated name | SQL transform code supported by the selected method | -| `$${bundle.target}` | Literal `${bundle.target}` text after YAML loading | Text intended for another tool; see transform exceptions below | +| Write | Meaning | Where to use it | +| ------------------------------------ | ----------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | +| `${values.landing}` | A project value after environment overrides | Definition values | +| `${values.cluster.workers}` | A nested value | Definition values | +| `${values.hosts.0}` | The first item in a value list | Definition values | +| `${environment.name}` | The selected environment, such as `dev` | Values and definitions | +| `${project.id}` or `${project.name}` | The project ID or optional display name | Values and definitions | +| `${file:query.sql}` | Contents of a project file | Whole value, or embedded when the result is scalar | +| `${file:shared.yaml#/columns/0}` | A value within a YAML file | Whole value or embedded scalar; the fragment is a JSON Pointer | +| `${secret:source-password}` | A reference to a secret, never the secret's contents | A whole value accepted by a connector or method | +| `${envvar:SOURCE_HOST}` | A value the deploying pipeline supplies, or text from an environment variable or an `.env` file | Non-secret values each stage or machine sets; see below | +| `{{ environment }}` or `{{ table }}` | A naming pattern evaluated with naming context | Standard or platform `naming` only | +| `{{ tables.orders }}` | A declared input's generated name | SQL transform code supported by the selected method | +| `$${bundle.target}` | Literal `${bundle.target}` text after YAML loading | Text intended for another tool; see transform exceptions below | Project `id`, `name` and `imports` are read literally. YAML mapping keys are not interpolated. Write references in values, not in field names or import patterns. @@ -91,25 +91,57 @@ Do not embed a secret reference in a URL or connection string. ## Read an environment variable -`${envvar:SOURCE_HOST}` reads text from an environment variable. Names use -letters, digits and underscores and cannot start with a digit. Ingestron uses +`${envvar:NAME}` names a value that is not written in the definitions. Names +use letters, digits and underscores and cannot start with a digit. What +happens to it depends on the platform's pack. + +### Supplied when deploying [#supplied-when-deploying] + +Databricks, Azure Data Factory and Synapse get one set of files for every +environment, so a value from a variable is not read when generating. It +becomes a parameter of the generated files, and the pipeline that deploys +each environment supplies it: + +| Platform | The value becomes | The deploying pipeline sets | +| ------------------ | ---------------------------------------------------------------- | ---------------------------------------------------------------- | +| Databricks | a bundle variable that the environment's target leaves unset | `BUNDLE_VAR_`, where `` is `NAME` in lower case | +| Azure Data Factory | an ARM template parameter that the environment's file leaves out | that parameter, after `factoryName`, when deploying the template | +| Synapse | a sqlcmd variable that the environment's script does not set | an environment variable `NAME`, in the same case | + +So the files are the same on every machine and in CI, and a laptop needs no +CI values to plan. `ingestron request` lists each such value with the name to +set; see [Values to provide](/docs/how-to/ask-the-platform-team#values-to-provide). +A workspace host supplied this way is `DATABRICKS_HOST`. + +Only a value can wait until deploying. A variable in a flow's `source`, a +contract id, a resource name or anything else that decides what is generated +or what it is called is +[`DEPLOY_VALUE`](/docs/reference/errors#DEPLOY_VALUE): write it in +`project.yaml` for each environment. Data Factory and Synapse take only a whole +value from a variable: `${envvar:…}` inside a longer value, such as a path, is +`DEPLOY_VALUE` there too, while Databricks puts the variable into the text. + +### Read when generating [#read-when-generating] + +DuckDB, passports and the editor read the value when generating. Ingestron uses the first non-empty value from the environment of the process that runs it, then `.env.`, then `.env`. An empty value counts as unset. A variable that is unset everywhere reports [`ENVVAR_MISSING`](/docs/reference/errors#ENVVAR_MISSING). The CLI reports it -as an error. The editor reports it as information that names the file to add it -to, so the rest of the project is still checked. +as an error. The editor reports it as information, so the rest of the project +is still checked: for DuckDB it names the file to add it to, and for a value +supplied when deploying it names what the deploying pipeline sets instead. -An environment variable's value becomes ordinary resolved text and may reach -generated files. Use `${secret:…}` for credentials instead. +A value read when generating becomes ordinary text and may reach generated +files. Use `${secret:…}` for credentials instead. ## Choose where a value lives @@ -120,13 +152,14 @@ Start with the first place and move down only when the value does not fit: reviewer sees every change. The `prod` entry in [the example above](#project-values-and-environments) replaces `landing` and `host` this way. -2. **`.env` files**, for a value that differs for each person or machine, such - as a folder on your laptop or a sandbox workspace, or that your - infrastructure code produces, such as the name of a storage account it - creates. Do not commit them. -3. **Pipeline variables in CI.** Set the variable in the pipeline and pass it to - the step that runs Ingestron. The process environment wins over the files, so - CI needs no `.env`. +2. **`.env` files**, for a value read when generating that differs for each + person or machine, such as a folder on your laptop for DuckDB. Do not commit + them. +3. **Pipeline variables in CI.** For Databricks, Data Factory and Synapse, set + the variable for each environment in the pipeline that deploys, as + [Deploy through environments](/docs/how-to/deploy-through-environments#values-supplied-when-deploying) + shows. For DuckDB, pass it to the step that runs Ingestron: the process + environment wins over the files, so CI needs no `.env`. 4. **`${secret:…}`**, for every credential. A `.env` file holds values that are not secret: a line that Ingestron cannot read is reported without quoting it, but a problem with the value itself may show it. diff --git a/content/docs/how-to/(build)/add-an-environment.mdx b/content/docs/how-to/(build)/add-an-environment.mdx index 407b6f6..0514d80 100644 --- a/content/docs/how-to/(build)/add-an-environment.mdx +++ b/content/docs/how-to/(build)/add-an-environment.mdx @@ -1,6 +1,6 @@ --- title: Add an environment -description: Generate another environment, such as test, from the same definitions. +description: Add another environment, such as test, to the files every environment shares, and see what it changes. --- 1. Add it to `environments` in `project.yaml`, with the values that differ: @@ -12,28 +12,48 @@ description: Generate another environment, such as test, from the same definitio -3. Generate it: +3. Generate it. Every environment is generated together, so plan and apply + take no environment: ```sh -ingestron plan --env test -ingestron apply --env test +ingestron plan +ingestron apply ``` -Click the environment in the status bar, or run **Ingestron: Select environment**, and choose `test`. Then run **Ingestron: Plan** then **Ingestron: Apply** from the Command Palette (⇧⌘P on macOS, Ctrl+Shift+P elsewhere). The output is in the Ingestron output panel. Problems show in the Problems panel as you type, before you plan. +Run **Ingestron: Plan**, then **Ingestron: Apply**, from the Command Palette (⇧⌘P on macOS, Ctrl+Shift+P elsewhere). The output is in the Ingestron output panel. To see the new environment's problems and previews, click the environment in the status bar, or run **Ingestron: Select environment**, and choose `test`. -The new environment gets its own folder, `build/test/`. An environment named -`prod` or `production` resolves to production mode in the current Platform model; every other one in -development mode. These modes affect the generated resources, not deployment by -Ingestron. See [Environments and platform](/docs/concepts/environments-and-platform). +The new environment gets no folder of its own. It joins the files each +platform already has: here, a `test` target in the bundle's `databricks.yml`, +with the values that differ between environments: + + + +On other platforms it is a parameter file, `build/adf/parameters/test.json`, a +script, `build/synapse/environments/test.sql`, or a folder, +`build/duckdb/test/`. [Generated files and ownership](/docs/concepts/generated-files#where-the-files-go) +shows the whole layout. + +Only an environment named `dev` is in development mode; every other one, +`test` included, is in production mode, and schedules run only in `prod` or +`production`, so `test`'s refresh job is paused. A platform can set `mode` and +`schedule.paused` itself; see the +[platform defaults](/docs/reference/packs/databricks#platform-configuration). +Environments may differ only in values, such as names, paths and hosts: a table +built in one environment only is refused as +[`ENVIRONMENTS_DIFFER`](/docs/reference/errors#ENVIRONMENTS_DIFFER). + +To deploy the new environment, add it to your deployment pipeline as +[Deploy through environments](/docs/how-to/deploy-through-environments#add-an-environment) +shows. ## Check what differs @@ -51,6 +71,7 @@ For the example above, it prints: ## Platform +- `mode`: `development` → `production` - `workspace.root_path`: `/Workspace/Shared/.bundle/finance/dev` → `/Workspace/Shared/.bundle/finance/test` - `tags.environment`: `dev` → `test` @@ -64,8 +85,8 @@ For the example above, it prints: place, and lists only what differs: - **Platform:** the platform document's values, and the defaults that follow - from the environment. Compared with `prod`, it also shows `mode` and - `schedule.paused`, which production changes. + from the environment, such as `mode`. Compared with `prod`, it also shows + `schedule.paused`, because only `prod` runs its schedule. - **Names:** the pipeline and job names, which are not per table. - **Table:** one section for each table that differs: its full name, each step's method and settings, the connection's settings and the table's diff --git a/content/docs/how-to/(standards)/ask-the-platform-team.mdx b/content/docs/how-to/(standards)/ask-the-platform-team.mdx index 831c29b..a53a50a 100644 --- a/content/docs/how-to/(standards)/ask-the-platform-team.mdx +++ b/content/docs/how-to/(standards)/ask-the-platform-team.mdx @@ -21,8 +21,8 @@ environment in the status bar, unsaved, to review and send. Each section appears only when it has something to say, in this order: - **Values to provide:** each environment variable the definitions read, - where it is used, and whether it is set; never its value. See - [Values to provide](#values-to-provide). + where it is used, whether it is set, and the name the deploying pipeline + supplies it as; never its value. See [Values to provide](#values-to-provide). - The identity pipelines and jobs run as, and the workspace. - **Switched on:** each `manage` switch the platform turns on, what it does and what it needs on the platform, from the pack's @@ -37,7 +37,8 @@ Each section appears only when it has something to say, in this order: schedules and whether each is paused, and what Azure Data Factory, the Synapse database or the DuckDB job needs. Ingestron lists these from the definitions; it has not checked that they exist. -- **SQL to run once (review first):** the generated `prerequisites.sql`. +- **SQL to run once (review first):** the generated + `prerequisites/.sql`. Nothing is sent anywhere: send the request the way your team takes requests. @@ -65,7 +66,7 @@ Its request for `dev` is: A platform document that names `run_as` and turns on `manage` switches adds what each switch needs and the SQL to run once; see -[Create the schemas](/docs/platforms/databricks#create-the-schemas). +[Before you deploy](/docs/platforms/databricks#prerequisites) on the Databricks page. ### Azure Data Factory @@ -97,9 +98,10 @@ the request for `dev` adds the job, with the role ADF needs to start it under ## Values to provide [#values-to-provide] -A value only the platform team knows, such as a storage path, can come from -an environment variable with `${envvar:…}`. A variable that is not set yet -does not stop the request: the request opens by asking for it. +A value only the platform team knows, or one each stage of a deployment +supplies, such as a storage path, can come from an environment variable with +`${envvar:…}`. A variable that is not set yet does not stop the request: the +request opens by asking for it. In the `shop` project above, read the landing folder from a variable in `project.yaml`: @@ -118,7 +120,9 @@ With `LANDING_PATH` set nowhere, `ingestron request --env dev` prints: These values come from environment variables. Set them in the pipeline that runs Ingestron for this environment, or in `.env.dev` on your own computer, kept out of Git. -- `LANDING_PATH`, used for `values.landing` in `project.yaml`: not set yet. +The files of the packs that build every environment together take these values when deploying: the pipeline that deploys them supplies each value for each environment, under the names given. + +- `LANDING_PATH`, used for `values.landing` in `project.yaml`: not set yet. The deploying pipeline supplies it as `BUNDLE_VAR_landing_path` (databricks). Until a variable is set, this request shows it as `${envvar:NAME}`, and leaves out what cannot be worked out without it. @@ -141,6 +145,13 @@ Ingestron has not checked these; the generated files expect them. - No schedule: the pipelines run only when someone starts them. ``` +For Databricks, Azure Data Factory and Synapse, the generated files take such +a value when deploying, not when generating: the line names what the +deploying pipeline sets, here the bundle variable `BUNDLE_VAR_landing_path`. +For Data Factory it is the ARM template parameter to pass, and for Synapse the +sqlcmd variable. [Deploy through environments](/docs/how-to/deploy-through-environments#values-supplied-when-deploying) +shows where each goes in the pipeline. + In VS Code, **Ingestron: Prepare platform request** opens the same request, unsaved: @@ -161,11 +172,17 @@ unsaved: its tables need, and leaves out the SQL to run once. - A table whose contract or method holds such a variable is left out, and the request names it. +- Only values can wait until deploying. A variable that decides what is + generated or what it is called, such as a flow's `source`, is refused by + `ingestron plan` as [`DEPLOY_VALUE`](/docs/reference/errors#DEPLOY_VALUE) + for Databricks, Data Factory and Synapse; write it in `project.yaml` for + each environment. The request then leaves out the deploy names and says to + run `ingestron plan`. [Environment files](/docs/concepts/yaml-references#use-environment-files) says how to set the variables on your computer, and -[Check generated files in CI](/docs/how-to/check-in-ci#give-the-check-its-variables) -how to set them in CI. +[Deploy through environments](/docs/how-to/deploy-through-environments#values-supplied-when-deploying) +how to supply them in CI. ## When the definitions have problems diff --git a/test/build-a-project.test.ts b/test/build-a-project.test.ts index deda1a2..a57f562 100644 --- a/test/build-a-project.test.ts +++ b/test/build-a-project.test.ts @@ -107,11 +107,11 @@ describe("how-to/keep-a-table-current", () => { it("upserts the starter's orders with the page's fragment", () => { const cwd = shop(); - const before = readFileSync(join(cwd, "build/dev/tables/bronze.orders.sql"), "utf8"); + const before = readFileSync(join(cwd, "build/duckdb/dev/tables/bronze.orders.sql"), "utf8"); expect(before).toContain('CREATE OR REPLACE TABLE "bronze"."orders" AS'); merge(join(cwd, "flows/orders.yaml"), fences(section(page, "## On DuckDB"), "yaml")[0]!); expect(cli(cwd, ["apply", "--no-diff"]).code).toBe(0); - const after = readFileSync(join(cwd, "build/dev/tables/bronze.orders.sql"), "utf8"); + const after = readFileSync(join(cwd, "build/duckdb/dev/tables/bronze.orders.sql"), "utf8"); expect(after).toContain('INSERT OR REPLACE INTO "bronze"."orders"'); expect(after).toContain('ORDER BY "ordered_at" DESC, "_ingestron_file" DESC'); }); @@ -203,7 +203,7 @@ describe("how-to/write-a-contract", () => { const out = join(databricks, "out"); expect(cli(databricks, ["apply", "--no-diff", "--out", out]).code).toBe(0); const bronze = readFileSync( - join(out, "dev/src", projectId(databricks), "bronze/orders.sql"), + join(out, "databricks/src", projectId(databricks), "bronze/orders.sql"), "utf8", ); expect(bronze).toContain( @@ -213,7 +213,7 @@ describe("how-to/write-a-contract", () => { const duckdb = copy("duckdb"); retype(join(duckdb, "contracts/orders.yaml"), "customer", "string", "object"); expect(cli(duckdb, ["apply", "--no-diff", "--out", join(duckdb, "out")]).code).toBe(0); - expect(readFileSync(join(duckdb, "out/dev/tables/bronze.orders.sql"), "utf8")).toContain( + expect(readFileSync(join(duckdb, "out/duckdb/dev/tables/bronze.orders.sql"), "utf8")).toContain( 'CAST("customer" AS VARCHAR)', ); }); @@ -234,7 +234,7 @@ describe("how-to/write-a-contract", () => { const out = join(cwd, "out"); const id = projectId(cwd); expect(cli(cwd, ["apply", "--no-diff", "--out", out]).code).toBe(0); - const bronze = readFileSync(join(out, "dev/src", id, "bronze/orders.sql"), "utf8"); + const bronze = readFileSync(join(out, "databricks/src", id, "bronze/orders.sql"), "utf8"); expect(bronze).toContain( "CONSTRAINT `customer_id_minimum` EXPECT (`customer_id` IS NULL OR `customer_id` >= 1)", ); @@ -250,7 +250,7 @@ describe("how-to/write-a-contract", () => { stringify({ ...project, imports: [...project.imports, "platform.yaml"] }), ); expect(cli(cwd, ["apply", "--no-diff", "--out", out]).code).toBe(0); - expect(readFileSync(join(out, "dev/src", id, "apply_contract.py"), "utf8")).toContain( + expect(readFileSync(join(out, "databricks/tasks", id, "apply_contract.py"), "utf8")).toContain( '"name": "customer_id_unique"', ); }); diff --git a/test/reader-journeys.test.ts b/test/reader-journeys.test.ts index 401dfe0..1ac85fb 100644 --- a/test/reader-journeys.test.ts +++ b/test/reader-journeys.test.ts @@ -205,18 +205,19 @@ describe("concepts/generated-files", () => { it("quotes the header line the Databricks pack writes", () => { const cwd = copy("retail"); - expect(cli(cwd, ["apply", "--no-diff", "--env", "dev"]).code).toBe(0); - const first = readFileSync(join(cwd, "build/dev/src/retail/bronze/orders.sql"), "utf8").split( - "\n", - )[0]!; + expect(cli(cwd, ["apply", "--no-diff"]).code).toBe(0); + const first = readFileSync( + join(cwd, "build/databricks/src/retail/bronze/orders.sql"), + "utf8", + ).split("\n")[0]!; expect(runTime.replace(/\n\s*/g, " ")).toContain(`\`${first}\``); }); it("says which ADF pipelines carry the annotation, as the pack writes them", () => { const annotations = (name: string) => { const cwd = copy(name); - expect(cli(cwd, ["apply", "--no-diff", "--env", "dev"]).code).toBe(0); - const folder = join(cwd, "build/dev/adf/pipeline"); + expect(cli(cwd, ["apply", "--no-diff"]).code).toBe(0); + const folder = join(cwd, "build/adf/factory/pipeline"); return readdirSync(folder).map( (file) => JSON.parse(readFileSync(join(folder, file), "utf8")).properties.annotations, ); @@ -234,12 +235,12 @@ describe("concepts/generated-files", () => { it("says plan reports a differing file as a conflict when .generated.json is gone", () => { const cwd = copy("duckdb"); expect(cli(cwd, ["apply", "--no-diff"]).code).toBe(0); - rmSync(join(cwd, "build/dev/.generated.json")); - appendFileSync(join(cwd, "build/dev/run.sql"), "-- mine\n"); + rmSync(join(cwd, "build/duckdb/.generated.json")); + appendFileSync(join(cwd, "build/duckdb/dev/run.sql"), "-- mine\n"); const planned = cli(cwd, ["plan", "--no-diff"]); expect(planned.code).toBe(1); - expect(planned.output).toMatch(/^conflict +run\.sql$/m); - expect(planned.output).toMatch(/^adopt +README\.md$/m); + expect(planned.output).toMatch(/^conflict +dev\/run\.sql$/m); + expect(planned.output).toMatch(/^adopt +dev\/README\.md$/m); expect(runTime.replace(/\s+/g, " ")).toContain( "reports every file that differs from what it would generate as a conflict", ); From e47716b96c3969ee4ce5b4b265f026a91bba427a Mon Sep 17 00:00:00 2001 From: The Architect Date: Thu, 5 Nov 2026 17:34:15 +1300 Subject: [PATCH 07/39] PlatformDefaults reads the pack's platformFields The platform defaults table lists the Platform fields a pack declares it reads, in its order, keeping those whose description states a default, instead of a list kept in the docs. Signed-off-by: The Architect --- lib/reference.ts | 41 +++++++++------------------------- test/platform-defaults.test.ts | 23 +++++++++++++++---- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/lib/reference.ts b/lib/reference.ts index 279f622..a03838f 100644 --- a/lib/reference.ts +++ b/lib/reference.ts @@ -132,39 +132,20 @@ export function shown(value: unknown): string { } /** - * The Platform fields each pack reads, in the order the reference lists - * them: the Databricks pack builds its bundle from all of them. Of the - * Platform's fields, the ADF pack reads only `schedule.paused`, after a - * trigger's own `paused` setting. Without either, pack-adf 1.x falls back to - * `mode` and 2.0 to the environment's name, so `mode` is not listed. - */ -const platformFieldsRead: Record = { - databricks: [ - "mode", - "compute.serverless", - "schedule.timezone", - "schedule.paused", - "manage.schemas", - "manage.tags", - "manage.comments", - "manage.checks", - ], - adf: ["schedule.paused"], -}; - -/** - * A pack's platform defaults: each Platform field it reads, with the default - * the field's description gives, as the editor shows it. Throws for a pack - * that reads none, or a field whose description states no default. + * A pack's platform defaults: each Platform field the pack declares it reads + * (its `platformFields`), in the pack's order, with the default the field's + * description states. Fields with no default, such as `run_as`, are left + * out. Throws for a pack that reads no Platform field, or for a field the + * Platform document does not describe. */ export function platformDefaults(pack: string): { field: FieldInfo; default: string }[] { - const read = platformFieldsRead[pack]; - if (!read) throw new Error(`The ${pack} pack reads no platform defaults.`); + const read = packs.find((candidate) => candidate.id === pack)?.platformFields; + if (!read?.length) throw new Error(`The ${pack} pack reads no platform defaults.`); const fields = documentReference("Platform"); - return read.map((path) => { + return read.flatMap((path) => { const field = fields.find((candidate) => fieldPath(candidate) === path); - const stated = /Default: (.+?)\.?$/.exec(field?.description ?? "")?.[1]; - if (!field || !stated) throw new Error(`The Platform field "${path}" states no default.`); - return { field, default: stated }; + if (!field) throw new Error(`The Platform document has no field "${path}".`); + const stated = /Default: (.+?)\.?$/.exec(field.description ?? "")?.[1]; + return stated ? [{ field, default: stated }] : []; }); } diff --git a/test/platform-defaults.test.ts b/test/platform-defaults.test.ts index 3f85de2..0f5138e 100644 --- a/test/platform-defaults.test.ts +++ b/test/platform-defaults.test.ts @@ -10,6 +10,8 @@ import { describe, expect, it } from "vitest"; import { loadDefinitions } from "@ingestron/definitions"; import { PlatformDefaults } from "../components/reference/platform-defaults"; import { exampleFolder } from "../lib/examples"; +import { adf } from "@ingestron/pack-adf"; +import { databricks } from "@ingestron/pack-databricks"; import { packs, platformDefaults } from "../lib/reference"; /** The platform a sample resolves to in an environment. */ @@ -36,13 +38,16 @@ describe("platform defaults", () => { expect(html).toContain( 'mode', ); + expect(html).toContain("production for every environment except dev"); expect(html).toContain( - "production for environments named prod or production, otherwise development", + "running in prod or production, or where mode is set to production; otherwise paused", ); - expect(html).toContain("paused outside production"); + // Only fields whose description states a default: the pack also reads run_as, which has none. + expect(databricks.platformFields).toContain("run_as"); }); - it("lists only what the ADF pack reads, and nothing for a pack that reads none", () => { + it("lists only what the ADF pack reads, from its platformFields, and nothing for a pack that reads none", () => { + expect(adf.platformFields).toEqual(["schedule.paused"]); // The ADF pack reads the platform's schedule.paused, never its mode. expect(platformDefaults("adf").map((row) => row.field.path.join("."))).toEqual([ "schedule.paused", @@ -68,6 +73,15 @@ describe("platform defaults", () => { }); expect(platform(bare, "prod").mode).toBe("production"); expect(platform(bare, "production").mode).toBe("production"); + // Generated together, as the CLI does, every environment but dev is in production mode. + const together = (environment: string) => + loadDefinitions(bare, { packs, environment, environmentsTogether: true }).model!.platform; + writeFileSync( + project, + readFileSync(project, "utf8").replace(" dev: {}", " dev: {}\n test: {}"), + ); + expect(together("dev").mode).toBe("development"); + expect(together("test").mode).toBe("production"); // A schedule with only its cron runs in UTC, paused outside production. const scheduled = exampleFolder("platform-setup"); expect(platform(scheduled, "dev").schedule).toMatchObject({ timezone: "UTC", paused: true }); @@ -78,7 +92,8 @@ describe("platform defaults", () => { expect(stated).toMatchObject({ "compute.serverless": "true", "schedule.timezone": "UTC", - "schedule.paused": "paused outside production", + "schedule.paused": + "running in prod or production, or where mode is set to production; otherwise paused", "manage.tags": "false", }); }); From 652fdc286b116956541cf1d62086510917877e1c Mon Sep 17 00:00:00 2001 From: The Architect Date: Thu, 5 Nov 2026 17:34:15 +1300 Subject: [PATCH 08/39] Platform pages on 2.0 output, each with From CI Databricks and Azure Data Factory are rewritten in the platform pages' structure on the 2.0 files: one bundle with a target per environment, and one factory with an ARM parameter file per environment, deployed by target or by template with the pre- and post-deployment script. Synapse deploys with sqlcmd from build/synapse, and DuckDB in Azure uploads the environment's folder. Each deploy section renders its step of the CI templates. Shape ADF pipelines and Land files only render the pack's settings in place of hand-written tables. Old section ids stay as anchors. Signed-off-by: The Architect --- content/docs/platforms/adf-pipelines.mdx | 43 +- content/docs/platforms/adf.mdx | 528 +++++++++++---------- content/docs/platforms/databricks.mdx | 350 +++++++------- content/docs/platforms/duckdb-in-azure.mdx | 50 +- content/docs/platforms/duckdb.mdx | 17 +- content/docs/platforms/landing-only.mdx | 55 +-- content/docs/platforms/synapse.mdx | 88 +++- lib/routes.ts | 18 +- test/platform-pages.test.ts | 112 ++--- test/reader-journeys.test.ts | 60 ++- 10 files changed, 723 insertions(+), 598 deletions(-) diff --git a/content/docs/platforms/adf-pipelines.mdx b/content/docs/platforms/adf-pipelines.mdx index 3bf0654..f6f111d 100644 --- a/content/docs/platforms/adf-pipelines.mdx +++ b/content/docs/platforms/adf-pipelines.mdx @@ -13,15 +13,14 @@ up the copies first with [Azure Data Factory](/docs/platforms/adf). The Northwind example copies four tables with one ForEach, retries each copy, adds audit columns and files its resources in one folder. Its ADF files: - + -With ADF alone, on the `adf-landing` route, these folders are directly in -`build//`. The pipeline's `tables` parameter holds each table's -query and landing folder, so nothing is read at run time: +The pipeline's `tables` parameter holds each table's query and landing folder, +so nothing is read at run time: @@ -29,7 +28,7 @@ The Copy inside the ForEach carries the retry policy: @@ -37,7 +36,7 @@ And it adds the audit columns to every row it lands: @@ -108,11 +107,7 @@ The tables of one pipeline share `pattern`, `parallel`, `activity` and `activity` is the Copy activity's policy, for every ADF method: -| Setting | Meaning | Values | -| ---------------- | ------------------------------------------ | ----------- | -| `retries` | How often a failed copy is tried again | 0 to 10 | -| `retry_interval` | Seconds between tries | 30 to 86400 | -| `timeout` | Minutes a copy may run before ADF stops it | 10 to 10080 | + -`timeout` must be at least 10 minutes, which is ADF's own minimum; a smaller -value is reported as a settings problem, so no generated file holds one ADF would -refuse. Keys you leave out keep ADF's defaults. These retries happen inside the +No generated file holds a value ADF would refuse: a `timeout` under ADF's own +minimum is reported as a settings problem. These retries happen inside the run; `retries` at the method's top level still retries the whole window from the trigger (see [Retry and pause](/docs/platforms/adf#retry-and-pause)). With [`after_copy`](/docs/platforms/duckdb-in-azure#start-it-after-the-copies), @@ -136,16 +130,7 @@ win. `audit_columns` adds columns to every landed row, by name. A value is one of the run values ADF fills in when it runs, or text: -| Value | What ADF writes | Type | -| ------------------- | --------------------------------------------- | --------- | -| `run.trigger_time` | The run's trigger time | timestamp | -| `run.window_start` | The start of the run's window | timestamp | -| `run.window_end` | The end of the run's window | timestamp | -| `run.id` | The pipeline run's ID | string | -| `run.pipeline` | The pipeline's name | string | -| `source.object` | The source object, such as `erp.dbo.Invoices` | string | -| `source.connection` | The connection's ID | string | -| other text | The text, such as `${environment.name}` | string | + Each linked service, dataset, pipeline, trigger and activity has a default -name, which `resource_names` changes. Where each name goes, which tables share -a resource, per-environment names and safe renames are in +name, which `resource_names` changes; a name is the same in every environment. +Where each name goes, which tables share a resource and safe renames are in [ADF resource names](/docs/how-to/name-things-your-way#adf-resource-names). ## Next - [ADF resource names](/docs/how-to/name-things-your-way#adf-resource-names): name each linked service, dataset, pipeline and trigger. -- [Publish the factory](/docs/platforms/adf#publish): bring the reviewed - pipelines into your factory. +- [Deploy the factory](/docs/platforms/adf#deploy): build the ARM template + and deploy it to each environment's factory. - [Plan, apply and review](/docs/how-to/plan-and-generate): see what a change of settings does to each pipeline before you write it. diff --git a/content/docs/platforms/adf.mdx b/content/docs/platforms/adf.mdx index 13eb3e3..86523a7 100644 --- a/content/docs/platforms/adf.mdx +++ b/content/docs/platforms/adf.mdx @@ -1,23 +1,21 @@ --- title: Azure Data Factory -description: Copy a source into landing storage with ADF, prepare the runtime and Key Vault, schedule the copies, and publish the factory. +description: Copy a source into landing storage with ADF, from one factory for every environment, and deploy it as an ARM template with a parameter file per environment. --- When Databricks cannot reach the source, for example a database on a private network, Azure Data Factory copies it. ADF reaches the source through an integration runtime and copies each table into ADLS Gen2 landing storage on a -schedule. Ingestron writes the factory's linked services, datasets, Copy -pipelines and triggers from your definitions; on this route, the Databricks -pack writes the pipelines that read the landed files into tables. +schedule. Ingestron writes one factory for every environment, its linked +services, datasets, Copy pipelines, triggers and integration runtimes, and a +parameter file for each environment; on this route, the Databricks pack writes +the pipelines that read the landed files into tables. ADF feeds the other routes too. Everything on this page, from the runtime and -Key Vault to the schedules and publishing, applies to them as well; their own -pages cover what reads the files. To follow this page on one of them, use that -route's `--via` when you create the project, and expect its plan to list that -route's own files beside the factory's, in place of the Databricks ones shown -here: +Key Vault to the schedules and deployment, applies to them as well; their own +pages cover what reads the files: - [Synapse serverless views](/docs/platforms/synapse) (`adf-synapse`): views over the landed files. @@ -26,6 +24,48 @@ here: - [Land files only](/docs/platforms/landing-only) (`adf-landing`): the landed files are the result. +## What Ingestron writes [#what-you-get] + +For the `private-sql-server` starter, which copies an ERP's changed rows +through a self-hosted runtime, with `dev` and `prod` environments: + + + +`factory/` is the factory's folder, as Data Factory Studio would keep it in +Git, and what Microsoft's build reads. Its JSON holds the first environment's +values, so it reads as a working factory. Each table gets a Copy activity that +reads the rows changed in the trigger's window and lands them as Parquet: + + + +The SQL Server linked service reads its password from Key Vault and connects +through the runtime: + + + +`arm-template-parameters-definition.json` replaces Microsoft's default +parameterisation: it lists only the properties that differ between +environments, here the Key Vault, the landing storage and the trigger's state, +so the template gets a parameter for each and nothing else: + + + +Each environment's file sets every one of them, under the name Microsoft's +build gives it. In `prod` the trigger starts: + + + +A value only the deploying pipeline knows, written as `${envvar:NAME}`, is a +parameter that the environment's file leaves out; see +[Supplied when deploying](/docs/concepts/yaml-references#supplied-when-deploying). + ## What it looks like in Azure Data Factory [#on-the-platform] -## Create the project +## Create it [#create-it] + + + + + -[Install the command and the extension](/docs/start/install) first. Then, from a parent folder: +From a parent folder: ```sh -ingestron init sales --connector postgresql --connectivity private --via adf-databricks --yes -cd sales +ingestron init erp --template private-sql-server --yes +cd erp ``` -In VS Code, run **Ingestron: New project** and choose **PostgreSQL**, +For your own source, name its connector and the route instead, such as +`ingestron init sales --connector postgresql --connectivity private --via adf-databricks`. + + + + +Run **Ingestron: New project** and choose the starter, or your source, **private** and the ADF route into Databricks. When a source supports only one route, it is selected for you. See [Create a project](/docs/how-to/start-from-a-template) for the remaining -questions. For another source, choose it from the -[connector catalogue](/docs/connectors); its page lists the routes it supports. - -The files on this page come from the finished PostgreSQL project, which also -sets the ADF resource names; a new project uses the default names. To follow -its exact settings and filenames, -and extract it into an empty folder. Open the folder containing `project.yaml` -in VS Code, or change into it in your terminal. You can generate it with the -synthetic values before arranging live source access. The source hosts, -accounts, tables and contract are synthetic: replace them and adapt the -contract before you deploy. - -To compare sources, holds a -connection for every native connector in one project. You don't need to copy -them into yours. - -### Change the project values - -The connection and platform files read these values. Change the server, -database, object, landing path, Key Vault and runtime to match your setup. -`source_password` holds a Key Vault secret reference. `resource_prefix` -supplies the prefix the example's ADF names use. - - - -### Where each setting goes - - +questions. -The connection holds the source's credentials, the platform the shared -infrastructure, and the standard the shared ingestion defaults. Choices for one -table belong in the flow. Ingestron resolves these files together before it -generates anything. + + -## Platform prerequisites +The [connector catalogue](/docs/connectors) lists every source and the routes +it supports. The files on this page come from the starter: -These must exist before the generated pipelines can run. Ingestron generates -files; it does not provision accounts, drivers, runtime machines, network -access or consent, and generation does not inspect Azure. + +To compare sources, holds a connection for every +native connector in one project. -- **An integration runtime that reaches the source.** A self-hosted runtime - runs on infrastructure you manage and needs the source's drivers and network - access. An Azure runtime uses Azure's managed integration runtime. -- **Key Vault**, holding the source's secrets, with access for the factory's - identity. -- **ADLS Gen2 landing storage**, with write access for the factory's identity. - Microsoft 365 uses its configured application for landing storage instead. -- **Source access:** the permissions, drivers, versions and authentication - setup in the source's vendor prerequisites, linked from its connector page. +The environments differ in their landing storage and Key Vault: -`ingestron request`, or **Ingestron: Prepare platform request** in VS Code, -lists them with this project's values for the team that creates them. For the -finished PostgreSQL project, `ingestron request --env dev` ends with: + - +The platform names the shared infrastructure the copies use: -See [Ask the platform team](/docs/how-to/ask-the-platform-team) for the rest of -the request. + -## Runtime and Key Vault +The connection reaches the ERP privately, with its password in Key Vault, and +the flow names the tables to copy: -Name the runtime, Key Vault and landing storage in the platform, under the -method your flows select: + - + -The example configures `adf-table`, for native full selections. Use `adf-copy` -for SQL-family tables or `adf-files` for file copies; these settings mean the -same for all three methods. +Then generate both environments: -`integration_runtime` names an existing ADF runtime; naming it does not create -or rename it. `runtime_kind` says which kind it is, and must match the runtime -deployed in ADF. The default is `self-hosted`, as in this example. Set -`runtime_kind: azure` in the method settings when the source needs Azure's -runtime. Windows SQL authentication, File System and HDFS need a self-hosted -runtime; other sources' needs are in their connector pages. +```sh +ingestron plan --no-diff +``` - +```text +adf: warnings +(project) warning PACK_NOTICE + Integration runtime `shir-erp` (self-hosted): the deployment writes it by name and type, as Data Factory Studio does; where it creates it, register a node, with network access and any required source drivers, before the copies run. + Before the first deployment to a factory that already has these integration runtimes, run `az deployment group what-if` with the template and that environment's parameter file, and check that it changes no runtime in a way you did not describe. +databricks → build/databricks (dev, prod) +create databricks.yml +create resources/erp.pipeline.yml +create resources/erp_refresh.job.yml +create src/erp/bronze/customers.sql +create src/erp/bronze/invoices.sql + +5 files: 5 create. Ready to apply. +adf → build/adf (dev, prod) +create factory/arm-template-parameters-definition.json +create factory/dataset/ds_erp_erp_erp.json +create factory/dataset/ds_erp_landing.json +create factory/integrationRuntime/shir-erp.json +create factory/linkedService/ls_erp_erp_erp.json +create factory/linkedService/ls_erp_keyvault.json +create factory/linkedService/ls_erp_landing.json +create factory/pipeline/pl_erp_erp.json +create factory/trigger/tr_erp_erp.json +create parameters/dev.json +create parameters/prod.json + +11 files: 11 create. Ready to apply. +``` -## Choose the source and its tables +The warning is a notice, not a problem: the files are still written. It +appears on every plan, apply and check while the factory uses a runtime, as +[Runtime and Key Vault](#runtime-and-key-vault) explains. Run +`ingestron apply` to write the files. -Open the [connector reference](/docs/connectors/postgresql) for your source. -Its connection table lists the required fields and the fields each -authentication mode needs. Secrets use `${secret:name}`: put the value in Key -Vault under that name. +## Settings that shape it [#settings] - +Each setting has an owner: the connection holds the source's address and +credentials, the platform the shared infrastructure, the standard the shared +schedule, and the flow each table's choices; see +[Keep settings with their owner](/docs/concepts/standards-and-packs#keep-settings-with-their-owner). +The examples use `adf-copy`, for SQL-family tables; `adf-table` (native full +selections) and `adf-files` (file copies) take the same settings. - +### Runtime and Key Vault [#runtime-and-key-vault] -Each flow selects its ADF method, so you can also use ADF for a public or -workspace source that another pack can read. The `object` selects the native -table, collection, dataset or relative path. Give each source its own landing -folder. + -For native table copies, ADF writes a full selection each window. On the +The factory gets each runtime its linked services use, as +`factory/integrationRuntime/.json`, so a deployment never drops a +linked service's runtime. An ARM deployment sets a runtime's whole resource, +so describe a runtime linked to a shared one, or an Azure runtime's region +and compute, with the method's `runtime` setting; see the +[ADF pack reference](/docs/reference/packs/adf). Before the first deployment to +a factory that already has its runtimes, run `az deployment group what-if` +with the template and that environment's parameter file, as the notice says. +Windows SQL authentication, File System and HDFS need a self-hosted runtime; +other sources' needs are on their connector pages. + +### Choose the source and its tables [#choose-the-source-and-its-tables] + +The [connector page](/docs/connectors) for your source lists the connection's +fields, each authentication mode's, and the settings it takes in +`ingest.with.source_options`, such as +[REST's pagination](/docs/connectors/rest#adf-settings). Secrets use +`${secret:name}`: put the value in Key Vault under that name. The `object` +selects the native table, collection, dataset or relative path; give each +source its own landing folder. + +Native table copies write a full selection each window. On the `adf-databricks` route, repeated snapshots append repeated rows with the default store; choose [upsert or history](/docs/how-to/keep-a-table-current) -when you need current rows or change history. These store modes need a -reliable ordering column, and a row missing from a later snapshot does not mean -it was deleted. SQL-family copies also support watermark mode. Native table -sources refuse incremental mode rather than invent a watermark. - -Some connectors take settings of their own in `ingest.with.source_options`, -such as REST pagination or a Snowflake staging path. Each connector's page in -the [catalogue](/docs/connectors) lists the ones it accepts, such as -[REST's](/docs/connectors/rest#adf-settings). - -Only the options the selected connector supports are accepted. An unknown -connection or extraction setting stops generation with a diagnostic, and fixed -settings in your standard still take precedence. +for current rows or change history. SQL-family copies, as in this starter, +can copy only the rows changed in each window (`mode: incremental`). An +unknown setting stops generation with a diagnostic, and the standard's fixed +settings still win. -## Schedule the copies +### Define the time windows [#define-the-time-windows] -### Define the time windows - -The generated pipelines run from tumbling window triggers, which divide time -into consecutive, non-overlapping windows. `start` is the first window's -start, in UTC, and `every` sets each window's length. For full selections, -every window copies the selection again. For SQL incremental copies, the -window's bounds filter the watermark column. File copies in `new` mode use the -window to select changed files. - - +The pipelines run from tumbling window triggers, which divide time into +consecutive, non-overlapping windows. `start` is the first window's start, in +UTC, and `every` sets each window's length. A full selection copies again each +window; an incremental copy filters the watermark column by the window's +bounds; file copies in `new` mode select the files changed in it. -### Retry and pause +### Retry and pause [#retry-and-pause] A retry repeats a failed window; it does not move that window's bounds. -`paused` keeps the generated trigger stopped. Without it, the trigger follows -the platform's `schedule.paused`, and otherwise the -[platform default](/docs/reference/packs/adf#platform-configuration). Check the -generated trigger before deploying, especially when `start` is in the past. +Triggers start only in `prod` or `production`; in every other environment +they stay stopped. `paused` keeps a trigger stopped there too, and +`paused: false` starts one elsewhere. Without it, the trigger follows the +platform's `schedule.paused`; see the +[platform defaults](/docs/reference/packs/adf#platform-configuration). To stop only the copies, for example while the source database is down for -maintenance, set `paused` for the copy method in the platform. For the -finished PostgreSQL project, add it beside the other `adf-table` settings in -`platform.yaml`; for your project, use the method your flows select: +maintenance, set `paused` for the copy method in the platform, beside the +other `adf-copy` settings: ```yaml settings: - adf-table: + adf-copy: paused: true ``` `paused: true` in the platform's `schedule` stops the Databricks refresh job and the triggers together; see -[Refresh on a schedule](/docs/platforms/databricks#schedule). - -On the `adf-databricks` route, ADF and Databricks run on their own schedules: -deploying their files does not combine them into one run. On `adf-duckdb`, -ADF can start the DuckDB job after the copies; see +[Refresh on a schedule](/docs/platforms/databricks#schedule). On the +`adf-databricks` route, ADF and Databricks run on their own schedules. On +`adf-duckdb`, ADF can start the DuckDB job after the copies; see [Run DuckDB in Azure](/docs/platforms/duckdb-in-azure). ### Tables that share a schedule SQL-family tables in one connection and source database share their runtime, encryption, schedule, retries and paused state. File tables in one connection -share their runtime and schedule. Give tables that need their own shared -settings separate connections; a watermark per table does not make a separate -trigger. +share their runtime and schedule. Give tables that need their own settings +separate connections. Native full-copy sources get a pipeline per table. -Native full-copy sources generate a pipeline per table. Their shared linked -service and dataset names still need compatible definitions; generation -refuses conflicting shared resources. +### Names and pipeline shape [#set-policy-and-names] -## Set policy and names +Resource names are the same in every environment, because one template is +deployed to each factory. [ADF resource names](/docs/how-to/name-things-your-way#adf-resource-names) +covers where each name goes, sharing rules and safe renames, and +[Shape ADF pipelines](/docs/platforms/adf-pipelines) covers ForEach, retries +inside a run, audit columns and factory folders. -The finished example names its shared resources in `platform.yaml`, and its -pipeline, trigger and Copy activity in the PostgreSQL flow. Leave those roles -out to keep the default names. -[ADF resource names](/docs/how-to/name-things-your-way#adf-resource-names) -covers where each name goes, per-environment names, sharing rules and safe -renames, and [Shape ADF pipelines](/docs/platforms/adf-pipelines) covers -ForEach, retries inside a run, audit columns and factory folders. +## Before you deploy [#prerequisites] -For the allowed methods and the fixed settings across both packs, follow -[Set your organisation's standard](/docs/how-to/configure-a-standard). A flow -cannot override a fixed setting. + -## Generate and inspect - - - - -```sh -ingestron plan --env dev --no-diff -ingestron apply --env dev -``` +Ingestron creates nothing in Azure and does not test credentials, run queries +or inspect the factory. These must exist before the pipelines can run: - - +- **An integration runtime that reaches the source.** A self-hosted runtime + runs on infrastructure you manage, with the source's drivers and network + access; register a node on it where the deployment creates it. +- **Key Vault**, holding the source's secrets, with access for the factory's + identity. +- **ADLS Gen2 landing storage**, with write access for the factory's identity. + Microsoft 365 uses its configured application instead. +- **Source access:** the permissions, drivers, versions and authentication in + the source's vendor prerequisites, linked from its connector page. -Select **dev** with **Ingestron: Select environment**, then run -**Ingestron: Plan** and **Ingestron: Apply**. +`ingestron request --env prod`, or **Ingestron: Prepare platform request** in +VS Code, lists them with this project's values: - - + -For the finished PostgreSQL project, the plan lists the ADF files after the -Databricks ones: +See [Ask the platform team](/docs/how-to/ask-the-platform-team) for the whole +request. -```text -dev/adf → build/dev/adf -create dataset/ds_sales_parquet.json -create dataset/ds_sales_postgresql_postgresql_records.json -create linkedService/ls_sales_lake.json -create linkedService/ls_sales_postgresql.json -create linkedService/ls_sales_secrets.json -create pipeline/pl_sales_orders_dev.json -create trigger/tr_sales_orders_dev.json - -7 files: 7 create. Ready to apply. -``` +## Deploy [#deploy] -Inspect `build/dev/adf/linkedService`, `dataset`, `pipeline` and `trigger`. -With ADF alone, on the `adf-landing` route, these folders are directly in -`build/dev/`. The pipeline's Copy source and linked service use PostgreSQL's -native types, and the Databricks files under `build/dev/databricks` read the -matching landing folder. If you kept a new project's default names, use the -filenames your plan shows. + -Development triggers are stopped by default -([platform defaults](/docs/reference/packs/adf#platform-configuration)). -Generation checks the definitions -and builds the factory's files; it does not test credentials, run queries or -deploy the factory. +One factory serves one project. Microsoft's post-deployment script deletes +every pipeline, dataset, linked service, trigger and integration runtime in +the factory that the template does not hold, so give each project, and each +environment, a factory of its own. Leave the test and production factories +unconnected to Git, as Microsoft recommends. -To see a failure safely, remove `server` from `connections/postgresql.yaml` -and plan again: +Build the ARM template once with Microsoft's +[`@microsoft/azure-data-factory-utilities`](https://www.npmjs.com/package/@microsoft/azure-data-factory-utilities), +on Node.js 20 or 22, from the project folder. It validates every resource and +writes `arm/ARMTemplateForFactory.json` and `arm/PrePostDeploymentScript.ps1`. +The factory ID only names the template's resources; the build downloads its +tool from `adf.azure.com` and changes nothing in Azure: -```text -dev: the definitions have problems -flows/postgresql.yaml:14:5 error METHOD_CHECK - "adf-table" for table "postgresql_records": the connection needs "settings.server" as non-empty text. +```sh +npm install --prefix .adf @microsoft/azure-data-factory-utilities@1.0.3 +node .adf/node_modules/@microsoft/azure-data-factory-utilities/lib/index export build/adf/factory arm ``` -The diagnostic names the missing setting, and no generated files change. -Restore the field before you apply. - -## Publish the factory [#publish] +Replace `` with the development factory's resource ID, +`/subscriptions//resourceGroups//providers/Microsoft.DataFactory/factories/`. +Keep `.adf/` and `arm/` out of Git: they are rebuilt for each deployment. -With ADF alone (the `adf-landing` route), the factory's files are in -`build//`; when the standard lists another pack too, they are in -`build//adf/`. Ingestron does not publish the factory. +Then, for each environment in turn, signed in to its subscription, stop the +triggers, deploy the template with the environment's parameter file, and run +the script again to delete what the template no longer holds and start the +triggers whose state is `Started`. Here for `prod`: -### Review the generated resources +```powershell +./arm/PrePostDeploymentScript.ps1 -ArmTemplate arm/ARMTemplateForFactory.json -ArmTemplateParameters build/adf/parameters/prod.json -ResourceGroupName -DataFactoryName -PreDeployment $true -DeleteDeployment $false +az deployment group create --resource-group --mode Incremental --template-file arm/ARMTemplateForFactory.json --parameters "@build/adf/parameters/prod.json" factoryName= +./arm/PrePostDeploymentScript.ps1 -ArmTemplate arm/ARMTemplateForFactory.json -ArmTemplateParameters build/adf/parameters/prod.json -ResourceGroupName -DataFactoryName -PreDeployment $false -DeleteDeployment $true +``` -Inspect the `linkedService`, `dataset`, `pipeline` and `trigger` folders. -Confirm the names, credential references, landing paths, schedules and paused -state for the environment you are publishing. Prepare the -[platform prerequisites](#platform-prerequisites) and the connector's access -requirements before running the pipelines. +Replace `` and `` with the environment's resource group and +factory. The script needs PowerShell 7 and the Az modules. Always give it +`-ArmTemplateParameters`: without the environment's file it cannot tell that +`prod`'s triggers start. Keep `--mode Incremental`; Microsoft warns that +`Complete` deletes the resource group's other resources. When +`ingestron request` lists values to provide, add `=` after +`factoryName` for each. -Keep Ingestron's ownership files in its output folder; the ADF resource JSON -is what you bring into the factory's configured Git root. +What reads the files is deployed beside the factory, from its own folder: +[Databricks](/docs/platforms/databricks#deploy), +[Synapse](/docs/platforms/synapse#deploy) or +[DuckDB in Azure](/docs/platforms/duckdb-in-azure#deploy). -### Use the factory's publishing workflow +### From CI [#deploy-from-ci] -Bring the reviewed resources into a feature branch of the factory repository, -then review and merge them into its collaboration branch. In ADF, publish from -that collaboration branch. Committing to Git alone does not update the live -factory. Publishing includes all the factory's pending changes, so review the -complete set with the factory owner. +In a pipeline, the build runs once and each environment's stage deploys the +same template. From the GitHub Actions template, the build: -Follow Microsoft's -[source-control and publishing instructions](https://learn.microsoft.com/en-us/azure/data-factory/source-control#publish-code-changes) -for the factory's Git integration and permissions. For an automated release -pipeline, use your organisation's established ADF deployment process. +