diff --git a/ci-templates/azure-devops/azure-pipelines.yml b/ci-templates/azure-devops/azure-pipelines.yml new file mode 100644 index 0000000..cc572f4 --- /dev/null +++ b/ci-templates/azure-devops/azure-pipelines.yml @@ -0,0 +1,71 @@ +# Deploys the committed build/ folder: checks it, builds the Data Factory +# template once, then deploys the same files to dev, test and prod in turn. +# Copy this file and templates/deploy-environment.yml to your definitions +# repository, add this file as a pipeline, and delete the steps of each +# platform your project does not use. +# +# Set up once, as https://docs.ingestron.io/docs/how-to/deploy-through-environments +# explains: +# - Azure DevOps environments named dev, test and prod. Add approvals to test +# and prod, a branch control check to prod that allows only main, and an +# exclusive lock check to each, so runs deploy an environment one at a time, +# in order (each stage sets lockBehavior: sequential). +# - For each environment, an Azure Resource Manager service connection that +# uses workload identity federation, and a variable group named +# ingestron- with the variables the template reads. +# - The pipeline variable ADF_DEV_FACTORY_ID: the dev factory's resource ID, +# which names the template's resources. Nothing is deployed with it. +trigger: + branches: + include: [main] +pr: none + +stages: + - stage: Build + jobs: + - job: build + pool: + vmImage: ubuntu-latest + steps: + - checkout: self + - task: UseNode@1 + inputs: + version: 22.x + # An exact version: change it when you upgrade, as + # https://docs.ingestron.io/docs/how-to/upgrade#choose-the-version-you-run says. + - script: npm install -g ingestron@2.1.0 + displayName: Install Ingestron + - script: ingestron check + displayName: ingestron check + # Data Factory: Microsoft's export validates the factory, then writes + # the ARM template and the pre- and post-deployment script to arm/. + # It downloads its build tool from adf.azure.com. + - script: | + npm install --prefix "$(Agent.TempDirectory)/adf" @microsoft/azure-data-factory-utilities@1.0.3 + adf="$(Agent.TempDirectory)/adf/node_modules/@microsoft/azure-data-factory-utilities/lib/index" + node "$adf" export build/adf/factory "$(ADF_DEV_FACTORY_ID)" arm + displayName: "Data Factory: build the template" + # Every environment deploys these files, so prod gets the bytes dev + # got. Without Data Factory, copy build alone. + - script: cp -R arm build "$(Build.ArtifactStagingDirectory)/" + displayName: Collect the generated files + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(Build.ArtifactStagingDirectory) + artifact: deploy + + - template: templates/deploy-environment.yml + parameters: + environment: dev + serviceConnection: ingestron-dev + dependsOn: Build + - template: templates/deploy-environment.yml + parameters: + environment: test + serviceConnection: ingestron-test + dependsOn: Deploy_dev + - template: templates/deploy-environment.yml + parameters: + environment: prod + serviceConnection: ingestron-prod + dependsOn: Deploy_test diff --git a/ci-templates/azure-devops/ingestron-check.yml b/ci-templates/azure-devops/ingestron-check.yml index 2424a39..9b7f902 100644 --- a/ci-templates/azure-devops/ingestron-check.yml +++ b/ci-templates/azure-devops/ingestron-check.yml @@ -10,10 +10,12 @@ pool: vmImage: ubuntu-latest steps: - checkout: self - - task: NodeTool@0 + - task: UseNode@1 inputs: - versionSpec: 22.x - - script: npm install -g ingestron@1 + version: 22.x + # An exact version: change it when you upgrade, as + # https://docs.ingestron.io/docs/how-to/upgrade#choose-the-version-you-run says. + - script: npm install -g ingestron@2.1.0 displayName: Install Ingestron - script: ingestron check displayName: ingestron check diff --git a/ci-templates/azure-devops/templates/deploy-environment.yml b/ci-templates/azure-devops/templates/deploy-environment.yml new file mode 100644 index 0000000..c85c1a1 --- /dev/null +++ b/ci-templates/azure-devops/templates/deploy-environment.yml @@ -0,0 +1,155 @@ +# Deploys the files the Build stage published to one environment. Its +# deployment job targets the Azure DevOps environment of the same name, so that +# environment's approvals and checks gate it. +# +# Variables to set in the variable group ingestron-: +# - AZURE_SUBSCRIPTION_ID, AZURE_LOCATION: the subscription and region of the +# environment's resource group. +# - ADF_RESOURCE_GROUP, ADF_FACTORY_NAME: the environment's data factory. +# - SYNAPSE_SERVER: -ondemand.sql.azuresynapse.net. +# - DUCKDB_ACCOUNT, DUCKDB_FILE_SYSTEM, DUCKDB_PATH: where the DuckDB job reads +# run.sql, as the job's sqlUrl names it. +# Every step signs in as the environment's service connection. The Azure CLI +# steps set useGlobalConfig, so that what they start, such as databricks bundle, +# uses the signed-in session: see Microsoft Learn, "Authenticate with Azure +# DevOps on Azure Databricks". +parameters: + - name: environment + type: string + - name: serviceConnection + type: string + - name: dependsOn + type: string + +stages: + - stage: Deploy_${{ parameters.environment }} + displayName: Deploy ${{ parameters.environment }} + dependsOn: ${{ parameters.dependsOn }} + # With the environment's exclusive lock check, runs deploy it one at a time, + # in the order they started. + lockBehavior: sequential + variables: + - group: ingestron-${{ parameters.environment }} + - name: files + value: $(Pipeline.Workspace)/deploy + jobs: + - deployment: deploy + displayName: Deploy ${{ parameters.environment }} + pool: + vmImage: ubuntu-latest + environment: ${{ parameters.environment }} + strategy: + runOnce: + deploy: + steps: + - download: current + artifact: deploy + + # Data Factory. One factory per project: the post-deployment + # script deletes every resource in the factory that the + # template does not hold. + - task: AzurePowerShell@5 + displayName: "Data Factory: stop the triggers" + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + azurePowerShellVersion: LatestVersion + pwsh: true + ScriptType: FilePath + ScriptPath: $(files)/arm/PrePostDeploymentScript.ps1 + ScriptArguments: >- + -ArmTemplate $(files)/arm/ARMTemplateForFactory.json + -ArmTemplateParameters $(files)/build/adf/parameters/${{ parameters.environment }}.json + -ResourceGroupName $(ADF_RESOURCE_GROUP) -DataFactoryName $(ADF_FACTORY_NAME) + -PreDeployment $true -DeleteDeployment $false + - task: AzureResourceManagerTemplateDeployment@3 + displayName: "Data Factory: deploy the template" + inputs: + deploymentScope: Resource Group + azureResourceManagerConnection: ${{ parameters.serviceConnection }} + subscriptionId: $(AZURE_SUBSCRIPTION_ID) + action: Create Or Update Resource Group + resourceGroupName: $(ADF_RESOURCE_GROUP) + location: $(AZURE_LOCATION) + templateLocation: Linked artifact + csmFile: $(files)/arm/ARMTemplateForFactory.json + csmParametersFile: $(files)/build/adf/parameters/${{ parameters.environment }}.json + # Add - for each ARM parameter that + # `ingestron request` lists under "Values to provide". + overrideParameters: -factoryName $(ADF_FACTORY_NAME) + deploymentMode: Incremental + - task: AzurePowerShell@5 + displayName: "Data Factory: clean up and start the triggers" + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + azurePowerShellVersion: LatestVersion + pwsh: true + ScriptType: FilePath + ScriptPath: $(files)/arm/PrePostDeploymentScript.ps1 + ScriptArguments: >- + -ArmTemplate $(files)/arm/ARMTemplateForFactory.json + -ArmTemplateParameters $(files)/build/adf/parameters/${{ parameters.environment }}.json + -ResourceGroupName $(ADF_RESOURCE_GROUP) -DataFactoryName $(ADF_FACTORY_NAME) + -PreDeployment $false -DeleteDeployment $true + + # Databricks: the bundle's target for this environment. A + # deploy that would delete a resource stops and asks; it is + # not approved here. + - bash: | + curl -fsSL -o "$(Agent.TempDirectory)/databricks.tar.gz" https://github.com/databricks/cli/releases/download/v1.23.0/databricks_cli_1.23.0_linux_amd64.tar.gz + echo "a5ec088f7f6cc5974327a42df99261e22fbc6f7ca9cddb945333c238dd9e2289 $(Agent.TempDirectory)/databricks.tar.gz" | sha256sum -c - + mkdir -p "$(Agent.TempDirectory)/databricks" && tar -xzf "$(Agent.TempDirectory)/databricks.tar.gz" -C "$(Agent.TempDirectory)/databricks" databricks + echo "##vso[task.prependpath]$(Agent.TempDirectory)/databricks" + displayName: "Databricks: install the CLI" + - task: AzureCLI@2 + displayName: "Databricks: deploy the bundle" + env: + DATABRICKS_AUTH_TYPE: azure-cli + # One line for each BUNDLE_VAR_ name that `ingestron request` lists: + # BUNDLE_VAR_landing_path: $(LANDING_PATH) + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + useGlobalConfig: true + workingDirectory: $(files)/build/databricks + inlineScript: | + databricks bundle validate -t ${{ parameters.environment }} + databricks bundle deploy -t ${{ parameters.environment }} + + # Synapse serverless: sqlcmd (Go), checked by its SHA-256, + # signed in as the service connection. + - bash: | + curl -fsSL -o "$(Agent.TempDirectory)/sqlcmd.tar.bz2" https://github.com/microsoft/go-sqlcmd/releases/download/v1.10.0/sqlcmd-linux-amd64.tar.bz2 + echo "92516d98c63d99b0994de5b61350c91f6915f9b76f139a59039fbcb225c2e987 $(Agent.TempDirectory)/sqlcmd.tar.bz2" | sha256sum -c - + mkdir -p "$(Agent.TempDirectory)/sqlcmd" && tar -xjf "$(Agent.TempDirectory)/sqlcmd.tar.bz2" -C "$(Agent.TempDirectory)/sqlcmd" sqlcmd + echo "##vso[task.prependpath]$(Agent.TempDirectory)/sqlcmd" + displayName: "Synapse: install sqlcmd" + - task: AzureCLI@2 + displayName: "Synapse: create or alter the views" + # One line for each sqlcmd variable that `ingestron request` lists: + # env: + # SYNAPSE_SOURCE: $(SYNAPSE_SOURCE) + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + useGlobalConfig: true + workingDirectory: $(files)/build/synapse + inlineScript: >- + sqlcmd -S "$(SYNAPSE_SERVER)" --authentication-method ActiveDirectoryDefault + -i environments/${{ parameters.environment }}.sql + + # DuckDB in Azure: copy the environment's SQL to where the job + # reads it. + - task: AzureCLI@2 + displayName: "DuckDB: upload the SQL" + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + useGlobalConfig: true + workingDirectory: $(files) + inlineScript: >- + az storage fs directory upload --account-name "$(DUCKDB_ACCOUNT)" + --file-system "$(DUCKDB_FILE_SYSTEM)" --source "build/duckdb/${{ parameters.environment }}/*" + --destination-path "$(DUCKDB_PATH)" --recursive --auth-mode login diff --git a/ci-templates/github-actions/deploy-environment.yml b/ci-templates/github-actions/deploy-environment.yml new file mode 100644 index 0000000..79645ea --- /dev/null +++ b/ci-templates/github-actions/deploy-environment.yml @@ -0,0 +1,126 @@ +# Deploys the files deploy.yml built to one environment. It runs in the GitHub +# environment of the same name, so that environment's reviewers and branch rule +# gate it, and its variables and secrets are the ones used. Required reviewers +# on a private repository need GitHub Enterprise. +# +# Variables to set in each GitHub environment: +# - AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID: the identity this +# environment deploys as, with a federated credential for the job's subject: +# repo:/:environment: for a repository +# created, renamed or transferred before 15 July 2026, and +# repo:@/@:environment: +# for one created, renamed or transferred since, or opted in +# (gh api repos// gives .owner.id and .id). +# - ADF_RESOURCE_GROUP, ADF_FACTORY_NAME: the environment's data factory. +# - DATABRICKS_CLIENT_ID: the service principal the bundle deploys as, with a +# federation policy for the same subject. +# - SYNAPSE_SERVER: -ondemand.sql.azuresynapse.net. +# - DUCKDB_ACCOUNT, DUCKDB_FILE_SYSTEM, DUCKDB_PATH: where the DuckDB job reads +# run.sql, as the job's sqlUrl names it. +name: deploy environment +on: + workflow_call: + inputs: + environment: + description: The environment to deploy, as project.yaml names it. + required: true + type: string +permissions: + contents: read + id-token: write +jobs: + deploy: + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + concurrency: + group: deploy-${{ inputs.environment }} + cancel-in-progress: false + env: + ENVIRONMENT: ${{ inputs.environment }} + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: deploy + - uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + enable-AzPSSession: true + + # Data Factory. One factory per project: the post-deployment script + # deletes every resource in the factory that the template does not hold. + - name: "Data Factory: stop the triggers" + uses: azure/powershell@f5b8adcfff1904872c7b98d4012d4914d74b1a82 # v3.0.0 + env: + ADF_RESOURCE_GROUP: ${{ vars.ADF_RESOURCE_GROUP }} + ADF_FACTORY_NAME: ${{ vars.ADF_FACTORY_NAME }} + with: + azPSVersion: latest + inlineScript: | + ./arm/PrePostDeploymentScript.ps1 -ArmTemplate arm/ARMTemplateForFactory.json ` + -ArmTemplateParameters "build/adf/parameters/$env:ENVIRONMENT.json" ` + -ResourceGroupName $env:ADF_RESOURCE_GROUP -DataFactoryName $env:ADF_FACTORY_NAME ` + -PreDeployment $true -DeleteDeployment $false + - name: "Data Factory: deploy the template" + env: + ADF_RESOURCE_GROUP: ${{ vars.ADF_RESOURCE_GROUP }} + ADF_FACTORY_NAME: ${{ vars.ADF_FACTORY_NAME }} + # Add name=value after factoryName for each ARM parameter that + # `ingestron request` lists under "Values to provide". + run: | + az deployment group create --resource-group "$ADF_RESOURCE_GROUP" --mode Incremental \ + --template-file arm/ARMTemplateForFactory.json \ + --parameters "@build/adf/parameters/$ENVIRONMENT.json" factoryName="$ADF_FACTORY_NAME" + - name: "Data Factory: clean up and start the triggers" + uses: azure/powershell@f5b8adcfff1904872c7b98d4012d4914d74b1a82 # v3.0.0 + env: + ADF_RESOURCE_GROUP: ${{ vars.ADF_RESOURCE_GROUP }} + ADF_FACTORY_NAME: ${{ vars.ADF_FACTORY_NAME }} + with: + azPSVersion: latest + inlineScript: | + ./arm/PrePostDeploymentScript.ps1 -ArmTemplate arm/ARMTemplateForFactory.json ` + -ArmTemplateParameters "build/adf/parameters/$env:ENVIRONMENT.json" ` + -ResourceGroupName $env:ADF_RESOURCE_GROUP -DataFactoryName $env:ADF_FACTORY_NAME ` + -PreDeployment $false -DeleteDeployment $true + + # Databricks: the bundle's target for this environment. A deploy that + # would delete a resource stops and asks; it is not approved here. + - uses: databricks/setup-cli@9625fdde9ba2607df0087e0d3ec7449198bf2180 # v1.23.0 + - name: "Databricks: deploy the bundle" + working-directory: build/databricks + env: + DATABRICKS_AUTH_TYPE: github-oidc + DATABRICKS_CLIENT_ID: ${{ vars.DATABRICKS_CLIENT_ID }} + # One line for each BUNDLE_VAR_ name that `ingestron request` lists: + # BUNDLE_VAR_landing_path: ${{ vars.LANDING_PATH }} + run: | + databricks bundle validate -t "$ENVIRONMENT" + databricks bundle deploy -t "$ENVIRONMENT" + + # Synapse serverless: sqlcmd (Go), checked by its SHA-256, signed in as + # the identity azure/login used. + - name: "Synapse: install sqlcmd" + run: | + curl -fsSL -o "$RUNNER_TEMP/sqlcmd.tar.bz2" https://github.com/microsoft/go-sqlcmd/releases/download/v1.10.0/sqlcmd-linux-amd64.tar.bz2 + echo "92516d98c63d99b0994de5b61350c91f6915f9b76f139a59039fbcb225c2e987 $RUNNER_TEMP/sqlcmd.tar.bz2" | sha256sum -c - + mkdir -p "$RUNNER_TEMP/sqlcmd" && tar -xjf "$RUNNER_TEMP/sqlcmd.tar.bz2" -C "$RUNNER_TEMP/sqlcmd" sqlcmd + echo "$RUNNER_TEMP/sqlcmd" >> "$GITHUB_PATH" + - name: "Synapse: create or alter the views" + working-directory: build/synapse + env: + SYNAPSE_SERVER: ${{ vars.SYNAPSE_SERVER }} + # One line for each sqlcmd variable that `ingestron request` lists: + # SYNAPSE_SOURCE: ${{ vars.SYNAPSE_SOURCE }} + run: sqlcmd -S "$SYNAPSE_SERVER" --authentication-method ActiveDirectoryDefault -i "environments/$ENVIRONMENT.sql" + + # DuckDB in Azure: copy the environment's SQL to where the job reads it. + - name: "DuckDB: upload the SQL" + env: + DUCKDB_ACCOUNT: ${{ vars.DUCKDB_ACCOUNT }} + DUCKDB_FILE_SYSTEM: ${{ vars.DUCKDB_FILE_SYSTEM }} + DUCKDB_PATH: ${{ vars.DUCKDB_PATH }} + run: | + az storage fs directory upload --account-name "$DUCKDB_ACCOUNT" --file-system "$DUCKDB_FILE_SYSTEM" \ + --source "build/duckdb/$ENVIRONMENT/*" --destination-path "$DUCKDB_PATH" --recursive --auth-mode login diff --git a/ci-templates/github-actions/deploy.yml b/ci-templates/github-actions/deploy.yml new file mode 100644 index 0000000..90b3457 --- /dev/null +++ b/ci-templates/github-actions/deploy.yml @@ -0,0 +1,90 @@ +# Deploys the committed build/ folder: checks it, builds the Data Factory +# template once, then deploys the same files to dev, test and prod in turn. +# Copy this file and deploy-environment.yml to .github/workflows/ in your +# definitions repository, and delete the steps of each platform your project +# does not use. +# +# Set up once, as https://docs.ingestron.io/docs/how-to/deploy-through-environments +# explains: +# - GitHub environments named dev, test and prod. Give test and prod required +# reviewers, and let only the main branch deploy to prod. Required reviewers +# on a private repository need GitHub Enterprise; without them, test and +# prod deploy after dev with no approval. +# - In each environment, the variables deploy-environment.yml reads. +# - The repository variable ADF_DEV_FACTORY_ID: the dev factory's resource ID, +# which names the template's resources. Nothing is deployed with it. +# Each action is pinned to a commit, with its version beside it, and Ingestron +# to an exact version: change them when you upgrade. +name: deploy +on: + push: + branches: [main] +permissions: + contents: read +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + - run: npm install -g ingestron@2.1.0 + - run: ingestron check + + build: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + # Data Factory: Microsoft's export validates the factory, then writes the + # ARM template and the pre- and post-deployment script to arm/. It + # downloads its build tool from adf.azure.com. + - name: "Data Factory: build the template" + env: + ADF_DEV_FACTORY_ID: ${{ vars.ADF_DEV_FACTORY_ID }} + run: | + npm install --prefix "$RUNNER_TEMP/adf" @microsoft/azure-data-factory-utilities@1.0.3 + adf="$RUNNER_TEMP/adf/node_modules/@microsoft/azure-data-factory-utilities/lib/index" + node "$adf" export build/adf/factory "$ADF_DEV_FACTORY_ID" arm + # Every environment deploys these files, so prod gets the bytes dev got. + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: deploy + path: | + arm/ + build/ + if-no-files-found: error + + dev: + permissions: + contents: read + id-token: write + needs: build + uses: ./.github/workflows/deploy-environment.yml + with: + environment: dev + secrets: inherit + + test: + permissions: + contents: read + id-token: write + needs: dev + uses: ./.github/workflows/deploy-environment.yml + with: + environment: test + secrets: inherit + + prod: + permissions: + contents: read + id-token: write + needs: test + uses: ./.github/workflows/deploy-environment.yml + with: + environment: prod + secrets: inherit diff --git a/ci-templates/github-actions/ingestron-check.yml b/ci-templates/github-actions/ingestron-check.yml index 74d84f2..26869ca 100644 --- a/ci-templates/github-actions/ingestron-check.yml +++ b/ci-templates/github-actions/ingestron-check.yml @@ -1,5 +1,8 @@ # Fails the build when the committed build/ folder is not what the definitions # generate today. Copy to .github/workflows/ in your definitions repository. +# Each action is pinned to a commit, with its version beside it. Ingestron is +# pinned to an exact version: change it when you upgrade, as +# https://docs.ingestron.io/docs/how-to/upgrade#choose-the-version-you-run says. name: ingestron check on: pull_request: @@ -11,9 +14,9 @@ jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 22 - - run: npm install -g ingestron@1 + - run: npm install -g ingestron@2.1.0 - run: ingestron check diff --git a/components/example.tsx b/components/example.tsx index b5dc65a..0eb087b 100644 --- a/components/example.tsx +++ b/components/example.tsx @@ -76,8 +76,8 @@ function extractPart({ * Ingestron generates for it. With `pick` (a YAML or JSON path), or `from` * and `to` (text on the first and last lines), only that part, as it is in * the file. - * @example - * @example + * @example + * @example */ export function Example({ name, @@ -106,8 +106,8 @@ export function Example({ /** * The listing of a folder Ingestron generates for an example, such as - * `build/dev/adf`: folders first, then files. - * @example + * `build/adf`: folders first, then files. + * @example */ export function BuildTree({ name, folder }: { name: string; folder: string }) { return ( diff --git a/components/reference/built-in-templates.tsx b/components/reference/built-in-templates.tsx index d7c72ea..64b71bf 100644 --- a/components/reference/built-in-templates.tsx +++ b/components/reference/built-in-templates.tsx @@ -2,7 +2,7 @@ import { builtInTemplates } from "@ingestron/cli"; import { TableFrame } from "@/components/table-frame"; -/** One row per built-in template: its name, what it sets up, and what it asks. */ +/** One row per built-in template: its name and what it sets up. `init` asks none of them anything. */ export function BuiltInTemplates() { return ( @@ -11,7 +11,6 @@ export function BuiltInTemplates() { Template Sets up - Asks @@ -23,15 +22,6 @@ export function BuiltInTemplates() { {template.title}. {template.summary} - - {template.questions.length === 0 - ? "nothing" - : template.questions.map((question) => ( -
- {question.value}: {question.ask} -
- ))} - ))} diff --git a/components/reference/versions.tsx b/components/reference/versions.tsx index 9263396..16cad87 100644 --- a/components/reference/versions.tsx +++ b/components/reference/versions.tsx @@ -89,7 +89,7 @@ export function PlatformChecks() { const rows: [string, string][] = [ [ "databricks", - `The bundle schema of Databricks CLI ${found(databricks, /Bundle fields from Databricks CLI (v[\d.]+)/, "the Databricks CLI version")} (\`databricks bundle schema\`) and the Databricks documentation, accessed ${found(databricks, /## Sources \(accessed ([\d-]+)\)/, "the Databricks access date")}`, + `The bundle schema of Databricks CLI ${found(databricks, /Bundle fields\s+from Databricks CLI (v[\d.]+)/, "the Databricks CLI version")} (\`databricks bundle schema\`) and the Databricks documentation, accessed ${found(databricks, /## Sources\n[\s\S]*?\(accessed ([\d-]+)\)/, "the Databricks access date")}`, ], [ "adf", diff --git a/content/docs/(reference)/reference/cli.mdx b/content/docs/(reference)/reference/cli.mdx index 9479ebf..a9c5f4b 100644 --- a/content/docs/(reference)/reference/cli.mdx +++ b/content/docs/(reference)/reference/cli.mdx @@ -3,8 +3,8 @@ title: Command line description: The ingestron command's commands, options, output and exit codes. --- -The `ingestron` command reads a project folder and writes each environment's -files. [Install it](/docs/start/install) first. `ingestron init` creates a +The `ingestron` command reads a project folder and writes each platform's +files, for every environment. [Install it](/docs/start/install) first. `ingestron init` creates a project; see [Create a project](/docs/how-to/start-from-a-template). Then `ingestron plan` and `ingestron apply` are the everyday loop; see [Plan, apply and review](/docs/how-to/plan-and-generate). In CI, run @@ -17,7 +17,9 @@ project; see [Create a project](/docs/how-to/start-from-a-template). Then Run the command from the folder that holds `project.yaml`. It reads the project there and writes to `build` there. Four options adjust the run: -- `--env dev` limits the work to one environment; repeat it for more. +- `--env dev` limits `compare` and `request` to one environment; repeat it + for more. `plan`, `apply` and `check` always work on every environment + together, and refuse it. - `--definitions ` reads a project that is not the current folder. - `--out ` writes somewhere other than `build`. - `--no-diff` lists changed files without their line differences. @@ -36,18 +38,17 @@ ran it from, not in `shop`. ## Output -Each environment is written to its own folder under `--out`, such as -`build/dev/`. When a standard uses several packs, each pack writes to a -folder inside it, such as `build/dev/databricks/`. See -[Generated files and ownership](/docs/concepts/generated-files). +`plan` and `apply` print one block for each output folder under `--out`, such +as `databricks → build/databricks (dev, prod)`; see +[Where the files go](/docs/concepts/generated-files#where-the-files-go). ## Exit codes -| Code | Means | -| ---- | ----------------------------------------------------------------------------- | -| `0` | Success. | -| `1` | Problems in the definitions, or output that is not up to date or in conflict. | -| `2` | A mistake in the command, such as an unknown option. | +| Code | Means | +| ---- | ------------------------------------------------------------------------------------------------------------------------------- | +| `0` | Success. Warnings, such as a pack's notice, do not change it. | +| `1` | Problems in the definitions, or output that is not up to date or in conflict. `check` also fails while a value is `REPLACE_ME`. | +| `2` | A mistake in the command, such as an unknown option or `--env` given to `plan`, or a pack that cannot be loaded (`PACK_LOAD`). | ## Editor schemas @@ -58,8 +59,10 @@ folder inside it, such as `build/dev/databricks/`. See ## Use the command from code The CLI package is also a library: `run` takes the same arguments as the -command, and `projectSources`, `projectRoutes` and `projectRecipe` return the -sources, routes and questions that `init` and **Ingestron: New project** offer. +command, `generatedOutputs` gives every output folder's files as `plan` would +write them, and `projectSources`, `projectRoutes` and `projectRecipe` return +the sources, routes and recipes that `init` and **Ingestron: New project** +offer. The [`@ingestron/cli` README](https://github.com/ingestron/ingestron/tree/main/packages/cli#project-builder-api) describes them. diff --git a/content/docs/(reference)/reference/errors.mdx b/content/docs/(reference)/reference/errors.mdx index 3ed4d5a..88f88d0 100644 --- a/content/docs/(reference)/reference/errors.mdx +++ b/content/docs/(reference)/reference/errors.mdx @@ -4,7 +4,15 @@ description: What each problem Ingestron reports means, and how to fix it. --- Every problem is reported with the file, line and column of the value that -caused it, and Ingestron reports all problems in one run. +caused it, and Ingestron reports all problems in one run. A problem about the +whole project, such as a pack's notice, is reported as `(project)`. + +A warning, such as a value still [`REPLACE_ME`](#REPLACE_ME), does not stop +`plan` or `apply`; `check` reports a value still `REPLACE_ME` as an error. + +A pack that cannot be loaded stops every command before it reads the +definitions: the command prints [`PACK_LOAD`](#PACK_LOAD) alone and exits with +`2`, as it does for a usage mistake. ## Problems in the definitions diff --git a/content/docs/(reference)/reference/packs/adf.mdx b/content/docs/(reference)/reference/packs/adf.mdx index 0dc4cf5..6bb235c 100644 --- a/content/docs/(reference)/reference/packs/adf.mdx +++ b/content/docs/(reference)/reference/packs/adf.mdx @@ -3,15 +3,15 @@ title: Azure Data Factory pack description: The native ingestion methods, with their settings and defaults. --- -`adf@1` copies tables and files from supported native sources into +`adf@2` copies tables and files from supported native sources into landing storage. From there, the [Databricks pack](/docs/reference/packs/databricks) reads them into tables, the [Synapse serverless pack](/docs/reference/packs/synapse) shows them as views, or the [DuckDB pack](/docs/reference/packs/duckdb) builds tables from them. Synapse and DuckDB read the tables ADF copies, not files it copies as they are, and Synapse cannot read the JSON Microsoft 365 lands. List -the other pack first, as in `packs: [databricks@1, adf@1]`; see +the other pack first, as in `packs: [databricks@2, adf@2]`; see [Combine packs](/docs/concepts/standards-and-packs#combine-packs-when-their-actions-fit). -On its own, `packs: [adf@1]` with `store: { default: none }` ends each table at +On its own, `packs: [adf@2]` with `store: { default: none }` ends each table at the landed files; see [Land files only](/docs/platforms/landing-only). The settings below come from the pack itself. @@ -21,21 +21,27 @@ For a task walkthrough, use [Azure Data Factory](/docs/platforms/adf), [ADF resource names](/docs/how-to/name-things-your-way#adf-resource-names). The reference below keeps the method and setting anchors used by editor help. -After generation and review, [publish the ADF resources](/docs/platforms/adf#publish) -through the factory's deployment process. Generation itself does not publish. +The pack writes one factory for every environment, with a parameter file per +environment; see [What Ingestron writes](/docs/platforms/adf#what-you-get). +After generation and review, +[deploy it as an ARM template](/docs/platforms/adf-deploy#deploy). Generation itself +deploys nothing. ## Platform configuration A trigger's state comes from the copy method's own `paused` setting first, -then from the platform's `schedule.paused`. Where neither says, the ADF pack -uses this default, so development copies do not run on their own: +then from the platform's `schedule.paused`. Where neither says, a trigger +starts only in an environment named `prod` or `production`, so development +copies do not run on their own: -A stopped trigger is generated with `"runtimeState": "Stopped"`; a started one -runs every window since its `start` once it is published. +The factory's trigger holds the first environment's state, such as +`"runtimeState": "Stopped"`, and where the state differs, each environment's +parameter file sets it. A started trigger runs every window since its `start` +once it is deployed. ## Contract fields diff --git a/content/docs/(reference)/reference/packs/databricks.mdx b/content/docs/(reference)/reference/packs/databricks.mdx index c6e178b..8688cc9 100644 --- a/content/docs/(reference)/reference/packs/databricks.mdx +++ b/content/docs/(reference)/reference/packs/databricks.mdx @@ -3,20 +3,23 @@ title: Databricks pack description: Every method of the databricks pack, with its settings and defaults. --- -`databricks@1` writes one Databricks bundle per environment: Lakeflow -pipelines with SQL and Python files, Lakeflow Connect ingestion, and a -refresh job when the platform has a schedule or a task runs after the refresh. +`databricks@2` writes one Databricks bundle for every environment, with a +target per environment: Lakeflow pipelines with SQL and Python files, +Lakeflow Connect ingestion, and a refresh job when the platform has a schedule +or a task runs after the refresh. See +[What Ingestron writes](/docs/platforms/databricks#what-you-get). The settings below come from the pack itself. Before deployment, [plan and review](/docs/how-to/plan-and-generate) the files, -then [deploy the generated bundle](/docs/platforms/databricks#deploy). +then [deploy the generated bundle](/docs/platforms/databricks-deploy#deploy). ## Platform configuration Without a platform document, or where it leaves a setting out, the Databricks -pack uses these defaults. Without `run_as`, the pipelines and jobs run as the +pack uses these defaults. Each target of the bundle gets its environment's +`mode`, workspace, `run_as` and `permissions`. Without `run_as`, the pipelines and jobs run as the identity that deploys the bundle, and without `schedule`, nothing runs on a schedule. @@ -32,8 +35,9 @@ following capability reference describes each switch and its prerequisites. Some features change shared Unity Catalog objects, so the platform turns each on with a `manage` switch. While a switch is off, a definition or contract that uses the feature gets a `CAPABILITY_OFF` warning with a quick fix that turns it -on. When a switch is on, the bundle has a `prerequisites.sql` that lists the -grants to run once as an administrator. +on. When a switch is on, the bundle has a `prerequisites/.sql` for +each environment, which lists the grants to run once there as an +administrator; it is not deployed. Tags, column comments and table checks run in an `apply_contract` task after the refresh, on serverless jobs compute: Lakeflow pipelines cannot set tags or diff --git a/content/docs/(reference)/reference/packs/duckdb.mdx b/content/docs/(reference)/reference/packs/duckdb.mdx index 18ddbe5..be4c0cc 100644 --- a/content/docs/(reference)/reference/packs/duckdb.mdx +++ b/content/docs/(reference)/reference/packs/duckdb.mdx @@ -3,17 +3,17 @@ title: DuckDB pack description: DuckDB tables, transforms and local runs, with their settings and defaults. --- -`duckdb@1` loads files into tables in a DuckDB database, runs transforms, and +`duckdb@2` loads files into tables in a DuckDB database, runs transforms, and publishes each table as Parquet, with SQL that DuckDB runs on a laptop or -wherever DuckDB runs. List it first, as in `packs: [duckdb@1, adf@1]` when +wherever DuckDB runs. List it first, as in `packs: [duckdb@2, adf@2]` when ADF lands the files; see [Combine packs](/docs/concepts/standards-and-packs#combine-packs-when-their-actions-fit). The settings below come from the pack itself. For a task walkthrough, use [DuckDB](/docs/platforms/duckdb); the modes are explained in [Keep a table current](/docs/how-to/keep-a-table-current#on-duckdb). The pack writes `run.sql`, every table in -order, and `tables/`, each table's SQL alone, for each environment. Run -`run.sql` from the project folder with the DuckDB command line; generation -itself runs nothing. +order, and `tables/`, each table's SQL alone, in a folder for each +environment, `build/duckdb//`. Run `run.sql` from the project +folder with the DuckDB command line; generation itself runs nothing. diff --git a/content/docs/(reference)/reference/packs/index.mdx b/content/docs/(reference)/reference/packs/index.mdx index a89e150..cbf0be9 100644 --- a/content/docs/(reference)/reference/packs/index.mdx +++ b/content/docs/(reference)/reference/packs/index.mdx @@ -9,12 +9,12 @@ defaults, rendered from the pack itself. ## Available packs -| Pack | Responsibilities | Generated output | -| ----------------------------------------------------------------- | -------------------------------------------------------- | ------------------------------------------------------------------ | -| [Databricks](/docs/reference/packs/databricks) (`databricks@1`) | Supported direct ingestion, transforms and storage | Bundles, pipelines, SQL or Python files, and jobs where configured | -| [Azure Data Factory](/docs/reference/packs/adf) (`adf@1`) | Native copying from supported sources to landing storage | Linked services, datasets, pipelines and triggers | -| [Synapse serverless](/docs/reference/packs/synapse) (`synapse@1`) | SQL views over landed files and transforms | `views.sql` for the serverless database | -| [DuckDB](/docs/reference/packs/duckdb) (`duckdb@1`) | Tables, transforms and local runs, published as Parquet | `run.sql` and each table's SQL | +| Pack | Responsibilities | Generated output, for every environment | +| ----------------------------------------------------------------- | -------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| [Databricks](/docs/reference/packs/databricks) (`databricks@2`) | Supported direct ingestion, transforms and storage | [One bundle with a target per environment](/docs/platforms/databricks#what-you-get), in `build/databricks/` | +| [Azure Data Factory](/docs/reference/packs/adf) (`adf@2`) | Native copying from supported sources to landing storage | [One factory with a parameter file per environment](/docs/platforms/adf#what-you-get), in `build/adf/` | +| [Synapse serverless](/docs/reference/packs/synapse) (`synapse@2`) | SQL views over landed files and transforms | [`views.sql` with a script per environment](/docs/platforms/synapse#what-you-get), in `build/synapse/` | +| [DuckDB](/docs/reference/packs/duckdb) (`duckdb@2`) | Tables, transforms and local runs, published as Parquet | [`run.sql` and each table's SQL](/docs/platforms/duckdb#what-you-get), in `build/duckdb//` | Connector-specific access and ingestion requirements are in [Connectors](/docs/connectors). diff --git a/content/docs/(reference)/reference/packs/synapse.mdx b/content/docs/(reference)/reference/packs/synapse.mdx index 519ca63..676f092 100644 --- a/content/docs/(reference)/reference/packs/synapse.mdx +++ b/content/docs/(reference)/reference/packs/synapse.mdx @@ -3,15 +3,18 @@ title: Synapse serverless pack description: Views in Azure Synapse serverless SQL, with their settings and defaults. --- -`synapse@1` shows tables to reporting tools as views in Azure Synapse serverless +`synapse@2` shows tables to reporting tools as views in Azure Synapse serverless SQL: over the files another pack lands, over a transform's query, or over the files a store pack keeps. For views only, list it first: -`packs: [synapse@1, adf@1]`; see [Combine packs](/docs/concepts/standards-and-packs#combine-packs-when-their-actions-fit). The settings below come from the +`packs: [synapse@2, adf@2]`; see [Combine packs](/docs/concepts/standards-and-packs#combine-packs-when-their-actions-fit). The settings below come from the pack itself. For a task walkthrough, use [Synapse serverless views](/docs/platforms/synapse). -The pack writes `views.sql` for each environment; run it against the serverless -database when the definitions change. Generation itself creates nothing in Azure. +The pack writes one `views.sql` for every environment, with +`environments/.sql` setting each environment's values; run that +script with sqlcmd when the definitions change. See +[What Ingestron writes](/docs/platforms/synapse#what-you-get). Generation itself +creates nothing in Azure. diff --git a/content/docs/(reference)/reference/templates.mdx b/content/docs/(reference)/reference/templates.mdx index 65c95db..a855f3f 100644 --- a/content/docs/(reference)/reference/templates.mdx +++ b/content/docs/(reference)/reference/templates.mdx @@ -4,15 +4,15 @@ description: The built-in starters and how to create a project from one, the tem --- A template is a project folder with a `template.yaml`. `ingestron init` copies -the folder into a new project and asks the questions the file lists. A starter -is a template that ships with Ingestron: a complete project for one common -situation. See [Share templates and standards](/docs/how-to/share-a-template) +the folder into a new project, writes its `README.md`, and asks nothing. A +starter is a template that ships with Ingestron: a complete project for one +common situation, with `REPLACE_ME` where you write your own values. See [Share templates and standards](/docs/how-to/share-a-template) to make a template of your own. ## Create a project from a starter [#create-a-project-from-a-starter] -In the CLI, name the starter with `--template`. Without `--yes`, `init` asks the -starter's questions, which the table below lists: +In the CLI, name the starter with `--template`. The new `README.md` lists each +value still `REPLACE_ME`; `--set` writes them as the project is created: ```sh ingestron init my-project --template retail --yes @@ -35,15 +35,18 @@ source instead of a starter, see | ----------- | -------- | ------------------------------------------------------------------------ | | `title` | yes | The template's human-readable title | | `summary` | yes | One sentence on what the template sets up | -| `questions` | no | What `init` asks; each answer becomes `values.` in `project.yaml` | +| `questions` | no | Values for `project.yaml`, each written to `values.`; never asked | Each question: | Field | Meaning | | --------- | --------------------------------------------------------------------- | | `value` | The key in `project.yaml` `values`: lowercase letters, digits and `_` | -| `ask` | The question, as the user sees it | -| `default` | The answer used with `--yes` or an empty answer | +| `ask` | What the value is | +| `default` | The value written, unless `--set =` gives another | + +The built-in starters have no questions: they write `REPLACE_ME`, with a +comment, where a value is yours to give. Any other field is an error, so a misspelt one is found at once. Your editor checks `template.yaml` too after `ingestron schemas`. @@ -58,9 +61,13 @@ folder or Git URL can hold either name. ## Download a starter [#download-a-starter] -Each download holds a starter's synthetic definitions, without generated -files: run `ingestron apply` in its folder to write them. The platforms and -connectors listed are read from the resolved starter. +Downloads come without generated files: run `ingestron apply` in the folder +to write them. Each is what `ingestron init --template --yes` +writes: the starter's definitions, with `REPLACE_ME` where a value is yours +to give, and the `README.md` that lists each one and where to find it. `plan` +and `apply` warn about each `REPLACE_ME`, and `check` fails until each is +written. The platforms and connectors listed are read from the resolved +starter. diff --git a/content/docs/(reference)/reference/versions.mdx b/content/docs/(reference)/reference/versions.mdx index facef21..dee9726 100644 --- a/content/docs/(reference)/reference/versions.mdx +++ b/content/docs/(reference)/reference/versions.mdx @@ -23,3 +23,11 @@ documentation. What each was checked against: Connector maturity, such as Public Preview or Beta, is shown on each [connector page](/docs/connectors) with the date it was checked. + +## Deploy tools + +The [deployment templates](/docs/how-to/deploy-through-environments) pin the +tools their stages run: `@microsoft/azure-data-factory-utilities` 1.0.3, the +Databricks CLI v1.23.0, and sqlcmd (Go) v1.10.0. Their check stage installs +`ingestron@2.1.0`, and each GitHub action is pinned to a commit, with its +version beside it. diff --git a/content/docs/(reference)/tools/vscode.mdx b/content/docs/(reference)/tools/vscode.mdx index 170d6df..8f5ca9c 100644 --- a/content/docs/(reference)/tools/vscode.mdx +++ b/content/docs/(reference)/tools/vscode.mdx @@ -5,22 +5,27 @@ description: Write Ingestron projects in VS Code with problems as you type, navi The Ingestron extension works on any folder with a `project.yaml`. It uses the same definition and generation packages as the `ingestron` command. Editor -analysis uses the open files and selected environment. +analysis uses the open files and the selected environment; Plan, Apply and +Check work on every environment together, as the command does. ## Install Follow [Install the command and the extension](/docs/start/install) to build the extension from [ingestron/vscode](https://github.com/ingestron/vscode) and install it. -Open a folder that holds an Ingestron project. The environment the extension -uses appears in the status bar, such as `Ingestron: dev`; click it to choose -another. VS Code remembers the choice for each project in the workspace. +Open a folder that holds an Ingestron project. The selected environment +appears in the status bar, such as `Ingestron: dev`; click it to choose +another. It decides what the editor shows: the problems, the previews of +generated files, what each line becomes, and the platform request. Plan, +Apply and Check do not use it: every environment is generated together, into +one set of files per platform. VS Code remembers the choice for each project +in the workspace. ## Get started @@ -36,13 +41,18 @@ Definition problems appear in the Problems panel at their file and line, while you type and before you save: a broken link, a setting the standard fixes, a method that cannot read a source, a name that is not valid. Problems are for the environment in the status bar; click it, or run -**Ingestron: Select environment**, to choose another. +**Ingestron: Select environment**, to choose another. Problems that generating +finds, such as a value supplied when deploying that cannot be a name +(`DEPLOY_VALUE`), show too, and a pack's notice (`PACK_NOTICE`) is a warning +on the first line of `project.yaml` in every environment. A definition that uses `${envvar:NAME}` reads the variable from `.env.` and `.env` in the project folder, and from the environment VS Code was started with. A variable exported later in a terminal is not seen here; put it in `.env`. When the variable is set nowhere, the editor shows a blue information -mark, not an error, that names the file to add it to, such as +mark, not an error. For Databricks, Data Factory and Synapse, which take the +value when deploying, it names what the deploying pipeline sets, such as +`BUNDLE_VAR_landing_path`; for DuckDB it names the file to add it to, such as `.env.dev`. The rest of the project is still checked, and a value that depends on the missing one is not judged until it is set. A line in an `.env` file that Ingestron cannot read is a warning (`ENV_FILE_LINE`) at that line. See @@ -51,8 +61,8 @@ Ingestron cannot read is a warning (`ENV_FILE_LINE`) at that line. See A definitions file that no import in `project.yaml` matches, such as a new @@ -233,9 +243,13 @@ The **Ingestron** tab in the secondary side bar, on the right, opens with a **Project** section. For a workspace with one project it lists: - **Environments**, each with a circle, or a check and **selected** on the one - commands use now. Click an environment to select it, as in the status bar. + problems, previews and the platform request show. Click an environment to + select it, as in the status bar. - **Check**, **Plan**, **Apply**, **Compare environments**, **Prepare platform request** and **Show generated files**. Click one to run it for the project. + Check, Plan and Apply work on every environment together; Prepare platform + request and Show generated files use the selected one. Each item's tooltip + says which. With several projects, each has a row, named by its `id` and showing its folder in the workspace, that holds the same lists. Every row starts folded except @@ -278,15 +292,24 @@ Explorer. ## See what will be generated - **Ingestron: Show generated files** opens what would be generated for the - table at the cursor, in a read-only preview, without writing anything. It - updates as you edit the definitions. + table at the cursor, in the selected environment, in a read-only preview, + without writing anything. It updates as you edit the definitions. Its paths + are those `ingestron apply` writes, such as + `build/adf/factory/pipeline/…`; a folder shared by every environment, such + as a Databricks bundle or a factory with each environment's parameter file, + shows in each environment that uses it. - **Ingestron: Compare with build** shows the difference between that and the file in `build/`: what the next apply changes. -Plan, Apply, Check and the other commands work from a preview too. They act -on the preview's project and on the environment it shows, even when another -environment is selected for the project: Apply from a `dev` preview applies -`dev`. While a preview is active, the status bar shows its environment. +While plan would stop, nothing is generated, and the preview says why, such +as `Nothing is generated while plan would stop: prod has problems. Run Plan +to see them.`: problems in any environment stop every one, because every +environment is generated together. + +Plan, Apply, Check and the other commands work from a preview too, on the +preview's project; Apply from a `dev` preview writes every environment. +Prepare platform request uses the environment the preview shows, and while a +preview is active, the status bar shows that environment. .sql`. Nothing is sent or saved until you do so. See [Ask the platform team](/docs/how-to/ask-the-platform-team). ## Plan, apply and check **Ingestron: Plan**, **Ingestron: Apply** and **Ingestron: Check** run the -command for the project and environment, after saving open files. The output -is in the Ingestron output panel. +command for the project, after saving open files, for every environment +together: one block for each output folder, such as +`databricks → build/databricks (dev, prod)`. The output is in the Ingestron +output panel. You can start them from where you are working, as well as from the Command -Palette. Each acts on the project you started it from, for the environment -chosen for that project: +Palette. Each acts on the project you started it from: - **Right-click** a project folder, or a YAML file in a project, in the Explorer for **Ingestron: Check**, **Plan**, **Apply**, **Compare @@ -331,7 +355,7 @@ chosen for that project: @@ -345,28 +369,33 @@ chosen for that project: ## Generate on save To see the generated files change as you edit, click **Auto-apply: off** in the status bar, or run **Ingestron: Turn auto-apply on or off**. Turning it on -applies the selected environment straight away. While it is on, the switch -names that environment, such as **Auto-apply: on (dev)**, and saving a YAML, -SQL or Python file in the project applies it again. The Ingestron output panel -logs each run with the files it changed. - -It applies nothing while the definitions have problems, and nothing when a -generated file was changed by hand: the log says so, as `apply` would. It starts -off, and VS Code remembers the choice for each workspace. +applies every environment straight away, and the switch reads +**Auto-apply: on**. While it is on, saving a YAML, SQL or Python file in the +project applies again. The Ingestron output panel logs each run with a line +for each output folder, such as `databricks (dev, prod): up to date`, with the +files it changed below it, and a line for each folder that is no longer +generated. + +It applies nothing while the definitions have problems in any environment, +and says where to see them: for another environment, such as +`not applied: prod has problems (select prod to see them in the Problems panel)`. +A folder with a file changed by hand is not applied either, and its line says +so, as `apply` would. It starts off, and VS Code remembers the choice for each +workspace. ## Start a project @@ -374,23 +403,24 @@ off, and VS Code remembers the choice for each workspace. **Ingestron: New project** starts from a connection or from a starter. - **From a connection** asks for a source category, connection, network - reachability and supported route, then the project name, environments, - standard and configuration. You can search across all sources or within a - category. See [Create a project](/docs/how-to/start-from-a-template). + reachability and supported route. You can search across all sources or + within a category. - **From a starter** offers the ready-made projects, such as **DuckDB on a laptop**, which [runs locally with DuckDB](/docs/platforms/duckdb). - It asks for the project name, environments, your organisation's standard if - you have one (otherwise the starter's own is kept), and the starter's own - questions. -It asks for the destination folder last. When the folder is outside the -workspace, it offers to open it or add it to the workspace. +Nothing else is asked. The destination folder comes last, and its name is the +project's id; the environments are `dev` and `prod`, or the starter's, and the +standard is the route's or the starter's own. The new `README.md` opens in a +preview, and a message says how many values are still `REPLACE_ME`: the +README lists each with where to find it. When the folder is outside the +workspace, the message offers to open it or add it to the workspace. See +[Create a project](/docs/how-to/start-from-a-template). ## Check files with the YAML extension [#check-files-with-the-yaml-extension] diff --git a/content/docs/(reference)/troubleshooting.mdx b/content/docs/(reference)/troubleshooting.mdx index 96cbb87..7c94195 100644 --- a/content/docs/(reference)/troubleshooting.mdx +++ b/content/docs/(reference)/troubleshooting.mdx @@ -44,6 +44,36 @@ Node.js installed for the whole system. Install Node.js with a version manager instead, or set npm's prefix to a folder you own, such as `npm config set prefix ~/.npm-global`, and add its `bin` folder to your `PATH`. +### Plan, apply or check refuse `--env` [#plan-refuses-env] + +**Symptom:** the command prints its usage and exits with `2`: + +```text +Every environment is generated together, so plan does not take --env. Use compare for two environments, or request for one. +``` + +**Cause:** each platform's files hold every environment, so `plan`, `apply` +and `check` always work on all of them. Scripts and CI written for version 1 +passed `--env`. + +**Fix:** remove `--env` from `plan`, `apply` and `check`. To look at one +environment, use `ingestron compare --env dev --env prod` or +`ingestron request --env prod`. + +### A pack cannot be loaded + +**Symptom:** every command prints one +[`PACK_LOAD`](/docs/reference/errors#PACK_LOAD) error, writes nothing and +exits with `2`. + +**Cause:** a pack that came with the command has a mistake of its own, such as +a field it reads that the definitions do not have. Nothing in your project +causes it. + +**Fix:** install a version of Ingestron, or of the VS Code extension, that +fixes the pack, as [Upgrade Ingestron and its packs](/docs/how-to/upgrade) +describes. + ## Generated files ### Plan reports a conflict @@ -52,7 +82,7 @@ instead, or set npm's prefix to a folder you own, such as with `1`: ```text -conflict run.sql +conflict dev/run.sql Changed by hand since it was generated. Move the change into the definitions, or delete the file to generate it again. ``` @@ -80,23 +110,23 @@ could remove the lock. **Fix:** wait for a run that is still going. If none is, delete `.generated.lock` from the output folder the plan names (the line before the -changes, such as `dev → build/dev`). Then follow the next entry, because the -folder may hold a mix of old and new files. +changes, such as `duckdb → build/duckdb (dev)`). Then follow the next entry, +because the folder may hold a mix of old and new files. ### An apply stopped part-way **Symptom:** one of these: -- With several environments, `apply` ends with a line such as - `Applied: dev, test. Not applied: prod.` and exits with `1`. +- With several platforms, `apply` ends with a line such as + `Applied: adf. Not applied: databricks.` and exits with `1`. - A write fails, and `apply` ends with - `Could not write "tables/silver.customer_totals.sql": . Nothing was changed.` + `Could not write "dev/tables/silver.customer_totals.sql": . Nothing was changed.` - The process was stopped, such as a closed terminal or a cancelled CI job. The lock stays, and `ingestron check` may also print a file ending in `.tmp`: - `dev: not generated tables/silver.customer_totals.sql..tmp`. + `duckdb: not generated dev/tables/silver.customer_totals.sql..tmp`. -**Cause:** `apply` writes one output folder at a time. An environment whose plan -has a conflict is left as it was while the others are written. When a write +**Cause:** `apply` writes one output folder at a time. A folder whose plan has +a conflict is left as it was while the others are written. When a write fails, Ingestron puts back the files it had already changed in that folder. If it cannot put one back, it adds `Some files could not be put back.` A process that is stopped cannot put anything back: the folder holds a mix of old and new @@ -105,8 +135,9 @@ half-written. **Fix:** -- `Not applied: prod`: run `ingestron plan --env prod` and resolve what it - reports, such as a [conflict](#plan-reports-a-conflict), then apply again. +- `Not applied: databricks`: run `ingestron plan` and resolve what it reports + for that folder, such as a [conflict](#plan-reports-a-conflict), then apply + again. - `Nothing was changed.`: fix what the reason names, such as a folder you cannot write to or a full disk, and apply again. - A stopped process, or `Some files could not be put back.`: delete @@ -130,8 +161,9 @@ regenerated and committed, or a generated file was edited. `: no longer generated`, and exits with `1`. **Cause:** someone added a file inside an output folder, or a folder that -was generated before (for an environment you removed, or before you added a -second pack) is still in `build/`. +was generated before is still in `build/`: a platform's folder after its last +table went, or the folder per environment, such as `build/dev/`, that version +1 wrote. See [Upgrade from 1.x to 2.0](/docs/how-to/upgrade#upgrade-to-2). **Fix:** move the extra file out of `build/`. Delete the old folder once nothing deploys from it. See [Check generated files in CI](/docs/how-to/check-in-ci). @@ -198,6 +230,67 @@ patterns give two tables the same full name. **Fix:** rename one, or include `{{ flow }}` or `{{ source }}` in the table pattern. See [Name tables and resources](/docs/how-to/name-things-your-way). +### The standard names an older pack [#pack-version] + +**Symptom:** after upgrading to ingestron 2, plan stops with +[`PACK_VERSION`](/docs/reference/errors#PACK_VERSION), once for every +environment: + +```text +standard.yaml:4:9 error PACK_VERSION + This standard is for "duckdb@1", but the pack is duckdb@2. + Change it to duckdb@2; the upgrade notes say what changes in the generated files. + Fix: Change to "duckdb@2". +``` + +**Cause:** every pack in ingestron 2 is version 2, and the standard still +names version 1. + +**Fix:** change each pack in the standard's `packs` to `@2`, or apply the fix +in VS Code, then follow +[Upgrade from 1.x to 2.0](/docs/how-to/upgrade#upgrade-to-2). + +### Environments differ in more than values [#environments-differ] + +**Symptom:** a pack's plan stops with +[`ENVIRONMENTS_DIFFER`](/docs/reference/errors#ENVIRONMENTS_DIFFER), such as: + +```text +databricks: the definitions have problems +project.yaml:1:1 error ENVIRONMENTS_DIFFER + The pipeline is named "retail_dev" in dev and "retail_prod" in prod; resource keys and folders are named after it, so it must be the same in every environment. + Environments may differ only in values, such as names, paths and hosts; make the rest the same in every environment. +``` + +**Cause:** one bundle, factory or Synapse script serves every environment, so +environments may differ only in values. A pipeline, job or ADF resource name +that differs, a table built in some environments only, or code that differs in +more than a value cannot be written once. + +**Fix:** make it the same in every environment: here, remove +`{{ environment }}` from the standard's `naming.pipeline`. Catalogs, schemas, +paths and hosts may still differ. See +[Where the files go](/docs/concepts/generated-files#where-the-files-go). + +### A value supplied when deploying is refused [#deploy-value] + +**Symptom:** plan stops with +[`DEPLOY_VALUE`](/docs/reference/errors#DEPLOY_VALUE), such as: + +```text +flows/orders.yaml:4:9 error DEPLOY_VALUE + "${envvar:SOURCE}" is supplied only when deploying, so it cannot decide what is generated or what it is called. + Write this value in project.yaml, per environment; keep ${envvar:…} for values the deploying pipeline supplies, such as hosts, paths and accounts. +``` + +**Cause:** for Databricks, Data Factory and Synapse, `${envvar:…}` is a value +the deploying pipeline supplies. It cannot be a flow's `source`, a contract, +a name, or, for Data Factory and Synapse, part of a longer value. + +**Fix:** write the value in `project.yaml`, with each environment's value +under `environments`. See +[Supplied when deploying](/docs/concepts/yaml-references#supplied-when-deploying). + ### A new file is not generated **Symptom:** you added a flow, transform or other definition and `plan` lists @@ -242,9 +335,11 @@ SAMPLES_DIR isn't set; add it to .env.dev or set it before running Ingestron. **Cause:** the variable is unset or empty in the environment of the process that runs Ingestron, in `.env.` and in `.env`. A document that -uses it is not loaded, so what links to that document is reported too. The -editor keeps checking the rest of the project and does not judge a value that -depends on the missing one. +uses it is not loaded, so what links to that document is reported too, in +VS Code as well. This applies where a value is read when generating, as for +DuckDB; for Databricks, Data Factory and Synapse, the value is supplied when +deploying, and plan does not need it (see +[Supplied when deploying](/docs/concepts/yaml-references#supplied-when-deploying)). **Fix:** add `SAMPLES_DIR=./samples` to `.env`, or to `.env.dev` for `dev` alone, or set the variable before you run the command. In CI, set it in the @@ -289,16 +384,11 @@ the Command Palette. See [Install the command and the extension](/docs/start/ins **Symptom:** after `ingestron apply` in a terminal, **Ingestron: Plan** lists changes, or the other way round. -**Cause:** a common cause is a different version. The extension carries its -own copy of the command, which need not be the one you run. Another is a -different environment: the editor plans the one in the status bar, such as -`Ingestron: dev`, while `ingestron plan` without `--env` plans every -environment. - -**Check:** plan the same environment in both, such as `ingestron plan --env dev` -and `dev` in the status bar. +**Cause:** a different version. The extension carries its own copy of the +command, which need not be the one you run. Both plan every environment, so +the environment in the status bar makes no difference. -**Fix:** if they still differ, run the same version in both, as +**Fix:** run the same version in both, as [Upgrade Ingestron and its packs](/docs/how-to/upgrade) describes, and reload the window after installing the extension again. @@ -309,7 +399,7 @@ the window after installing the extension again. **Symptom:** the plan stops before any file is written: ```text -flows/postgresql.yaml:14:5 error METHOD_CHECK +flows/postgresql.yaml:15:5 error METHOD_CHECK "adf-table" for table "postgresql_records": the connection needs "settings.server" as non-empty text. ``` @@ -339,21 +429,43 @@ request** in VS Code. Its **ADF prerequisites** section lists, with this project's values, the integration runtime, the Key Vault, the landing folder and the source's permissions that must exist. Give the list to the team that creates them. See -[Platform prerequisites](/docs/platforms/adf#platform-prerequisites). +[Before you deploy](/docs/platforms/adf-deploy#prerequisites). + +### Every plan and check prints a notice about a runtime [#runtime-notice] + +**Symptom:** `plan`, `apply` and `check` print a warning before the folders, +and still succeed: + +```text +adf: warnings +(project) warning PACK_NOTICE + Integration runtime `shir-erp` (self-hosted): the deployment writes it by name and type, as Data Factory Studio does; where it creates it, register a node, with network access and any required source drivers, before the copies run. +``` + +**Cause:** the factory holds each integration runtime its linked services +use, and an ARM deployment sets a runtime's whole resource. The +[`PACK_NOTICE`](/docs/reference/errors#PACK_NOTICE) says what a deployment +does to it. It is a warning, so it shows on every run, in CI too. + +**Fix:** nothing to change in the definitions. Before the first deployment to +a factory that already has the runtime, run `az deployment group what-if` with +the template and that environment's parameter file, and describe a linked or +Azure runtime with the method's `runtime` setting; see +[Runtime and Key Vault](/docs/platforms/adf#runtime-and-key-vault). ### Development triggers do not run **Symptom:** the pipelines are in the factory, but a `dev` copy never starts on its schedule. -**Cause:** outside production, a generated trigger is stopped +**Cause:** triggers start only in `prod` and `production` ([platform defaults](/docs/reference/packs/adf#platform-configuration)). The -trigger file says `"runtimeState": "Stopped"`, and the production one says -`"Started"`. +trigger file in `factory/trigger/` says `"runtimeState": "Stopped"`, and +`parameters/prod.json` sets the trigger's state to `"Started"`. **Fix:** start the trigger in Data Factory when you want development copies to run, or set `paused: false` for the copy method in the platform and generate -again. Check `start` first: a started trigger runs every window since `start`. See [Retry and pause](/docs/platforms/adf#retry-and-pause). +again, which starts it in every environment. Check `start` first: a started trigger runs every window since `start`. See [Retry and pause](/docs/platforms/adf-pipelines#retry-and-pause). ## Synapse serverless views @@ -390,10 +502,10 @@ platform, so the external data source cannot reach them. point the landing folder inside it. See [Settings that shape it](/docs/platforms/synapse#settings). -### Running views.sql fails the first time +### Running the views fails the first time [#running-viewssql-fails-the-first-time] -**Symptom:** `views.sql` fails in Synapse for a new environment, although -`apply` succeeded. +**Symptom:** `sqlcmd -i environments/.sql` fails in Synapse for a +new environment, although `apply` succeeded. **Cause:** Ingestron writes the views but creates nothing in Synapse. The serverless database, a master key, the credential, the external data source and @@ -402,7 +514,20 @@ the workspace's read access to the storage must exist first. **Fix:** run `ingestron request --env dev`. Its **Synapse serverless database**, **External data source** and **Report users** sections list each statement and permission, with this project's names. Ask the team that owns the workspace to -create them, then run `views.sql` again. +create them, then run the environment's script again. + +### sqlcmd cannot find views.sql + +**Symptom:** `sqlcmd -i environments/.sql` stops at once, saying +it cannot open `views.sql`, or Synapse Studio reports an error at the first +`$(…)` in `views.sql`. + +**Cause:** sqlcmd reads the `:r views.sql` line from the folder it starts in, +not from the script's folder. `views.sql` holds sqlcmd variables, which only +sqlcmd replaces, so Synapse Studio cannot run it. + +**Fix:** run sqlcmd from `build/synapse`, with the environment's script. See +[Deploy](/docs/platforms/synapse#deploy). ## DuckDB @@ -450,7 +575,7 @@ building publishes nothing. **Fix:** run it as the generated `README.md` says, from the project folder: ```sh -duckdb -bail ingestron.duckdb -c ".read build/dev/run.sql" +duckdb -bail ingestron.duckdb -c ".read build/duckdb/dev/run.sql" ``` ### Publishing fails after the tables are built @@ -543,7 +668,7 @@ A Lakeflow Connect pipeline refers to a Unity Catalog connection, which Ingestron does not create. **Fix:** when the platform sets `manage.schemas`, an administrator runs the -generated `prerequisites.sql` once before the first deploy: it grants +generated `prerequisites/.sql` once before the first deploy: it grants `USE CATALOG` and `CREATE SCHEMA` on each catalog to the identity the pipelines run as, and the same must be given to the identity that deploys when it differs. Run `ingestron request` for everything else the platform team @@ -554,12 +679,15 @@ provides. **Symptom:** the job and the pipelines are deployed, and nothing refreshes on its schedule in `dev`. -**Cause:** outside production, the job is created paused +**Cause:** schedules run only in `prod` and `production` ([platform defaults](/docs/reference/packs/databricks#platform-configuration)). -Its file says `pause_status: PAUSED`. +`dev` is in development mode, which pauses every schedule, and another +environment's target in `databricks.yml` sets +`presets: { trigger_pause_status: PAUSED }`. **Fix:** unpause it in Databricks, or set `paused: false` in the platform's -`schedule` and generate again. See +`schedule` and generate again: the job then says `pause_status: UNPAUSED` in +every environment. See [Refresh on a schedule](/docs/platforms/databricks#schedule). Every code is explained in [Errors](/docs/reference/errors). diff --git a/content/docs/concepts/environments-and-platform.mdx b/content/docs/concepts/environments-and-platform.mdx index e8e6691..100a715 100644 --- a/content/docs/concepts/environments-and-platform.mdx +++ b/content/docs/concepts/environments-and-platform.mdx @@ -5,7 +5,8 @@ description: Reuse one project's definitions with different values and shared in An environment gives the same project another set of values, such as `dev`, `test` and `prod`. It does not change the tools or create anything in the -cloud. Plan and generate each environment on its own or all together. +cloud. Every environment is generated together, into one set of files per +platform that a deployment pipeline promotes from one environment to the next. ## Values that differ @@ -43,14 +44,28 @@ specific to each platform, and not every platform uses each one. See and [ADF runtime and Key Vault](/docs/platforms/adf#runtime-and-key-vault) for their effects. -## Separate generated output +## One set of files for every environment [#one-set-of-files] -The default output is `build//`. With several packs, each pack -gets its own subfolder. Ingestron fills in your values when it generates, but -some files still hold secret references, expressions or parameters that the -platform fills in when it runs. Generating never reads a secret or tests the -deployed infrastructure. + -Use [Add an environment](/docs/how-to/add-an-environment) to create another environment, -and compare the resolved choices before deployment. [Generated files and ownership](/docs/concepts/generated-files) -explains how Ingestron protects each output folder. +Ingestron fills in each environment's values when it generates, then writes +each platform's files once for every environment: a value that differs between +environments becomes a parameter of the bundle, the factory or the Synapse +script, set for each environment, and everything else is written once. So +environments may differ only in values; a table or resource present in some +environments only is +[`ENVIRONMENTS_DIFFER`](/docs/reference/errors#ENVIRONMENTS_DIFFER). +[Generated files and ownership](/docs/concepts/generated-files#where-the-files-go) +shows where each platform's files go. + +Some values are not known until deployment, such as a storage account each +stage of your pipeline supplies. Write them as `${envvar:NAME}`: for +Databricks, Data Factory and Synapse they become values the deploying pipeline +supplies, not values read when generating; see +[Read an environment variable](/docs/concepts/yaml-references#read-an-environment-variable). +Generating never reads a secret or tests the deployed infrastructure. + +Use [Add an environment](/docs/how-to/add-an-environment) to create another +environment and compare the resolved choices, and +[Deploy through environments](/docs/how-to/deploy-through-environments) to +deploy them in turn. diff --git a/content/docs/concepts/generated-files.mdx b/content/docs/concepts/generated-files.mdx index 301d1ee..b27edf4 100644 --- a/content/docs/concepts/generated-files.mdx +++ b/content/docs/concepts/generated-files.mdx @@ -1,17 +1,57 @@ --- title: Generated files and ownership -description: Where Ingestron writes, how it protects your edits, and how to check output in CI. +description: Where Ingestron writes each platform's files, how it protects your edits, and how to check output in CI. --- -Ingestron writes one folder per environment, `build//` by -default. When a standard uses several packs, each pack gets its own folder -inside it, such as `build/dev/databricks/` and `build/dev/adf/`. +Ingestron writes to an output folder, `build/` by default, with one folder for +each platform. Each holds that platform's files for every environment, laid +out as its own CI/CD tools expect. + +## Where the files go [#where-the-files-go] + +A project's files are in the same place whether its standard lists one pack or +several, so adding a second pack never moves the first one's files: + +| Folder | What it holds | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| `build/databricks/` | One bundle: `databricks.yml` with a target per environment, `resources/`, `src//`, and `prerequisites/.sql` | +| `build/adf/` | One factory: `factory/`, the folder Microsoft's build reads, and `parameters/.json` | +| `build/synapse/` | `views.sql`, and `environments/.sql` | +| `build/duckdb/` | A folder per environment, each with `run.sql` and `tables/` | +| `build/products/` | A folder per environment, with its data product passports | + +- **Databricks, Azure Data Factory and Synapse** get one set of files for every + environment. What is the same in every environment is written once, as it + is; a value that differs is a bundle variable, an ARM template parameter or + a SQLCMD variable, set in each environment's target, parameter file or + script. A reviewer reads the resources once, and the per-environment values + in one place. +- **DuckDB** has no deployment of its own to promote, so each environment it + runs gets its own folder, whose SQL differs only in where the files are. +- **Passports** describe one environment's tables, so each environment has its + own. + +A platform gets a folder only when some environment uses it. An environment +that runs on DuckDB alone, such as a laptop environment with +`runs-on: duckdb`, is not a target of the Databricks bundle or the factory. + +Environments may differ only in values, such as names, paths and hosts. A +table built in some environments only, or a resource name that differs, is +[`ENVIRONMENTS_DIFFER`](/docs/reference/errors#ENVIRONMENTS_DIFFER). Each +platform page's [What Ingestron writes](/docs/platforms/databricks#what-you-get) +section shows its files, and +[Deploy through environments](/docs/how-to/deploy-through-environments) deploys +them, one environment after another. + +## What protects your edits [#what-protects-your-edits] Generated SQL, Python and YAML files start with a comment saying where they came from and not to edit them. JSON files, such as the Azure Data Factory -resources, cannot hold comments. Each folder also holds `.generated.json`, -which records every generated file, whatever its format. With it Ingestron -knows: +resources, cannot hold comments. Each platform's folder also holds +`.generated.json`, which records every generated file, whatever its format. +For Data Factory it sits beside `factory/`, not in it, because Microsoft warns +that unrelated files in a factory's folder can stop it loading. With it +Ingestron knows: - which files it may update, and which it should delete when a table goes away; @@ -47,9 +87,11 @@ What names Ingestron in the files does nothing when they run: If you stop using Ingestron, keep the files and edit them yourself; you can delete each folder's `.generated.json`. While a project still generates a folder, keep its `.generated.json`: without it, `ingestron plan` reports every -file that differs from what it would generate as a conflict. When a project stops generating a folder, for -example after you remove an environment, `ingestron check` says so; delete that -folder's `.generated.json` to keep its files as your own, as +file that differs from what it would generate as a conflict. When a project +stops generating a folder, for example after you remove the last table of a +platform, or after upgrading from the folder per environment of version 1, +`ingestron check` says so; delete that folder's `.generated.json` to keep its +files as your own, as [Check generated files in CI](/docs/how-to/check-in-ci) shows. The one exception is [Run DuckDB in Azure](/docs/platforms/duckdb-in-azure). @@ -62,7 +104,7 @@ it, as that page shows. `ingestron apply` writes one output folder at a time. A folder with a conflict is left as it was, while the others are still written, and the command ends with a line such as -`Applied: dev. Not applied: prod.` and exits with `1`. +`Applied: databricks. Not applied: adf.` and exits with `1`. If writing a file fails, Ingestron puts back the files it already changed in that folder and says `Nothing was changed.` If it cannot put one back, or diff --git a/content/docs/concepts/layers.mdx b/content/docs/concepts/layers.mdx index 7975faf..17ae5be 100644 --- a/content/docs/concepts/layers.mdx +++ b/content/docs/concepts/layers.mdx @@ -46,7 +46,7 @@ tables in any layer. `region_sales` reads `store_sales` in `silver`. unless the standard's or the platform's `naming` sets it otherwise, so each layer is a schema. On Databricks, `store_sales` above is `` `dev`.`silver`.`store_sales` ``, and -its file is `build/dev/src/sales/silver/store_sales.sql`. A name pattern can +its file is `build/databricks/src/sales/silver/store_sales.sql`. A name pattern can use `{{ layer }}` anywhere; see [Name tables and resources](/docs/how-to/name-things-your-way). diff --git a/content/docs/concepts/standards-and-packs.mdx b/content/docs/concepts/standards-and-packs.mdx index 5073f81..854f3b4 100644 --- a/content/docs/concepts/standards-and-packs.mdx +++ b/content/docs/concepts/standards-and-packs.mdx @@ -67,7 +67,7 @@ interface. [Reachability](/docs/start/choose-a-route#reachability) explains the The first pack in the standard supplies project defaults such as naming, layers and column types. Rules are considered in pack order. The order is precedence, not the order things run: list the pack that stores or serves the -tables first, such as `[duckdb@1, adf@1]`, even though ADF copies first. Each resolved action records the +tables first, such as `[duckdb@2, adf@2]`, even though ADF copies first. Each resolved action records the pack that owns it, and each pack generates its own output folder when several packs are selected. diff --git a/content/docs/concepts/yaml-references.mdx b/content/docs/concepts/yaml-references.mdx index d28b92a..0e75b3e 100644 --- a/content/docs/concepts/yaml-references.mdx +++ b/content/docs/concepts/yaml-references.mdx @@ -10,20 +10,20 @@ Quotation marks do not disable references: a quoted YAML string is still resolve ## Choose the right syntax -| Write | Meaning | Where to use it | -| ------------------------------------ | ---------------------------------------------------- | -------------------------------------------------------------- | -| `${values.landing}` | A project value after environment overrides | Definition values | -| `${values.cluster.workers}` | A nested value | Definition values | -| `${values.hosts.0}` | The first item in a value list | Definition values | -| `${environment.name}` | The selected environment, such as `dev` | Values and definitions | -| `${project.id}` or `${project.name}` | The project ID or optional display name | Values and definitions | -| `${file:query.sql}` | Contents of a project file | Whole value, or embedded when the result is scalar | -| `${file:shared.yaml#/columns/0}` | A value within a YAML file | Whole value or embedded scalar; the fragment is a JSON Pointer | -| `${secret:source-password}` | A reference to a secret, never the secret's contents | A whole value accepted by a connector or method | -| `${envvar:SOURCE_HOST}` | Text from an environment variable or an `.env` file | Non-secret values for one person or machine; see below | -| `{{ environment }}` or `{{ table }}` | A naming pattern evaluated with naming context | Standard or platform `naming` only | -| `{{ tables.orders }}` | A declared input's generated name | SQL transform code supported by the selected method | -| `$${bundle.target}` | Literal `${bundle.target}` text after YAML loading | Text intended for another tool; see transform exceptions below | +| Write | Meaning | Where to use it | +| ------------------------------------ | ----------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | +| `${values.landing}` | A project value after environment overrides | Definition values | +| `${values.cluster.workers}` | A nested value | Definition values | +| `${values.hosts.0}` | The first item in a value list | Definition values | +| `${environment.name}` | The selected environment, such as `dev` | Values and definitions | +| `${project.id}` or `${project.name}` | The project ID or optional display name | Values and definitions | +| `${file:query.sql}` | Contents of a project file | Whole value, or embedded when the result is scalar | +| `${file:shared.yaml#/columns/0}` | A value within a YAML file | Whole value or embedded scalar; the fragment is a JSON Pointer | +| `${secret:source-password}` | A reference to a secret, never the secret's contents | A whole value accepted by a connector or method | +| `${envvar:SOURCE_HOST}` | A value the deploying pipeline supplies, or text from an environment variable or an `.env` file | Non-secret values each stage or machine sets; see below | +| `{{ environment }}` or `{{ table }}` | A naming pattern evaluated with naming context | Standard or platform `naming` only | +| `{{ tables.orders }}` | A declared input's generated name | SQL transform code supported by the selected method | +| `$${bundle.target}` | Literal `${bundle.target}` text after YAML loading | Text intended for another tool; see transform exceptions below | Project `id`, `name` and `imports` are read literally. YAML mapping keys are not interpolated. Write references in values, not in field names or import patterns. @@ -91,25 +91,57 @@ Do not embed a secret reference in a URL or connection string. ## Read an environment variable -`${envvar:SOURCE_HOST}` reads text from an environment variable. Names use -letters, digits and underscores and cannot start with a digit. Ingestron uses +`${envvar:NAME}` names a value that is not written in the definitions. Names +use letters, digits and underscores and cannot start with a digit. What +happens to it depends on the platform's pack. + +### Supplied when deploying [#supplied-when-deploying] + +Databricks, Azure Data Factory and Synapse get one set of files for every +environment, so a value from a variable is not read when generating. It +becomes a parameter of the generated files, and the pipeline that deploys +each environment supplies it: + +| Platform | The value becomes | The deploying pipeline sets | +| ------------------ | ---------------------------------------------------------------- | ---------------------------------------------------------------- | +| Databricks | a bundle variable that the environment's target leaves unset | `BUNDLE_VAR_`, where `` is `NAME` in lower case | +| Azure Data Factory | an ARM template parameter that the environment's file leaves out | that parameter, after `factoryName`, when deploying the template | +| Synapse | a sqlcmd variable that the environment's script does not set | an environment variable `NAME`, in the same case | + +So the files are the same on every machine and in CI, and a laptop needs no +CI values to plan. `ingestron request` lists each such value with the name to +set; see [Values to provide](/docs/how-to/ask-the-platform-team#values-to-provide). +A workspace host supplied this way is `DATABRICKS_HOST`. + +Only a value can wait until deploying. A variable in a flow's `source`, a +contract id, a resource name or anything else that decides what is generated +or what it is called is +[`DEPLOY_VALUE`](/docs/reference/errors#DEPLOY_VALUE): write it in +`project.yaml` for each environment. Data Factory and Synapse take only a whole +value from a variable: `${envvar:…}` inside a longer value, such as a path, is +`DEPLOY_VALUE` there too, while Databricks puts the variable into the text. + +### Read when generating [#read-when-generating] + +DuckDB, passports and the editor read the value when generating. Ingestron uses the first non-empty value from the environment of the process that runs it, then `.env.`, then `.env`. An empty value counts as unset. A variable that is unset everywhere reports [`ENVVAR_MISSING`](/docs/reference/errors#ENVVAR_MISSING). The CLI reports it -as an error. The editor reports it as information that names the file to add it -to, so the rest of the project is still checked. +as an error. The editor reports it as information, so the rest of the project +is still checked: for DuckDB it names the file to add it to, and for a value +supplied when deploying it names what the deploying pipeline sets instead. -An environment variable's value becomes ordinary resolved text and may reach -generated files. Use `${secret:…}` for credentials instead. +A value read when generating becomes ordinary text and may reach generated +files. Use `${secret:…}` for credentials instead. ## Choose where a value lives @@ -120,13 +152,14 @@ Start with the first place and move down only when the value does not fit: reviewer sees every change. The `prod` entry in [the example above](#project-values-and-environments) replaces `landing` and `host` this way. -2. **`.env` files**, for a value that differs for each person or machine, such - as a folder on your laptop or a sandbox workspace, or that your - infrastructure code produces, such as the name of a storage account it - creates. Do not commit them. -3. **Pipeline variables in CI.** Set the variable in the pipeline and pass it to - the step that runs Ingestron. The process environment wins over the files, so - CI needs no `.env`. +2. **`.env` files**, for a value read when generating that differs for each + person or machine, such as a folder on your laptop for DuckDB. Do not commit + them. +3. **Pipeline variables in CI.** For Databricks, Data Factory and Synapse, set + the variable for each environment in the pipeline that deploys, as + [Deploy through environments](/docs/how-to/deploy-through-environments#values-supplied-when-deploying) + shows. For DuckDB, pass it to the step that runs Ingestron: the process + environment wins over the files, so CI needs no `.env`. 4. **`${secret:…}`**, for every credential. A `.env` file holds values that are not secret: a line that Ingestron cannot read is reported without quoting it, but a problem with the value itself may show it. diff --git a/content/docs/how-to/(build)/add-a-source.mdx b/content/docs/how-to/(build)/add-a-source.mdx index acb0610..610ab7d 100644 --- a/content/docs/how-to/(build)/add-a-source.mdx +++ b/content/docs/how-to/(build)/add-a-source.mdx @@ -26,7 +26,7 @@ and apply it again first. ```csv customer_id,name,region - 1,Aroha,North + 1,Ana,North 2,Ben,South 3,Chen,North ``` @@ -67,9 +67,9 @@ and apply it again first. ``` ```text - dev → build/dev - update run.sql - create tables/bronze.customers.sql + duckdb → build/duckdb (dev) + update dev/run.sql + create dev/tables/bronze.customers.sql 5 files: 1 create, 1 update, 3 unchanged. Ready to apply. ``` @@ -128,7 +128,9 @@ from its `project.yaml`, renaming any value your project already has. ## Start beside what you already run `plan` and `apply` write only to the output folder, `build/` by default, so a -project owns only the files it generated. A first project can hold one source +project owns only the files it generated. Each platform's files are in a +folder of their own, such as `build/databricks/`; see +[Generated files and ownership](/docs/concepts/generated-files). A first project can hold one source while the rest of your platform stays as it is: - **Keep the project in a folder of its own,** in your repository or in a new diff --git a/content/docs/how-to/(build)/add-an-environment.mdx b/content/docs/how-to/(build)/add-an-environment.mdx index 407b6f6..21a4a7a 100644 --- a/content/docs/how-to/(build)/add-an-environment.mdx +++ b/content/docs/how-to/(build)/add-an-environment.mdx @@ -1,6 +1,6 @@ --- title: Add an environment -description: Generate another environment, such as test, from the same definitions. +description: Add another environment, such as test, to the files every environment shares, and see what it changes. --- 1. Add it to `environments` in `project.yaml`, with the values that differ: @@ -12,28 +12,46 @@ description: Generate another environment, such as test, from the same definitio -3. Generate it: +3. Generate it. Every environment is generated together, so plan and apply + take no environment: ```sh -ingestron plan --env test -ingestron apply --env test +ingestron plan +ingestron apply ``` -Click the environment in the status bar, or run **Ingestron: Select environment**, and choose `test`. Then run **Ingestron: Plan** then **Ingestron: Apply** from the Command Palette (⇧⌘P on macOS, Ctrl+Shift+P elsewhere). The output is in the Ingestron output panel. Problems show in the Problems panel as you type, before you plan. +Run **Ingestron: Plan**, then **Ingestron: Apply**, from the Command Palette (⇧⌘P on macOS, Ctrl+Shift+P elsewhere). The output is in the Ingestron output panel. To see the new environment's problems and previews, click the environment in the status bar, or run **Ingestron: Select environment**, and choose `test`. -The new environment gets its own folder, `build/test/`. An environment named -`prod` or `production` resolves to production mode in the current Platform model; every other one in -development mode. These modes affect the generated resources, not deployment by -Ingestron. See [Environments and platform](/docs/concepts/environments-and-platform). +The new environment gets no folder of its own. It joins the files each +platform already has: here, a `test` target in the bundle's `databricks.yml`, +with the values that differ between environments: + + + +Other platforms add their own file or folder for it; see +[Where the files go](/docs/concepts/generated-files#where-the-files-go). + +Only an environment named `dev` is in development mode; every other one, +`test` included, is in production mode, and schedules run only in `prod` or +`production`, so `test`'s refresh job is paused. A platform can set `mode` and +`schedule.paused` itself; see the +[platform defaults](/docs/reference/packs/databricks#platform-configuration). +Environments may differ only in values, such as names, paths and hosts: a table +built in one environment only is refused as +[`ENVIRONMENTS_DIFFER`](/docs/reference/errors#ENVIRONMENTS_DIFFER). + +To deploy the new environment, add it to your deployment pipeline as +[Deploy through environments](/docs/how-to/deploy-through-environments#add-an-environment) +shows. ## Check what differs @@ -51,6 +69,7 @@ For the example above, it prints: ## Platform +- `mode`: `development` → `production` - `workspace.root_path`: `/Workspace/Shared/.bundle/finance/dev` → `/Workspace/Shared/.bundle/finance/test` - `tags.environment`: `dev` → `test` @@ -64,8 +83,8 @@ For the example above, it prints: place, and lists only what differs: - **Platform:** the platform document's values, and the defaults that follow - from the environment. Compared with `prod`, it also shows `mode` and - `schedule.paused`, which production changes. + from the environment, such as `mode`. Compared with `prod`, it also shows + `schedule.paused`, because only `prod` runs its schedule. - **Names:** the pipeline and job names, which are not per table. - **Table:** one section for each table that differs: its full name, each step's method and settings, the connection's settings and the table's diff --git a/content/docs/how-to/(build)/add-quality-checks.mdx b/content/docs/how-to/(build)/add-quality-checks.mdx index 6bcaddd..626e834 100644 --- a/content/docs/how-to/(build)/add-quality-checks.mdx +++ b/content/docs/how-to/(build)/add-quality-checks.mdx @@ -51,7 +51,7 @@ Run **Ingestron: Plan** from the Command Palette (⇧⌘P on macOS, Ctrl+Shift+P The checks become constraints on the table: - + Tables written by Lakeflow Connect cannot run your own checks; put them on a transform that reads the table. diff --git a/content/docs/how-to/(build)/check-in-ci.mdx b/content/docs/how-to/(build)/check-in-ci.mdx index 3cdd04e..fe59428 100644 --- a/content/docs/how-to/(build)/check-in-ci.mdx +++ b/content/docs/how-to/(build)/check-in-ci.mdx @@ -1,73 +1,55 @@ --- title: Check generated files in CI -description: Fail a build when someone changed the definitions without generating. +description: Fail a pull request when someone changed the definitions without generating, with GitHub Actions or Azure DevOps. --- Commit the `build/` folder with your definitions. In CI, run `ingestron check` -from the definitions project folder. It needs only Node.js 22.13 or later: each -template installs the command from npm with `npm install -g ingestron@1`, the -newest 1.x release, then runs it. To choose when a new release changes your -generated files, install the exact version, as [Upgrade Ingestron and its packs](/docs/how-to/upgrade#choose-the-version-you-run) -shows. +from the definitions project folder on every pull request: it fails when +someone changed the definitions without generating, so the files you deploy +are always what the definitions say. It needs only Node.js 22.13 or later: +each template installs an exact version of the command from npm, +`npm install -g ingestron@2.1.0`, then runs it. Change the version when you +upgrade, as +[Upgrade Ingestron and its packs](/docs/how-to/upgrade#choose-the-version-you-run) +shows, so a new release never fails a pull request that changed nothing. + +To deploy what the check passed, one environment after another, see +[Deploy through environments](/docs/how-to/deploy-through-environments): its +pipelines run this check first. ## GitHub Actions Save as `.github/workflows/ingestron-check.yml`: -```yaml -name: ingestron check -on: - pull_request: - push: - branches: [main] -permissions: - contents: read -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 22 - - run: npm install -g ingestron@1 - - run: ingestron check -``` +