Docs for ingestron 2.0: one output folder per platform, deploy through environments

ci-templates/azure-devops/azure-pipelines.yml

@@ -0,0 +1,71 @@
+# Deploys the committed build/ folder: checks it, builds the Data Factory
+# template once, then deploys the same files to dev, test and prod in turn.
+# Copy this file and templates/deploy-environment.yml to your definitions
+# repository, add this file as a pipeline, and delete the steps of each
+# platform your project does not use.
+#
+# Set up once, as https://docs.ingestron.io/docs/how-to/deploy-through-environments
+# explains:
+# - Azure DevOps environments named dev, test and prod. Add approvals to test
+#   and prod, a branch control check to prod that allows only main, and an
+#   exclusive lock check to each, so runs deploy an environment one at a time,
+#   in order (each stage sets lockBehavior: sequential).
+# - For each environment, an Azure Resource Manager service connection that
+#   uses workload identity federation, and a variable group named
+#   ingestron-<environment> with the variables the template reads.
+# - The pipeline variable ADF_DEV_FACTORY_ID: the dev factory's resource ID,
+#   which names the template's resources. Nothing is deployed with it.
+trigger:
+  branches:
+    include: [main]
+pr: none
+
+stages:
+  - stage: Build
+    jobs:
+      - job: build
+        pool:
+          vmImage: ubuntu-latest
+        steps:
+          - checkout: self
+          - task: UseNode@1
+            inputs:
+              version: 22.x
+          # An exact version: change it when you upgrade, as
+          # https://docs.ingestron.io/docs/how-to/upgrade#choose-the-version-you-run says.
+          - script: npm install -g ingestron@2.1.0
+            displayName: Install Ingestron
+          - script: ingestron check
+            displayName: ingestron check
+          # Data Factory: Microsoft's export validates the factory, then writes
+          # the ARM template and the pre- and post-deployment script to arm/.
+          # It downloads its build tool from adf.azure.com.
+          - script: |
+              npm install --prefix "$(Agent.TempDirectory)/adf" @microsoft/azure-data-factory-utilities@1.0.3
+              adf="$(Agent.TempDirectory)/adf/node_modules/@microsoft/azure-data-factory-utilities/lib/index"
+              node "$adf" export build/adf/factory "$(ADF_DEV_FACTORY_ID)" arm
+            displayName: "Data Factory: build the template"
+          # Every environment deploys these files, so prod gets the bytes dev
+          # got. Without Data Factory, copy build alone.
+          - script: cp -R arm build "$(Build.ArtifactStagingDirectory)/"
+            displayName: Collect the generated files
+          - task: PublishPipelineArtifact@1
+            inputs:
+              targetPath: $(Build.ArtifactStagingDirectory)
+              artifact: deploy
+
+  - template: templates/deploy-environment.yml
+    parameters:
+      environment: dev
+      serviceConnection: ingestron-dev
+      dependsOn: Build
+  - template: templates/deploy-environment.yml
+    parameters:
+      environment: test
+      serviceConnection: ingestron-test
+      dependsOn: Deploy_dev
+  - template: templates/deploy-environment.yml
+    parameters:
+      environment: prod
+      serviceConnection: ingestron-prod
+      dependsOn: Deploy_test

ci-templates/azure-devops/ingestron-check.yml

@@ -10,10 +10,12 @@ pool:
   vmImage: ubuntu-latest
 steps:
   - checkout: self
-  - task: NodeTool@0
+  - task: UseNode@1
     inputs:
-      versionSpec: 22.x
-  - script: npm install -g ingestron@1
+      version: 22.x
+  # An exact version: change it when you upgrade, as
+  # https://docs.ingestron.io/docs/how-to/upgrade#choose-the-version-you-run says.
+  - script: npm install -g ingestron@2.1.0
     displayName: Install Ingestron
   - script: ingestron check
     displayName: ingestron check

ci-templates/azure-devops/templates/deploy-environment.yml

@@ -0,0 +1,155 @@
+# Deploys the files the Build stage published to one environment. Its
+# deployment job targets the Azure DevOps environment of the same name, so that
+# environment's approvals and checks gate it.
+#
+# Variables to set in the variable group ingestron-<environment>:
+# - AZURE_SUBSCRIPTION_ID, AZURE_LOCATION: the subscription and region of the
+#   environment's resource group.
+# - ADF_RESOURCE_GROUP, ADF_FACTORY_NAME: the environment's data factory.
+# - SYNAPSE_SERVER: <workspace>-ondemand.sql.azuresynapse.net.
+# - DUCKDB_ACCOUNT, DUCKDB_FILE_SYSTEM, DUCKDB_PATH: where the DuckDB job reads
+#   run.sql, as the job's sqlUrl names it.
+# Every step signs in as the environment's service connection. The Azure CLI
+# steps set useGlobalConfig, so that what they start, such as databricks bundle,
+# uses the signed-in session: see Microsoft Learn, "Authenticate with Azure
+# DevOps on Azure Databricks".
+parameters:
+  - name: environment
+    type: string
+  - name: serviceConnection
+    type: string
+  - name: dependsOn
+    type: string
+
+stages:
+  - stage: Deploy_${{ parameters.environment }}
+    displayName: Deploy ${{ parameters.environment }}
+    dependsOn: ${{ parameters.dependsOn }}
+    # With the environment's exclusive lock check, runs deploy it one at a time,
+    # in the order they started.
+    lockBehavior: sequential
+    variables:
+      - group: ingestron-${{ parameters.environment }}
+      - name: files
+        value: $(Pipeline.Workspace)/deploy
+    jobs:
+      - deployment: deploy
+        displayName: Deploy ${{ parameters.environment }}
+        pool:
+          vmImage: ubuntu-latest
+        environment: ${{ parameters.environment }}
+        strategy:
+          runOnce:
+            deploy:
+              steps:
+                - download: current
+                  artifact: deploy
+
+                # Data Factory. One factory per project: the post-deployment
+                # script deletes every resource in the factory that the
+                # template does not hold.
+                - task: AzurePowerShell@5
+                  displayName: "Data Factory: stop the triggers"
+                  inputs:
+                    azureSubscription: ${{ parameters.serviceConnection }}
+                    azurePowerShellVersion: LatestVersion
+                    pwsh: true
+                    ScriptType: FilePath
+                    ScriptPath: $(files)/arm/PrePostDeploymentScript.ps1
+                    ScriptArguments: >-
+                      -ArmTemplate $(files)/arm/ARMTemplateForFactory.json
+                      -ArmTemplateParameters $(files)/build/adf/parameters/${{ parameters.environment }}.json
+                      -ResourceGroupName $(ADF_RESOURCE_GROUP) -DataFactoryName $(ADF_FACTORY_NAME)
+                      -PreDeployment $true -DeleteDeployment $false
+                - task: AzureResourceManagerTemplateDeployment@3
+                  displayName: "Data Factory: deploy the template"
+                  inputs:
+                    deploymentScope: Resource Group
+                    azureResourceManagerConnection: ${{ parameters.serviceConnection }}
+                    subscriptionId: $(AZURE_SUBSCRIPTION_ID)
+                    action: Create Or Update Resource Group
+                    resourceGroupName: $(ADF_RESOURCE_GROUP)
+                    location: $(AZURE_LOCATION)
+                    templateLocation: Linked artifact
+                    csmFile: $(files)/arm/ARMTemplateForFactory.json
+                    csmParametersFile: $(files)/build/adf/parameters/${{ parameters.environment }}.json
+                    # Add -<name> <value> for each ARM parameter that
+                    # `ingestron request` lists under "Values to provide".
+                    overrideParameters: -factoryName $(ADF_FACTORY_NAME)
+                    deploymentMode: Incremental
+                - task: AzurePowerShell@5
+                  displayName: "Data Factory: clean up and start the triggers"
+                  inputs:
+                    azureSubscription: ${{ parameters.serviceConnection }}
+                    azurePowerShellVersion: LatestVersion
+                    pwsh: true
+                    ScriptType: FilePath
+                    ScriptPath: $(files)/arm/PrePostDeploymentScript.ps1
+                    ScriptArguments: >-
+                      -ArmTemplate $(files)/arm/ARMTemplateForFactory.json
+                      -ArmTemplateParameters $(files)/build/adf/parameters/${{ parameters.environment }}.json
+                      -ResourceGroupName $(ADF_RESOURCE_GROUP) -DataFactoryName $(ADF_FACTORY_NAME)
+                      -PreDeployment $false -DeleteDeployment $true
+
+                # Databricks: the bundle's target for this environment. A
+                # deploy that would delete a resource stops and asks; it is
+                # not approved here.
+                - bash: |
+                    curl -fsSL -o "$(Agent.TempDirectory)/databricks.tar.gz" https://github.com/databricks/cli/releases/download/v1.23.0/databricks_cli_1.23.0_linux_amd64.tar.gz
+                    echo "a5ec088f7f6cc5974327a42df99261e22fbc6f7ca9cddb945333c238dd9e2289  $(Agent.TempDirectory)/databricks.tar.gz" | sha256sum -c -
+                    mkdir -p "$(Agent.TempDirectory)/databricks" && tar -xzf "$(Agent.TempDirectory)/databricks.tar.gz" -C "$(Agent.TempDirectory)/databricks" databricks
+                    echo "##vso[task.prependpath]$(Agent.TempDirectory)/databricks"
+                  displayName: "Databricks: install the CLI"
+                - task: AzureCLI@2
+                  displayName: "Databricks: deploy the bundle"
+                  env:
+                    DATABRICKS_AUTH_TYPE: azure-cli
+                    # One line for each BUNDLE_VAR_ name that `ingestron request` lists:
+                    # BUNDLE_VAR_landing_path: $(LANDING_PATH)
+                  inputs:
+                    azureSubscription: ${{ parameters.serviceConnection }}
+                    scriptType: bash
+                    scriptLocation: inlineScript
+                    useGlobalConfig: true
+                    workingDirectory: $(files)/build/databricks
+                    inlineScript: |
+                      databricks bundle validate -t ${{ parameters.environment }}
+                      databricks bundle deploy -t ${{ parameters.environment }}
+
+                # Synapse serverless: sqlcmd (Go), checked by its SHA-256,
+                # signed in as the service connection.
+                - bash: |
+                    curl -fsSL -o "$(Agent.TempDirectory)/sqlcmd.tar.bz2" https://github.com/microsoft/go-sqlcmd/releases/download/v1.10.0/sqlcmd-linux-amd64.tar.bz2
+                    echo "92516d98c63d99b0994de5b61350c91f6915f9b76f139a59039fbcb225c2e987  $(Agent.TempDirectory)/sqlcmd.tar.bz2" | sha256sum -c -
+                    mkdir -p "$(Agent.TempDirectory)/sqlcmd" && tar -xjf "$(Agent.TempDirectory)/sqlcmd.tar.bz2" -C "$(Agent.TempDirectory)/sqlcmd" sqlcmd
+                    echo "##vso[task.prependpath]$(Agent.TempDirectory)/sqlcmd"
+                  displayName: "Synapse: install sqlcmd"
+                - task: AzureCLI@2
+                  displayName: "Synapse: create or alter the views"
+                  # One line for each sqlcmd variable that `ingestron request` lists:
+                  # env:
+                  #   SYNAPSE_SOURCE: $(SYNAPSE_SOURCE)
+                  inputs:
+                    azureSubscription: ${{ parameters.serviceConnection }}
+                    scriptType: bash
+                    scriptLocation: inlineScript
+                    useGlobalConfig: true
+                    workingDirectory: $(files)/build/synapse
+                    inlineScript: >-
+                      sqlcmd -S "$(SYNAPSE_SERVER)" --authentication-method ActiveDirectoryDefault
+                      -i environments/${{ parameters.environment }}.sql
+
+                # DuckDB in Azure: copy the environment's SQL to where the job
+                # reads it.
+                - task: AzureCLI@2
+                  displayName: "DuckDB: upload the SQL"
+                  inputs:
+                    azureSubscription: ${{ parameters.serviceConnection }}
+                    scriptType: bash
+                    scriptLocation: inlineScript
+                    useGlobalConfig: true
+                    workingDirectory: $(files)
+                    inlineScript: >-
+                      az storage fs directory upload --account-name "$(DUCKDB_ACCOUNT)"
+                      --file-system "$(DUCKDB_FILE_SYSTEM)" --source "build/duckdb/${{ parameters.environment }}/*"
+                      --destination-path "$(DUCKDB_PATH)" --recursive --auth-mode login

ci-templates/github-actions/deploy-environment.yml

@@ -0,0 +1,126 @@
+# Deploys the files deploy.yml built to one environment. It runs in the GitHub
+# environment of the same name, so that environment's reviewers and branch rule
+# gate it, and its variables and secrets are the ones used. Required reviewers
+# on a private repository need GitHub Enterprise.
+#
+# Variables to set in each GitHub environment:
+# - AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID: the identity this
+#   environment deploys as, with a federated credential for the job's subject:
+#   repo:<owner>/<repository>:environment:<environment> for a repository
+#   created, renamed or transferred before 15 July 2026, and
+#   repo:<owner>@<owner id>/<repository>@<repository id>:environment:<environment>
+#   for one created, renamed or transferred since, or opted in
+#   (gh api repos/<owner>/<repository> gives .owner.id and .id).
+# - ADF_RESOURCE_GROUP, ADF_FACTORY_NAME: the environment's data factory.
+# - DATABRICKS_CLIENT_ID: the service principal the bundle deploys as, with a
+#   federation policy for the same subject.
+# - SYNAPSE_SERVER: <workspace>-ondemand.sql.azuresynapse.net.
+# - DUCKDB_ACCOUNT, DUCKDB_FILE_SYSTEM, DUCKDB_PATH: where the DuckDB job reads
+#   run.sql, as the job's sqlUrl names it.
+name: deploy environment
+on:
+  workflow_call:
+    inputs:
+      environment:
+        description: The environment to deploy, as project.yaml names it.
+        required: true
+        type: string
+permissions:
+  contents: read
+  id-token: write
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    concurrency:
+      group: deploy-${{ inputs.environment }}
+      cancel-in-progress: false
+    env:
+      ENVIRONMENT: ${{ inputs.environment }}
+    steps:
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: deploy
+      - uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
+        with:
+          client-id: ${{ vars.AZURE_CLIENT_ID }}
+          tenant-id: ${{ vars.AZURE_TENANT_ID }}
+          subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }}
+          enable-AzPSSession: true
+
+      # Data Factory. One factory per project: the post-deployment script
+      # deletes every resource in the factory that the template does not hold.
+      - name: "Data Factory: stop the triggers"
+        uses: azure/powershell@f5b8adcfff1904872c7b98d4012d4914d74b1a82 # v3.0.0
+        env:
+          ADF_RESOURCE_GROUP: ${{ vars.ADF_RESOURCE_GROUP }}
+          ADF_FACTORY_NAME: ${{ vars.ADF_FACTORY_NAME }}
+        with:
+          azPSVersion: latest
+          inlineScript: |
+            ./arm/PrePostDeploymentScript.ps1 -ArmTemplate arm/ARMTemplateForFactory.json `
+              -ArmTemplateParameters "build/adf/parameters/$env:ENVIRONMENT.json" `
+              -ResourceGroupName $env:ADF_RESOURCE_GROUP -DataFactoryName $env:ADF_FACTORY_NAME `
+              -PreDeployment $true -DeleteDeployment $false
+      - name: "Data Factory: deploy the template"
+        env:
+          ADF_RESOURCE_GROUP: ${{ vars.ADF_RESOURCE_GROUP }}
+          ADF_FACTORY_NAME: ${{ vars.ADF_FACTORY_NAME }}
+        # Add name=value after factoryName for each ARM parameter that
+        # `ingestron request` lists under "Values to provide".
+        run: |
+          az deployment group create --resource-group "$ADF_RESOURCE_GROUP" --mode Incremental \
+            --template-file arm/ARMTemplateForFactory.json \
+            --parameters "@build/adf/parameters/$ENVIRONMENT.json" factoryName="$ADF_FACTORY_NAME"
+      - name: "Data Factory: clean up and start the triggers"
+        uses: azure/powershell@f5b8adcfff1904872c7b98d4012d4914d74b1a82 # v3.0.0
+        env:
+          ADF_RESOURCE_GROUP: ${{ vars.ADF_RESOURCE_GROUP }}
+          ADF_FACTORY_NAME: ${{ vars.ADF_FACTORY_NAME }}
+        with:
+          azPSVersion: latest
+          inlineScript: |
+            ./arm/PrePostDeploymentScript.ps1 -ArmTemplate arm/ARMTemplateForFactory.json `
+              -ArmTemplateParameters "build/adf/parameters/$env:ENVIRONMENT.json" `
+              -ResourceGroupName $env:ADF_RESOURCE_GROUP -DataFactoryName $env:ADF_FACTORY_NAME `
+              -PreDeployment $false -DeleteDeployment $true
+
+      # Databricks: the bundle's target for this environment. A deploy that
+      # would delete a resource stops and asks; it is not approved here.
+      - uses: databricks/setup-cli@9625fdde9ba2607df0087e0d3ec7449198bf2180 # v1.23.0
+      - name: "Databricks: deploy the bundle"
+        working-directory: build/databricks
+        env:
+          DATABRICKS_AUTH_TYPE: github-oidc
+          DATABRICKS_CLIENT_ID: ${{ vars.DATABRICKS_CLIENT_ID }}
+          # One line for each BUNDLE_VAR_ name that `ingestron request` lists:
+          # BUNDLE_VAR_landing_path: ${{ vars.LANDING_PATH }}
+        run: |
+          databricks bundle validate -t "$ENVIRONMENT"
+          databricks bundle deploy -t "$ENVIRONMENT"
+
+      # Synapse serverless: sqlcmd (Go), checked by its SHA-256, signed in as
+      # the identity azure/login used.
+      - name: "Synapse: install sqlcmd"
+        run: |
+          curl -fsSL -o "$RUNNER_TEMP/sqlcmd.tar.bz2" https://github.com/microsoft/go-sqlcmd/releases/download/v1.10.0/sqlcmd-linux-amd64.tar.bz2
+          echo "92516d98c63d99b0994de5b61350c91f6915f9b76f139a59039fbcb225c2e987  $RUNNER_TEMP/sqlcmd.tar.bz2" | sha256sum -c -
+          mkdir -p "$RUNNER_TEMP/sqlcmd" && tar -xjf "$RUNNER_TEMP/sqlcmd.tar.bz2" -C "$RUNNER_TEMP/sqlcmd" sqlcmd
+          echo "$RUNNER_TEMP/sqlcmd" >> "$GITHUB_PATH"
+      - name: "Synapse: create or alter the views"
+        working-directory: build/synapse
+        env:
+          SYNAPSE_SERVER: ${{ vars.SYNAPSE_SERVER }}
+          # One line for each sqlcmd variable that `ingestron request` lists:
+          # SYNAPSE_SOURCE: ${{ vars.SYNAPSE_SOURCE }}
+        run: sqlcmd -S "$SYNAPSE_SERVER" --authentication-method ActiveDirectoryDefault -i "environments/$ENVIRONMENT.sql"
+
+      # DuckDB in Azure: copy the environment's SQL to where the job reads it.
+      - name: "DuckDB: upload the SQL"
+        env:
+          DUCKDB_ACCOUNT: ${{ vars.DUCKDB_ACCOUNT }}
+          DUCKDB_FILE_SYSTEM: ${{ vars.DUCKDB_FILE_SYSTEM }}
+          DUCKDB_PATH: ${{ vars.DUCKDB_PATH }}
+        run: |
+          az storage fs directory upload --account-name "$DUCKDB_ACCOUNT" --file-system "$DUCKDB_FILE_SYSTEM" \
+            --source "build/duckdb/$ENVIRONMENT/*" --destination-path "$DUCKDB_PATH" --recursive --auth-mode login