diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index b5b719cef..c19e1bf77 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -411,6 +411,9 @@ extends: odbcDependsOn: - ConsolidateOdbc installOdbcWheel: true + # Conda: build+validate conda packages on every x64 leg (win-64). + # ARM64 has no conda target (can't provision a win-arm64 host on x64). + buildConda: ${{ eq(config.arch, 'x64') }} # ========================= # MACOS BUILD STAGES @@ -440,6 +443,8 @@ extends: odbcDependsOn: - ConsolidateOdbc installOdbcWheel: true + # Conda: build+validate the osx-arm64 conda package on every macOS leg. + buildConda: true # ========================= # LINUX BUILD STAGES @@ -474,6 +479,11 @@ extends: odbcDependsOn: - ConsolidateOdbc installOdbcWheel: true + # Conda: build on BOTH manylinux_2_28 legs — x86_64 natively (linux-64) + # and aarch64 cross-targeted via QEMU binfmt (linux-aarch64), matching + # the two glibc Linux wheels on PyPI. musl has no conda target, so the + # musllinux legs still skip conda. + buildConda: ${{ eq(config.tag, 'manylinux_2_28') }} # ========================= # CONSOLIDATE STAGE @@ -566,3 +576,43 @@ extends: # mssql-python build stages now install the external mssql-python-odbc wheel # (from ConsolidateOdbc) and run the full pytest suite against it — so the # external-package resolution is already validated end-to-end during the build. + + # ========================================================================= + # CONSOLIDATE CONDA STAGE + # ========================================================================= + # Gathers the conda packages emitted by the conda-producing legs (win-64, + # osx-64, osx-arm64, linux-64, linux-aarch64) into a single conda/ tree and + # publishes them as `drop_ConsolidateConda_ConsolidateArtifacts`. BEST-EFFORT: + # it depends ONLY on the conda-producing legs (not the win-arm64 or musl legs, + # which have no conda target) and its job never hard-fails on a short count, so + # a conda hiccup can never block the wheel deliverable. The release pipeline + # enforces the hard conda gate (required subdirs + #706 pairing) before + # anything is published. + - stage: ConsolidateConda + displayName: 'Consolidate All Conda Packages' + dependsOn: + # win-64 (x64 only — win-arm64 has no conda host on an x64 agent). + # NOTE: these legs now emit ONLY the per-Python mssql-python (binding) conda; + # the win-64 companion (mssql-python-odbc) is built once in ODBC_BuildAll. + - Win_py310_x64 + - Win_py311_x64 + - Win_py312_x64 + - Win_py313_x64 + - Win_py314_x64 + # osx-64 + osx-arm64 (every macOS universal2 leg builds BOTH: arm64 native, + # x64 under Rosetta 2) + - MacOS_py310 + - MacOS_py311 + - MacOS_py312 + - MacOS_py313 + - MacOS_py314 + # linux-64 (glibc x86_64 host, native) + linux-aarch64 (x86_64 host + QEMU) + - Linux_manylinux_2_28_x86_64 + - Linux_manylinux_2_28_aarch64 + # win-64 companion (mssql-python-odbc) is built ONCE here (Python-agnostic). + - ODBC_BuildAll + jobs: + - template: /OneBranchPipelines/jobs/consolidate-conda-artifacts-job.yml@self + parameters: + # Match effectiveOneBranchType so scheduled builds run as 'Official'. + oneBranchType: '${{ variables.effectiveOneBranchType }}' diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index 6176156f2..6a16ea6c1 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -32,6 +32,30 @@ parameters: type: boolean default: true # Safe to enable - uses Maven ContentType for testing + # [TEST] Conda release readiness. When true, a DECOUPLED stage downloads the + # consolidated conda packages (drop_ConsolidateConda_ConsolidateArtifacts) and + # enforces the hard completeness + #706 pairing gate. It publishes NOTHING; it is + # the safe place to prove the conda set is release-ready. + - name: releaseConda + displayName: '[TEST] Validate Conda Release Readiness (does not publish)' + type: boolean + default: false + + # [TEST] Conda publish rehearsal. Wires the SAME anaconda-client publish path as + # the official pipeline for workflow testing. condaChannel is EMPTY by default and + # the publish step refuses to run without it, so this TEST pipeline can never push + # to the production 'microsoft' channel by accident. Point it at a personal/test + # Anaconda.org channel to rehearse the upload. + - name: publishToConda + displayName: '[TEST] Publish Conda Packages to Anaconda.org (rehearsal - set a TEST channel)' + type: boolean + default: false + + - name: condaChannel + displayName: '[TEST] Anaconda.org channel/org (leave empty to block publishing; set a TEST channel to rehearse)' + type: string + default: '' + # Variables variables: # Common variables @@ -42,6 +66,11 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # ANACONDA_API_TOKEN for the conda publish rehearsal lives in this variable group. + # Included ONLY when publishing so the validate-only path doesn't require it. + - ${{ if eq(parameters.publishToConda, true) }}: + - group: 'Anaconda Publishing' + # Select which consolidated artifact to download based on the target package. # Both are produced by the same build pipeline (definition 2199): # mssql-python -> drop_Consolidate_ConsolidateArtifacts @@ -422,3 +451,54 @@ extends: Write-Host "3. Verify symbols in SqlClientDrivers org (if published)" Write-Host "4. For PRODUCTION release, use official-release-pipeline.yml" Write-Host "=====================================" + + # ===================================================================== + # [TEST] CONDA RELEASE READINESS (decoupled; publishes nothing) + # ===================================================================== + # Runs when releaseConda=true OR publishToConda=true. Downloads the consolidated + # conda packages from build definition 2199 and enforces the release-time hard + # gate (exact count + #706 binding/companion pairing). dependsOn: [] keeps it + # independent of the dummy release stage. When publishToConda=true, a second + # releaseJob rehearses the Anaconda.org upload (to whatever TEST channel is + # supplied), but ONLY after this gate succeeds. + - ${{ if or(eq(parameters.releaseConda, true), eq(parameters.publishToConda, true)) }}: + - stage: ValidateCondaRelease + displayName: '[TEST] Validate & Publish Conda Release' + dependsOn: [] + jobs: + - job: ValidateConda + displayName: '[TEST] Validate consolidated conda packages' + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + steps: + - template: /OneBranchPipelines/steps/conda-release-step.yml@self + parameters: + labelPrefix: '[TEST] ' + + # [TEST] Conda publish rehearsal (releaseJob). Runs only when + # publishToConda=true and ONLY after the ValidateConda gate succeeds. The + # publish step refuses to run with an empty condaChannel, so this can + # never reach the production 'microsoft' channel by default. + - ${{ if eq(parameters.publishToConda, true) }}: + - job: PublishConda + displayName: '[TEST] Publish Conda Packages to Anaconda.org' + dependsOn: ValidateConda + templateContext: + type: releaseJob + pool: + type: windows + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' + steps: + - template: /OneBranchPipelines/steps/conda-publish-step.yml@self + parameters: + condaChannel: ${{ parameters.condaChannel }} + condaLabel: 'main' + labelPrefix: '[TEST] ' diff --git a/OneBranchPipelines/jobs/consolidate-conda-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-conda-artifacts-job.yml new file mode 100644 index 000000000..3fa9b3f7b --- /dev/null +++ b/OneBranchPipelines/jobs/consolidate-conda-artifacts-job.yml @@ -0,0 +1,125 @@ +# Consolidate Conda Artifacts Job Template +# Collects the per-platform conda packages (mssql-python binding + its companion +# mssql-python-odbc) that each build leg staged under conda// and gathers +# them into a single conda/ tree for the release pipeline to publish. +# +# BEST-EFFORT (build pipeline): conda is a downstream repackage of the ESRP-signed +# wheels and must NEVER block the primary wheel deliverable, so a missing/short set +# only WARNS here. The HARD count gate lives in the release pipeline (which refuses +# to publish an incomplete conda set), symmetric with how the wheel/odbc drops are +# best-effort collected in the build and gated at release time. +# +# Expected packages (validated conda subdirs). The per-Python BINDING (mssql-python) +# is emitted by each build leg. The version-locked COMPANION (mssql-python-odbc) is +# still emitted per-Python by the macOS/Linux legs, but on Windows it is built ONCE +# (Python-agnostic -- Lib\site-packages is not version-pathed) in the ODBC_BuildAll +# stage and collected here from that stage's drop instead of once per Python leg: +# win-64 : 5 py x mssql-python + 1 companion (once, ODBC_BuildAll) = 6 +# osx-64 : 5 py x 2 (Intel Mac, cross-built under Rosetta 2) = 10 +# osx-arm64 : 5 py x 2 (Apple Silicon, native) = 10 +# linux-64 : 5 py x 2 (glibc x86_64 host, native) = 10 +# linux-aarch64 : 5 py x 2 (x86_64 host + QEMU, best-effort) = 10 +# ------------------------------------------------------------------------------ +# TOTAL (PyPI parity minus win-arm64 + musllinux) = 46 +# win-arm64 (no import-validation host on x64) and musllinux (no conda musl subdir) +# are intentionally NOT conda-built. This job is BEST-EFFORT and never hard-fails on +# a short set; the release pipeline's conda-release-step enforces the hard gate +# (required subdirs present + #706 pairing) before anything is published. +parameters: + - name: oneBranchType + type: string + default: 'Official' + +jobs: + - job: ConsolidateArtifacts + displayName: 'Consolidate All Conda Packages' + condition: succeeded() + + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'ubuntu-latest' + + variables: + # Consolidation only moves files; no binaries to scan. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + + steps: + - checkout: self + fetchDepth: 1 + + # The conda packages are staged INSIDE the mssql-python build-leg artifacts + # (drop_Win_*, drop_MacOS_*, drop_Linux_*) under conda//, EXCEPT the + # win-64 companion (mssql-python-odbc), which is built ONCE in ODBC_BuildAll and + # rides its drop under conda/win-64/. Scope the download to those stages so every + # leg's conda output plus the once-built Windows companion is gathered in one + # place. The wheels ride along in the download but are ignored below (we only + # pick *.conda / *.tar.bz2). + - task: DownloadPipelineArtifact@2 + displayName: 'Download All Platform Artifacts' + inputs: + buildType: 'current' + itemPattern: | + drop_Win_*/** + drop_MacOS_*/** + drop_Linux_*/** + drop_ODBC_BuildAll_*/** + targetPath: '$(Pipeline.Workspace)/all-artifacts' + + - bash: | + set -e + echo "Collecting conda packages (preserving / layout)..." + mkdir -p $(ob_outputDirectory)/conda + + # Copy every mssql-python* conda package into conda//. The glob + # 'mssql-python*' matches BOTH the binding (mssql-python-*) and the + # companion (mssql-python-odbc-*); each build leg wrote them under a + # conda// folder, so the parent dir name IS the target subdir. + found=0 + while IFS= read -r p; do + subdir=$(basename "$(dirname "$p")") + mkdir -p "$(ob_outputDirectory)/conda/$subdir" + cp -v "$p" "$(ob_outputDirectory)/conda/$subdir/" + found=1 + done < <(find $(Pipeline.Workspace)/all-artifacts -type f \( -name 'mssql-python*.conda' -o -name 'mssql-python*.tar.bz2' \)) + + echo "" + echo "Consolidated conda tree:" + find $(ob_outputDirectory)/conda -type f | sort + + PKG_COUNT=$(find $(ob_outputDirectory)/conda -type f \( -name '*.conda' -o -name '*.tar.bz2' \) | wc -l) + echo "" + echo "Per-subdir conda package counts:" + for d in $(ob_outputDirectory)/conda/*/; do + [ -d "$d" ] || continue + sub=$(basename "$d") + n=$(find "$d" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) | wc -l) + printf ' %-14s %s\n' "$sub" "$n" + done + echo "Total conda package count: $PKG_COUNT (full PyPI-parity set = 46)" + + # BEST-EFFORT: warn only, never exit non-zero — a conda hiccup on any leg + # must not fail this build or block the wheel release. The release pipeline's + # conda-release-step enforces the hard gate (required subdirs + #706 pairing) + # before anything is published. + if [ "$found" != "1" ]; then + echo "##vso[task.logissue type=warning]No conda packages found in the build-leg artifacts." + else + echo "Collected $PKG_COUNT conda package(s) (best-effort; release-time gate enforces completeness)." + fi + displayName: 'Consolidate conda packages' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Consolidated Conda Artifacts' + inputs: + targetPath: '$(ob_outputDirectory)' + # Distinct name so it does not collide with the wheel consolidate artifact + # (drop_Consolidate_ConsolidateArtifacts) or the odbc one + # (drop_ConsolidateOdbc_ConsolidateArtifacts) in the same run. Matches the + # OneBranch auto-name for a stage named `ConsolidateConda`. + artifact: 'drop_ConsolidateConda_ConsolidateArtifacts' + publishLocation: 'pipeline' diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 3198908bc..4df81e7f2 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -32,6 +32,31 @@ parameters: type: boolean default: false # Safety: Default to false to prevent accidental releases + # Conda release readiness. When true, a DECOUPLED stage downloads the consolidated + # conda packages (drop_ConsolidateConda_ConsolidateArtifacts) and enforces the hard + # completeness + #706 pairing gate. It is independent of the wheel PyPI release, so + # a conda check can never block a wheel release. + - name: releaseConda + displayName: 'Validate Conda Release Readiness (does not publish)' + type: boolean + default: false + + # Conda PRODUCTION publish to Anaconda.org. When true, AFTER the readiness gate + # (ValidateConda) passes, a releaseJob uploads the consolidated conda packages via + # anaconda-client (ESRP has no Conda ContentType). The version-locked companion + # (mssql-python-odbc) is uploaded BEFORE the binding (mssql-python) per #706. + # Decoupled from the wheel release; default false so a normal release never + # touches Anaconda. + - name: publishToConda + displayName: 'Publish Conda Packages to Anaconda.org (PRODUCTION)' + type: boolean + default: false + + - name: condaChannel + displayName: 'Anaconda.org channel/org to publish conda packages to' + type: string + default: 'microsoft' + # Variables variables: # Common variables @@ -42,6 +67,12 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # ANACONDA_API_TOKEN for conda publishing lives in this org-scoped variable group. + # Included ONLY when actually publishing, so a normal wheel release doesn't require + # the group to exist. + - ${{ if eq(parameters.publishToConda, true) }}: + - group: 'Anaconda Publishing' + # Select which consolidated artifact to download/publish based on the target # package. Both are produced by the same build pipeline (definition 2199): # mssql-python -> drop_Consolidate_ConsolidateArtifacts @@ -404,3 +435,52 @@ extends: Write-Host "3. Verify package on PyPI: https://pypi.org/project/${{ parameters.releasePackage }}/" Write-Host "4. Verify symbols in SqlClientDrivers org (if published)" Write-Host "=====================================" + + # ===================================================================== + # CONDA RELEASE READINESS (decoupled; does not gate the wheel release) + # ===================================================================== + # Runs when releaseConda=true OR publishToConda=true. Downloads the consolidated + # conda packages from build definition 2199 and enforces the release-time hard + # gate (exact count + #706 binding/companion pairing). dependsOn: [] keeps it + # independent of ReleasePackages, so a conda problem can never block the wheel + # PyPI release. When publishToConda=true, a second releaseJob publishes to + # Anaconda.org, but ONLY after this gate succeeds. + - ${{ if or(eq(parameters.releaseConda, true), eq(parameters.publishToConda, true)) }}: + - stage: ValidateCondaRelease + displayName: 'Validate & Publish Conda Release' + dependsOn: [] + jobs: + - job: ValidateConda + displayName: 'Validate consolidated conda packages' + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + steps: + - template: /OneBranchPipelines/steps/conda-release-step.yml@self + + # Conda PRODUCTION publish (releaseJob). Runs only when publishToConda=true + # and ONLY after the ValidateConda gate succeeds, so we never upload an + # incomplete or mis-paired set. anaconda-client upload (ESRP has no Conda + # ContentType); companion-before-binding per #706. + - ${{ if eq(parameters.publishToConda, true) }}: + - job: PublishConda + displayName: 'Publish Conda Packages to Anaconda.org' + dependsOn: ValidateConda + templateContext: + type: releaseJob + isProduction: true + pool: + type: windows + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' + steps: + - template: /OneBranchPipelines/steps/conda-publish-step.yml@self + parameters: + condaChannel: ${{ parameters.condaChannel }} + condaLabel: 'main' diff --git a/OneBranchPipelines/scripts/.gitattributes b/OneBranchPipelines/scripts/.gitattributes new file mode 100644 index 000000000..dfdb8b771 --- /dev/null +++ b/OneBranchPipelines/scripts/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/OneBranchPipelines/scripts/build-conda-packages.ps1 b/OneBranchPipelines/scripts/build-conda-packages.ps1 new file mode 100644 index 000000000..5863b82df --- /dev/null +++ b/OneBranchPipelines/scripts/build-conda-packages.ps1 @@ -0,0 +1,228 @@ +<# +.SYNOPSIS + Build and validate the self-contained mssql-python conda package (which vendors the + ODBC Driver 18 payload) from prebuilt (ESRP-signed) wheels, fully offline. + +.DESCRIPTION + Repackages the wheels produced by build definition 2199 into conda packages using + conda-build, then proves the recipes are correct by solving a fresh environment + from the freshly built local channel and importing both packages. + + Runs on the OneBranch Windows 1ES pool (or locally). Builds the win_amd64 slice + for every Python version detected among the mssql_python wheels. Other platforms + (linux-*, osx-*, win_arm64) must be built on matching agents in a follow-up, the + same way the wheel build matrix fans out. + +.PARAMETER WheelsDir + Directory containing ALL downloaded wheels (both packages, all platforms/pythons). + +.PARAMETER RecipeRoot + Path to the repo's conda/ directory (contains mssql-python/ and mssql-python-odbc/). + +.PARAMETER OutputDir + Space-free working/output directory (conda croot, Miniforge install, built pkgs). + +.PARAMETER MssqlPythonVersion + Version to stamp on the mssql-python conda package (e.g. 1.13.0). + +.PARAMETER OdbcVersion + Version to stamp on the mssql-python-odbc conda package (e.g. 18.6.2.1). + +.PARAMETER PythonVersions + Optional comma-separated list (e.g. "3.11,3.12"). Empty = auto-detect from wheels. + +.PARAMETER CondaSubdir + Optional target subdir (e.g. win-arm64) to CROSS-target via CONDA_SUBDIR instead of + the host's native subdir. Empty = build the host's native subdir (win-64). Cross- + targeting only yields a VALIDATED package when the host can run the target Python for + the import check, so it is left unset for the native win-64 leg. + +.PARAMETER Package + Which package(s) to build: + 'all' - companion (ONCE) + binding (per-Python) [default] + 'odbc' - ONLY the Python-agnostic companion, built ONCE (ODBC_BuildAll stage); + validated by importing it under each target Python. + 'binding' - ONLY the per-Python binding; the companion is seeded from + -DriverCondaDir into the local channel so the version-locked + `mssql-python-odbc ==` dependency resolves for the solve/import. + +.PARAMETER DriverCondaDir + Folder holding a prebuilt companion .conda (mssql-python-odbc) under a / + layout, to seed into the local channel (binding mode) instead of rebuilding the + companion per-Python. Empty in 'all'/'odbc' mode. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string]$WheelsDir, + [Parameter(Mandatory = $true)][string]$RecipeRoot, + [Parameter(Mandatory = $true)][string]$OutputDir, + [Parameter(Mandatory = $true)][string]$MssqlPythonVersion, + [Parameter(Mandatory = $true)][string]$OdbcVersion, + [string]$PythonVersions = "", + [string]$CondaSubdir = "", + [ValidateSet('all', 'odbc', 'binding')] + [string]$Package = 'all', + [string]$DriverCondaDir = "" +) + +$ErrorActionPreference = 'Stop' + +function Assert-LastExit([string]$Message) { + if ($LASTEXITCODE -ne 0) { + Write-Error "FAILED (exit $LASTEXITCODE): $Message" + exit 1 + } +} + +Write-Host "==================== conda build inputs ====================" +Write-Host "WheelsDir : $WheelsDir" +Write-Host "RecipeRoot : $RecipeRoot" +Write-Host "OutputDir : $OutputDir" +Write-Host "MssqlPythonVersion : $MssqlPythonVersion" +Write-Host "OdbcVersion : $OdbcVersion" +Write-Host "PythonVersions : $(if ($PythonVersions) { $PythonVersions } else { '(auto-detect)' })" +Write-Host "CondaSubdir : $(if ($CondaSubdir) { $CondaSubdir } else { '(native)' })" +Write-Host "============================================================" + +New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null +$bld = Join-Path $OutputDir 'bld' +New-Item -ItemType Directory -Force -Path $bld | Out-Null + +# --------------------------------------------------------------------------- +# 1. Locate conda, or install Miniforge3 (conda-forge defaults, no license issues) +# --------------------------------------------------------------------------- +$conda = (Get-Command conda -ErrorAction SilentlyContinue).Source +if (-not $conda) { + Write-Host "=== conda not found on PATH; installing Miniforge3 ===" + $installer = Join-Path $OutputDir 'Miniforge3-Windows-x86_64.exe' + $forgeDir = Join-Path $OutputDir 'miniforge' + $url = 'https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Windows-x86_64.exe' + Write-Host "Downloading $url" + Invoke-WebRequest -Uri $url -OutFile $installer + # NSIS silent install; /D (target dir) MUST be last and unquoted. + Start-Process -FilePath $installer -ArgumentList '/S', '/InstallationType=JustMe', '/AddToPath=0', "/D=$forgeDir" -Wait + $conda = Join-Path $forgeDir 'Scripts\conda.exe' +} +if (-not (Test-Path $conda)) { + Write-Error "conda not available at '$conda' after install attempt." + exit 1 +} +Write-Host "Using conda: $conda" +& $conda --version +Assert-LastExit "conda --version" + +# --------------------------------------------------------------------------- +# 2. Install conda-build (pinned to the stable pre-26 series) +# --------------------------------------------------------------------------- +# Pin conda-build<26: the bleeding-edge 26.7.0 crashes with an internal +# "An unexpected error has occurred" during the LOCAL packaging phase (right +# after "Fixing permissions"); 26.7.1 is not yet released. The mature 25.x +# series builds these recipes cleanly and supports every key we use. +# NOTE: anaconda-client is intentionally NOT installed here — this script only +# builds + validates (it never runs `anaconda upload`). Publishing installs its +# own anaconda-client in conda-publish-step.yml. Keeping it out of the build env +# also drops the anaconda-auth conda plugin, which the crash report fingered. +Write-Host "=== installing conda-build (<26) ===" +& $conda install -y -n base "conda-build<26" +Assert-LastExit "conda install conda-build<26" + +# --------------------------------------------------------------------------- +# 3. Determine which Python versions to build (win_amd64 mssql_python wheels) +# --------------------------------------------------------------------------- +if ([string]::IsNullOrWhiteSpace($PythonVersions)) { + $pyvers = Get-ChildItem -Path $WheelsDir -Filter 'mssql_python-*win_amd64.whl' | + ForEach-Object { if ($_.Name -match 'cp3(\d+)') { "3.$($Matches[1])" } } | + Sort-Object -Unique +} +else { + $pyvers = $PythonVersions.Split(',') | ForEach-Object { $_.Trim() } | Where-Object { $_ } +} +if (-not $pyvers) { + Write-Error "No win_amd64 mssql_python wheels found in '$WheelsDir' to determine Python versions." + exit 1 +} +Write-Host "Building conda packages for Python versions: $($pyvers -join ', ')" + +# --------------------------------------------------------------------------- +# 4. Export the environment consumed by the recipes (jinja + build scripts) +# --------------------------------------------------------------------------- +$env:WHEELS_DIR = $WheelsDir +$env:MSSQL_PYTHON_VERSION = $MssqlPythonVersion +$env:MSSQL_ODBC_VERSION = $OdbcVersion + +# CROSS-target a non-native subdir when requested: conda-build and the verify env's +# `conda create` both honor CONDA_SUBDIR, so the packages are stamped for $CondaSubdir +# and the import check runs the target Python (via Rosetta 2 / QEMU on an emulating +# host). Empty = build the host's native subdir. +if ($CondaSubdir) { + $env:CONDA_SUBDIR = $CondaSubdir + Write-Host "Cross-targeting conda subdir: CONDA_SUBDIR=$($env:CONDA_SUBDIR)" +} + +# --------------------------------------------------------------------------- +# 5. Build the self-contained mssql-python package (per Python). The recipe vendors +# the ODBC Driver 18 payload by extracting the mssql-python-odbc wheel into its +# own site-packages, so there is NO separate companion package to build. +# --------------------------------------------------------------------------- +$bindRecipe = Join-Path $RecipeRoot 'mssql-python' + +if ($Package -eq 'odbc') { + Write-Host "NOTE: -Package odbc is a no-op in the self-contained model (the ODBC payload" + Write-Host "is vendored INTO mssql-python; there is no separate companion). Nothing to build." +} +else { + foreach ($py in $pyvers) { + Write-Host "=== [py $py] build mssql-python (self-contained: vendors the ODBC payload) ===" + & $conda build $bindRecipe --python $py --no-test --no-anaconda-upload --output-folder $bld + Assert-LastExit "conda build mssql-python (py $py)" + } +} + +# --------------------------------------------------------------------------- +# 6. Index the freshly built local channel +# --------------------------------------------------------------------------- +Write-Host "=== indexing local channel ===" +& $conda index $bld +Assert-LastExit "conda index" + +# --------------------------------------------------------------------------- +# 7. Validate: solve a fresh env from the local channel and import the package. +# Proves azure-identity + the folded-in openssl/krb5 deps resolve AND that the +# repackaged native binding imports with its vendored ODBC payload (driver loads +# at import). +# --------------------------------------------------------------------------- +$localChannel = "file:///" + ($bld -replace '\\', '/') +if ($Package -eq 'odbc') { + Write-Host "NOTE: -Package odbc is a no-op in the self-contained model; nothing to validate." +} +else { + foreach ($py in $pyvers) { + $envName = "verify_" + ($py -replace '\.', '') + Write-Host "=== [py $py] create verify env from local channel ===" + # -c microsoft (ahead of conda-forge) so azure-core/azure-identity/msal resolve from the + # lean `microsoft` channel, NOT conda-forge whose azure-core recipe over-declares flask/six + # -> celery/boto3/botocore (~9 MB); see conda-forge/azure-core-feedstock#71. + # --strict-channel-priority keeps the freshly built local package authoritative. + & $conda create -y -n $envName -c $localChannel -c microsoft -c conda-forge --strict-channel-priority --override-channels "python=$py" mssql-python + Assert-LastExit "conda create verify env (py $py)" + + Write-Host "=== [py $py] import mssql_python + prove the vendored ODBC payload is present ===" + & $conda run -n $envName python -c "import mssql_python; print('BINDING_OK', mssql_python.__version__)" + Assert-LastExit "import mssql_python (py $py)" + & $conda run -n $envName python -c "import mssql_python_odbc; print('ODBC_PAYLOAD_OK', mssql_python_odbc.__version__)" + Assert-LastExit "import mssql_python_odbc (py $py)" + + Write-Host "=== [py $py] DB-less driver-load proof (real ODBC driver must load, not just the shim) ===" + & $conda run -n $envName python (Join-Path $RecipeRoot 'driver_load_probe.py') + Assert-LastExit "driver-load proof (py $py)" + + Write-Host "=== [py $py] confirm resolved dependencies ===" + & $conda list -n $envName | Select-String -Pattern 'azure-identity|mssql-python|openssl|krb5' + } +} + +Write-Host "==================== built conda artifacts ====================" +Get-ChildItem -Path $bld -Recurse -Include *.conda, *.tar.bz2 | +Where-Object { $_.Name -like 'mssql-python*' } | +ForEach-Object { Write-Host " $($_.FullName)" } +Write-Host "CONDA_BUILD_OK" diff --git a/OneBranchPipelines/scripts/build-conda-packages.sh b/OneBranchPipelines/scripts/build-conda-packages.sh new file mode 100644 index 000000000..caedc44c0 --- /dev/null +++ b/OneBranchPipelines/scripts/build-conda-packages.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +# Build + validate the self-contained mssql-python conda package from prebuilt +# (ESRP-signed) wheels, fully offline via a local --find-links dir. The package +# VENDORS the ODBC Driver 18 payload -- the recipe extracts BOTH the code wheel and +# the mssql-python-odbc wheel into one site-packages -- so there is NO separate +# conda package (the v1.11.0 model). +# ============================================================================ +# Bash port of build-conda-packages.ps1 for the macOS and Linux build legs. +# conda-build provisions a per-subdir HOST env and installs the matching wheel +# (see conda/*/build.sh). It runs NATIVELY for linux-64 and osx-64, under QEMU +# binfmt for linux-aarch64, and as a CROSS-build for osx-arm64 on the Intel macOS +# agent (there is no reverse Rosetta, so the arm64 Python is never executed -- +# conda/*/build.sh extract the universal2 wheel without Python and the section-7 +# runtime import is skipped; the pipeline's static arm64-slice audit stands in). +# +# Args: +# $1 WheelsDir find-links dir holding the mssql-python + mssql-python-odbc wheels +# $2 RecipeRoot repo conda/ dir (mssql-python/ + mssql-python-odbc/) +# $3 OutputDir space-free work/output dir (Miniforge + croot + built pkgs) +# $4 MssqlPythonVersion version to stamp on mssql-python +# $5 OdbcVersion version to stamp on mssql-python-odbc +# $6 PythonVersions optional comma-separated (e.g. "3.11,3.12"); empty = auto-detect +# $7 CondaSubdir optional target subdir (e.g. osx-64, osx-arm64, +# linux-aarch64) to CROSS-target via CONDA_SUBDIR; empty = +# build the host's native subdir. The section-7 runtime +# import validation requires the host to be able to RUN the +# target's Python -- true natively, under Rosetta 2 (osx-64 +# on Apple Silicon) and under QEMU binfmt (linux-aarch64 on +# x86_64). For osx-arm64 on the Intel agent it is NOT, so +# that leg auto-skips the import (static arch audit stands in). +set -euo pipefail + +WheelsDir="${1:?WheelsDir required}" +RecipeRoot="${2:?RecipeRoot required}" +OutputDir="${3:?OutputDir required}" +MssqlPythonVersion="${4:?MssqlPythonVersion required}" +OdbcVersion="${5:?OdbcVersion required}" +PythonVersions="${6:-}" +CondaSubdir="${7:-}" + +echo "==================== conda build inputs ====================" +echo "WheelsDir : $WheelsDir" +echo "RecipeRoot : $RecipeRoot" +echo "OutputDir : $OutputDir" +echo "MssqlPythonVersion : $MssqlPythonVersion" +echo "OdbcVersion : $OdbcVersion" +echo "PythonVersions : ${PythonVersions:-(auto-detect)}" +echo "CondaSubdir : ${CondaSubdir:-(native)}" +echo "============================================================" + +mkdir -p "$OutputDir" +bld="$OutputDir/bld" +mkdir -p "$bld" + +# --------------------------------------------------------------------------- +# 1. Locate conda, or install Miniforge3 (conda-forge defaults) for THIS platform +# --------------------------------------------------------------------------- +conda="$(command -v conda || true)" +# Reuse an existing Miniforge install if a previous run already created one. On +# macOS the universal2 build invokes this script once per subdir (osx-64 AND +# osx-arm64) on the SAME agent, sharing $OutputDir; each run is a fresh shell so +# `command -v conda` is empty even though miniforge/ already exists. Without this +# guard the second run re-runs the installer into the existing dir and fails with +# "File or directory already exists: .../conda-bld/miniforge". +if [ -z "$conda" ] && [ -x "$OutputDir/miniforge/bin/conda" ]; then + echo "=== reusing existing Miniforge3 at $OutputDir/miniforge ===" + conda="$OutputDir/miniforge/bin/conda" +fi +if [ -z "$conda" ]; then + echo "=== conda not found on PATH; installing Miniforge3 ===" + os="$(uname -s)"; arch="$(uname -m)" + case "$os-$arch" in + Darwin-arm64) mf="Miniforge3-MacOSX-arm64.sh" ;; + Darwin-x86_64) mf="Miniforge3-MacOSX-x86_64.sh" ;; + Linux-x86_64) mf="Miniforge3-Linux-x86_64.sh" ;; + Linux-aarch64) mf="Miniforge3-Linux-aarch64.sh" ;; + *) echo "ERROR: unsupported platform '$os-$arch' for Miniforge" >&2; exit 1 ;; + esac + forgeDir="$OutputDir/miniforge" + installer="$OutputDir/$mf" + url="https://github.com/conda-forge/miniforge/releases/latest/download/$mf" + echo "Downloading $url" + curl -fL "$url" -o "$installer" + # -u = update/reuse an existing target dir instead of erroring, in case a prior + # run left a partial miniforge/ behind that failed the reuse check above. + bash "$installer" -b -u -p "$forgeDir" + conda="$forgeDir/bin/conda" +fi +if ! "$conda" --version >/dev/null 2>&1; then + echo "ERROR: conda not available at '$conda' after install attempt." >&2 + exit 1 +fi +echo "Using conda: $conda" +"$conda" --version + +# --------------------------------------------------------------------------- +# 2. Install conda-build (pinned to the stable pre-26 series) +# --------------------------------------------------------------------------- +# Pin conda-build<26: the bleeding-edge 26.7.0 crashes with an internal +# "An unexpected error has occurred" during the LOCAL packaging phase (right +# after "Fixing permissions"); 26.7.1 is not yet released. The mature 25.x +# series builds these recipes cleanly and supports every key we use. +# NOTE: anaconda-client is intentionally NOT installed here — this script only +# builds + validates (it never runs `anaconda upload`). Publishing installs its +# own anaconda-client in conda-publish-step.yml. Keeping it out of the build env +# also drops the anaconda-auth conda plugin, which the crash report fingered. +echo "=== installing conda-build (<26) ===" +"$conda" install -y -n base "conda-build<26" + +# --------------------------------------------------------------------------- +# 3. Determine which Python versions to build (auto-detect from mssql_python wheels) +# --------------------------------------------------------------------------- +if [ -z "$PythonVersions" ]; then + pyvers="$(ls "$WheelsDir"/mssql_python-*.whl 2>/dev/null \ + | grep -v 'mssql_python_odbc' \ + | sed -nE 's/.*-cp3([0-9]+)-.*/3.\1/p' | sort -u)" +else + pyvers="$(echo "$PythonVersions" | tr ',' '\n' | sed 's/[[:space:]]//g' | grep -v '^$')" +fi +if [ -z "$pyvers" ]; then + echo "ERROR: no mssql_python wheels in '$WheelsDir' to determine Python versions." >&2 + exit 1 +fi +echo "Building conda packages for Python versions: $(echo "$pyvers" | tr '\n' ' ')" + +# --------------------------------------------------------------------------- +# 4. Export the environment consumed by the recipes (jinja + build scripts) +# --------------------------------------------------------------------------- +export WHEELS_DIR="$WheelsDir" +export MSSQL_PYTHON_VERSION="$MssqlPythonVersion" +export MSSQL_ODBC_VERSION="$OdbcVersion" + +# Cross-subdir builds: force conda-build AND the verify `conda create` to target the +# requested subdir instead of the host's native one. Both honor CONDA_SUBDIR, so the +# packages are stamped for $CondaSubdir. The section-7 import validation solves that +# subdir and runs the target Python where the host can execute it (natively, under +# Rosetta 2 for osx-64, or under QEMU binfmt for linux-aarch64); on the osx-arm64 +# cross-build (Intel agent, no reverse Rosetta) section 7 auto-detects that the target +# Python can't run and skips the import. Left unset for a native build. +if [ -n "$CondaSubdir" ]; then + export CONDA_SUBDIR="$CondaSubdir" + echo "Cross-targeting conda subdir: CONDA_SUBDIR=$CONDA_SUBDIR" + # Emulated aarch64 cross-build: the verify env's target-arch Python (section 7) + # runs under qemu-user. Point qemu at the aarch64 glibc loader/libs (installed via + # libc6-arm64-cross on the leg) so it can find /lib/ld-linux-aarch64.so.1. Only the + # emulated aarch64 leg has this dir; elsewhere the var is a harmless no-op. + case "$CONDA_SUBDIR" in + *aarch64) + if [ -d /usr/aarch64-linux-gnu ]; then + export QEMU_LD_PREFIX="${QEMU_LD_PREFIX:-/usr/aarch64-linux-gnu}" + echo "Set QEMU_LD_PREFIX=$QEMU_LD_PREFIX for emulated aarch64 verify" + fi + ;; + esac +fi + +# --------------------------------------------------------------------------- +# 5. Build companion FIRST, then the binding, for each Python version +# --------------------------------------------------------------------------- +bindRecipe="$RecipeRoot/mssql-python" +for py in $pyvers; do + echo "=== [py $py] build mssql-python (self-contained: vendors the ODBC payload) ===" + "$conda" build "$bindRecipe" --python "$py" --no-test --no-anaconda-upload --output-folder "$bld" +done + +# --------------------------------------------------------------------------- +# 6. Index the freshly built local channel +# --------------------------------------------------------------------------- +echo "=== indexing local channel ===" +"$conda" index "$bld" + +# --------------------------------------------------------------------------- +# 7. Validate: solve a fresh env from the local channel and import the package. +# Proves azure-identity + the folded-in openssl/krb5 deps resolve AND that the +# repackaged native binding imports with its vendored ODBC payload (driver loads +# at import). +# --------------------------------------------------------------------------- +for py in $pyvers; do + envName="verify_${py//./}" + echo "=== [py $py] create verify env from local channel ===" + # -c microsoft (ahead of conda-forge) so azure-core/azure-identity/msal resolve from the + # lean `microsoft` channel, NOT conda-forge whose azure-core recipe over-declares flask/six + # -> celery/boto3/botocore (~9 MB); see conda-forge/azure-core-feedstock#71. + # --strict-channel-priority keeps the freshly built local package authoritative. + "$conda" create -y -n "$envName" -c "$bld" -c microsoft -c conda-forge --strict-channel-priority --override-channels "python=$py" mssql-python + # On a non-emulated cross-build (osx-arm64 on an Intel agent -- no reverse Rosetta) + # the solved target Python cannot execute here, so the runtime import / driver-load + # proof is impossible. The pipeline's static arm64-slice audit (lipo/otool/file on + # the arm64 Mach-O payload) is the stand-in for it on that leg -- identical assurance + # to the shipping PyPI universal2 arm64 slice, which is likewise only static-checked. + # Native and QEMU-emulated legs run the real import + driver-load probe below. + if ! "$conda" run -n "$envName" python -c "import sys" >/dev/null 2>&1; then + echo "=== [py $py] target Python not executable on host ($(uname -s)/$(uname -m), CONDA_SUBDIR=${CONDA_SUBDIR:-native}); skipping runtime import -- static arch audit covers this cross leg. ===" + continue + fi + echo "=== [py $py] import mssql_python + prove the vendored ODBC payload is present ===" + "$conda" run -n "$envName" python -c "import mssql_python; print('BINDING_OK', mssql_python.__version__)" + "$conda" run -n "$envName" python -c "import mssql_python_odbc; print('ODBC_PAYLOAD_OK', mssql_python_odbc.__version__)" + echo "=== [py $py] DB-less driver-load proof (real ODBC driver must load, not just the shim) ===" + "$conda" run -n "$envName" python "$RecipeRoot/driver_load_probe.py" + # Live Encrypt=yes TLS gate -- forces the driver to dlopen its OpenSSL backend + # (libssl/libcrypto), which the DB-less Encrypt=no probe above NEVER exercises. + # Runs (BLOCKING) only when CONDA_TLS_PROBE_CONN points at a reachable server; + # otherwise it SKIPS loudly (it never silently passes). CAVEAT: this is + # conclusive ONLY on a minimal base with NO system OpenSSL -- a system libssl + # lets the driver's dlopen fall through and MASK an unreachable conda + # /lib copy (exactly what full CI agents hide). The masking-IMMUNE guard + # is eng/scripts/audit_bundled_binaries.py, which reads the RUNPATH bytes and + # requires an $ORIGIN/.. climb regardless of any system libs; this gate is the + # complementary end-to-end backstop for a minimal-base leg. + if [ -n "${CONDA_TLS_PROBE_CONN:-}" ]; then + echo "=== [py $py] live Encrypt=yes TLS gate (OpenSSL backend must be reachable) ===" + "$conda" run -n "$envName" python "$RecipeRoot/tls_connect_probe.py" + else + echo "=== [py $py] Encrypt=yes TLS gate SKIPPED (set CONDA_TLS_PROBE_CONN on a minimal-base leg to enable) ===" + fi + echo "=== [py $py] confirm resolved dependencies ===" + "$conda" list -n "$envName" | grep -E 'azure-identity|mssql-python|openssl|krb5' || true +done + +echo "==================== built conda artifacts ====================" +find "$bld" -type f \( -name 'mssql-python*.conda' -o -name 'mssql-python*.tar.bz2' \) -print +echo "CONDA_BUILD_OK" diff --git a/OneBranchPipelines/stages/build-linux-single-stage.yml b/OneBranchPipelines/stages/build-linux-single-stage.yml index da258e0bf..712291852 100644 --- a/OneBranchPipelines/stages/build-linux-single-stage.yml +++ b/OneBranchPipelines/stages/build-linux-single-stage.yml @@ -32,6 +32,12 @@ parameters: - name: installOdbcWheel type: boolean default: false + # Conda: build + validate the linux-64 conda package on the glibc x86_64 HOST + # (not in the container) from this leg's manylinux wheels. Only meaningful for + # manylinux_2_28 x86_64 (no conda musl target; aarch64 host is x86_64+QEMU). + - name: buildConda + type: boolean + default: false stages: - stage: ${{ parameters.stageName }} @@ -103,7 +109,11 @@ stages: displayName: 'Setup and start Docker daemon' - script: | - sudo apt-get install -y qemu-user-static + # qemu-user-static: run aarch64 ELF binaries on the x86_64 host. + # libc6-arm64-cross: the aarch64 glibc runtime (loader + libc/libm/...) + # under /usr/aarch64-linux-gnu so qemu can resolve /lib/ld-linux-aarch64.so.1 + # for the emulated aarch64 conda build/verify (QEMU_LD_PREFIX points here). + sudo apt-get install -y qemu-user-static libc6-arm64-cross displayName: 'Enable QEMU (for aarch64)' - script: | @@ -447,7 +457,62 @@ stages: echo "✓ Containers cleaned up" displayName: 'Cleanup containers' condition: always() # Always run cleanup, even if build/test fails - + + # ========================= + # CONDA PACKAGES (linux-64 — glibc x86_64 host, native) + # ========================= + # Repackage the manylinux_2_28 x86_64 wheels (built in the container, now + # on the host at $(Build.SourcesDirectory)/dist via the bind mount) into + # linux-64 conda packages. conda-build runs on the glibc x86_64 HOST + # (native, no container, no QEMU) and pip-installs the manylinux wheels, + # which are compatible with the Ubuntu host's newer glibc. Non-blocking: + # conda never fails the wheel build. + - ${{ if and(parameters.buildConda, parameters.installOdbcWheel, eq(parameters.arch, 'x86_64')) }}: + - template: /OneBranchPipelines/steps/conda-build-validate-step-posix.yml@self + parameters: + condaSubdir: 'linux-64' + mssqlWheelDir: '$(Build.SourcesDirectory)/dist' + odbcWheelDir: '$(Build.SourcesDirectory)/odbc_wheels' + odbcWheelFilter: 'mssql_python_odbc-*manylinux_2_28_x86_64.whl' + + # ========================= + # CONDA PACKAGES (linux-aarch64 — cross-target via QEMU binfmt) + # ========================= + # PyPI ships a separate manylinux_2_28_aarch64 wheel; conda needs a matching + # linux-aarch64 package. The agent host is x86_64, so we CROSS-target + # linux-aarch64 with CONDA_SUBDIR=linux-aarch64: the base conda stays x86_64 + # (native, fast) and only the verify env's aarch64 Python runs under QEMU + # user-mode emulation (binfmt registered below) for the import/driver-load + # check. The aarch64 mssql-python wheel is manylinux_2_28 (its glibc floor + # matches the recipe's __glibc>=2.28). Because this leans on emulation it is + # BEST-EFFORT (continueOnError): a QEMU hiccup drops the linux-aarch64 conda + # packages for the run but never fails the wheel build. Release-time gating + # REQUIRES linux-aarch64 by default (PyPI parity), with a documented escape + # hatch: if a green run shows the emulated leg is flaky, drop it from + # requiredSubdirs in conda-release-step (it stays in allowedSubdirs, so it is + # still accepted when present). + - ${{ if and(parameters.buildConda, parameters.installOdbcWheel, eq(parameters.arch, 'aarch64')) }}: + - bash: | + set -euo pipefail + # Register QEMU user-mode handlers with the persistent/fix-binary flag so + # the x86_64 host can execute aarch64 ELF binaries OUTSIDE a container + # (conda runs the aarch64 verify Python directly on the host). Same + # emulator the wheel build already relies on; best-effort. + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes || true + displayName: 'Register QEMU binfmt for host aarch64 execution' + continueOnError: true + - template: /OneBranchPipelines/steps/conda-build-validate-step-posix.yml@self + parameters: + condaSubdir: 'linux-aarch64' + condaTargetSubdir: 'linux-aarch64' + mssqlWheelDir: '$(Build.SourcesDirectory)/dist' + odbcWheelDir: '$(Build.SourcesDirectory)/odbc_wheels' + odbcWheelFilter: 'mssql_python_odbc-*manylinux_2_28_aarch64.whl' + # BEST-EFFORT override: this leg runs the aarch64 verify Python under + # QEMU emulation (see rationale above), so a QEMU hiccup must not fail + # the wheel build. Every NON-emulated conda leg is blocking by default. + continueOnError: true + # Publish artifacts to Azure Pipelines for downstream consumption # OneBranch requires specific artifact naming: drop__ - task: PublishPipelineArtifact@1 diff --git a/OneBranchPipelines/stages/build-macos-single-stage.yml b/OneBranchPipelines/stages/build-macos-single-stage.yml index 20962ce66..593821558 100644 --- a/OneBranchPipelines/stages/build-macos-single-stage.yml +++ b/OneBranchPipelines/stages/build-macos-single-stage.yml @@ -29,6 +29,11 @@ parameters: - name: installOdbcWheel type: boolean default: false + # Conda: build + validate the osx-arm64 conda package from this leg's wheels. + # Requires installOdbcWheel (the external odbc wheel is repackaged/solved too). + - name: buildConda + type: boolean + default: false stages: - stage: ${{ parameters.stageName }} @@ -258,7 +263,93 @@ stages: echo "Wheels after retag:" ls -lh dist/ displayName: 'Ensure universal2 platform tag' - + + # ========================= + # STATIC arm64 SLICE AUDIT (osx-arm64 cross-build guardrail) + # ========================= + # macos-latest is an INTEL Mac agent (the Apple-Silicon macOS image is a + # paused limited preview), so the osx-arm64 conda package below is CROSS-built + # and its arm64 runtime CANNOT be executed here. This BLOCKING audit is the + # stand-in for that missing runtime import: it proves the shipped payload + # actually contains real arm64 Mach-O slices. A thinned / x86_64-only binary + # stamped osx-arm64 fails the leg. Same assurance level as the PyPI universal2 + # arm64 slice, which PR-validation (also Intel) likewise only static-checks. + - ${{ if and(parameters.buildConda, parameters.installOdbcWheel) }}: + - bash: | + set -euo pipefail + fail=0 + echo "--- 1. binding native extension (ddbc_bindings) must carry an arm64 slice ---" + whl="$(ls "$(Build.SourcesDirectory)"/dist/mssql_python-*.whl 2>/dev/null | grep -v mssql_python_odbc | head -1)" + [ -n "$whl" ] || { echo "ERROR: no mssql-python wheel in dist/"; exit 1; } + tmp="$(mktemp -d)"; unzip -oq "$whl" -d "$tmp" + ext="$(find "$tmp" -name 'ddbc_bindings*.so' | head -1)" + [ -n "$ext" ] || { echo "ERROR: ddbc_bindings*.so not found in $(basename "$whl")"; exit 1; } + archs="$(lipo -archs "$ext" 2>/dev/null || true)" + echo " $(basename "$ext"): ${archs:-$(file -b "$ext")}" + if echo "$archs" | grep -qw arm64 || file -b "$ext" | grep -qw arm64; then + echo " OK: ddbc_bindings has an arm64 slice" + else + echo " ERROR: ddbc_bindings has NO arm64 slice (thinned/x86_64-only) -- refusing to stamp osx-arm64"; fail=1 + fi + echo "--- 2. companion driver dylibs must be arm64 Mach-O ---" + odbc="$(find "$(Pipeline.Workspace)/odbc_wheels" -name 'mssql_python_odbc-*macosx*universal2.whl' | head -1)" + [ -n "$odbc" ] || { echo "ERROR: no macOS odbc wheel under odbc_wheels/"; exit 1; } + otmp="$(mktemp -d)"; unzip -oq "$odbc" -d "$otmp" + armdir="$(find "$otmp" -path '*/libs/macos/arm64/lib' -type d | head -1)" + [ -n "$armdir" ] || { echo "ERROR: libs/macos/arm64/lib not found in odbc wheel"; exit 1; } + found=0 + for dy in "$armdir"/*.dylib; do + [ -e "$dy" ] || continue + found=1 + a="$(lipo -archs "$dy" 2>/dev/null || true)" + echo " $(basename "$dy"): ${a:-$(file -b "$dy")}" + if echo "$a" | grep -qw arm64 || file -b "$dy" | grep -qw arm64; then :; else + echo " ERROR: $(basename "$dy") has NO arm64 slice"; fail=1 + fi + done + [ "$found" = 1 ] || { echo "ERROR: no *.dylib under $armdir"; fail=1; } + [ "$fail" = 0 ] || { echo "STATIC arm64 SLICE AUDIT FAILED"; exit 1; } + echo "STATIC arm64 SLICE AUDIT PASSED" + displayName: 'Static arm64 slice audit (osx-arm64 cross-build guardrail)' + + # ========================= + # CONDA PACKAGE (osx-arm64 — CROSS-built on the Intel agent) + # ========================= + # macos-latest is an Intel Mac (the Apple-Silicon image is a paused preview) + # and there is no reverse Rosetta, so the arm64 Python cannot run here. We + # CROSS-build with CONDA_SUBDIR=osx-arm64: conda/*/build.sh extract the + # universal2 wheel's arm64 slice WITHOUT running Python, and the section-7 + # runtime import auto-skips (the static arm64-slice audit above is the gate). + # continueOnError makes the runtime import non-fatal on THIS leg only, + # mirroring the QEMU-emulated linux-aarch64 precedent. + - ${{ if and(parameters.buildConda, parameters.installOdbcWheel) }}: + - template: /OneBranchPipelines/steps/conda-build-validate-step-posix.yml@self + parameters: + condaSubdir: 'osx-arm64' + condaTargetSubdir: 'osx-arm64' + continueOnError: true + odbcWheelDir: '$(Pipeline.Workspace)/odbc_wheels' + odbcWheelFilter: 'mssql_python_odbc-*macosx*universal2.whl' + pythonVersions: ${{ parameters.pythonVersion }} + + # ========================= + # CONDA PACKAGE (osx-64 — NATIVE on the Intel agent) + # ========================= + # macos-latest is an Intel Mac, so osx-64 is the NATIVE build: conda-build, + # the verify env's x86_64 Python and the universal2 wheel all run for real + # here, so the import + DB-less driver-load proof is genuine (BLOCKING). PyPI + # ships ONE universal2 wheel for both Mac arches; conda has no universal2 + # subdir, so this native osx-64 package plus the cross-built osx-arm64 one + # above together close the Intel/Apple-Silicon parity gap vs. PyPI. + - ${{ if and(parameters.buildConda, parameters.installOdbcWheel) }}: + - template: /OneBranchPipelines/steps/conda-build-validate-step-posix.yml@self + parameters: + condaSubdir: 'osx-64' + condaTargetSubdir: 'osx-64' + odbcWheelDir: '$(Pipeline.Workspace)/odbc_wheels' + odbcWheelFilter: 'mssql_python_odbc-*macosx*universal2.whl' + pythonVersions: ${{ parameters.pythonVersion }} + # ========================= # ARTIFACT PUBLISHING # ========================= diff --git a/OneBranchPipelines/stages/build-odbc-all-stage.yml b/OneBranchPipelines/stages/build-odbc-all-stage.yml index 79b5d07a4..d9d24f38e 100644 --- a/OneBranchPipelines/stages/build-odbc-all-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-all-stage.yml @@ -146,6 +146,13 @@ stages: # so the check proves the actual ODBC DRIVER binary is packaged, not just a # sibling. Linux uses the version-major prefix so a driver MINOR bump won't # false-fail this gate. + # + # NOTE on libltdl.so.7: libodbcinst.so.2 NEEDs it, but on Linux it is NOT + # vendored in the wheel -- it is a documented system dependency (README: + # apt libltdl7 / dnf libtool-ltdl / apk libtool). macOS keeps its co-located + # libltdl.7.dylib. So this gate does NOT require libltdl.so.7 in the Linux + # payload; it only asserts the proprietary driver subtree is present and no + # foreign-platform binaries leaked. $expect = @{ 'win_amd64' = @{ Must = @('libs/windows/x64/', 'msodbcsql18.dll', 'mssql-auth.dll', 'libs/LICENSING'); Forbid = @('libs/linux/', 'libs/macos/', 'libs/windows/arm64/') } 'win_arm64' = @{ Must = @('libs/windows/arm64/', 'msodbcsql18.dll', 'mssql-auth.dll', 'libs/LICENSING'); Forbid = @('libs/linux/', 'libs/macos/', 'libs/windows/x64/') } @@ -185,6 +192,66 @@ stages: Write-Host "All 7 ODBC wheels verified: each ships only its own platform's driver." displayName: 'Assert each ODBC wheel is platform-correct' + # ========================= + # WIN-64 COMPANION CONDA (built ONCE, here with the wheels) + # ========================= + # Build the mssql-python-odbc CONDA package for win-64 ONCE, in the same + # single host that produces the py3-none- wheels -- the conda + # analog of the wheel. Windows conda site-packages (Lib\site-packages) is + # NOT Python-version-pathed, so one package serves every Python (the recipe + # has no `python` in host on Windows and bld.bat extracts the wheel with + # `tar`); conda-build emits a single win-64 package with no `pyXY` build + # string. This removes the previous redundancy where EVERY per-Python + # mssql-python leg rebuilt an identical companion. The mssql-python (binding) + # conda stays per-Python on its own legs and SEEDS this package to satisfy + # its version-locked `mssql-python-odbc ==` pin. + # + # macOS/Linux companion conda is unchanged (still built per-Python on those + # legs) because POSIX site-packages IS version-pathed -- that restructure is + # deferred; this change is Windows-only. + - powershell: | + $ErrorActionPreference = 'Stop' + # find-links dir holding ONLY the win_amd64 ODBC wheel: the recipe's + # bld.bat globs mssql_python_odbc-*-py3-none-win_*.whl, so keep the + # win-64 slice unambiguous (win_arm64 has no conda target here). + $links = Join-Path "$(Agent.TempDirectory)" 'odbc-conda-wheels' + New-Item -ItemType Directory -Force -Path $links | Out-Null + $whl = Get-ChildItem "$(ob_outputDirectory)\wheels" -Filter 'mssql_python_odbc-*win_amd64.whl' | Select-Object -First 1 + if (-not $whl) { Write-Error 'win_amd64 ODBC wheel not found for conda build'; exit 1 } + Copy-Item $whl.FullName -Destination $links -Force + if ($whl.Name -notmatch '^mssql_python_odbc-([^-]+)-') { Write-Error "Cannot parse version from $($whl.Name)"; exit 1 } + $odbcVer = $Matches[1] + Write-Host "Building win-64 mssql-python-odbc conda (once) from $($whl.Name) -> $odbcVer" + + # -Package odbc: build the single companion (no --python) and PROVE it is + # Python-agnostic by importing it under both ends of the supported range. + # MssqlPythonVersion is unused in odbc mode; pass odbcVer as a placeholder. + & "$(Build.SourcesDirectory)/OneBranchPipelines/scripts/build-conda-packages.ps1" ` + -WheelsDir $links ` + -RecipeRoot "$(Build.SourcesDirectory)/conda" ` + -OutputDir "$(Agent.TempDirectory)/odbc-conda-bld" ` + -MssqlPythonVersion $odbcVer ` + -OdbcVersion $odbcVer ` + -PythonVersions '3.10,3.14' ` + -Package odbc + if ($LASTEXITCODE -ne 0) { Write-Error "companion conda build+validate failed (exit $LASTEXITCODE)"; exit 1 } + + # Stage the single built package under conda/win-64/ so it rides the ODBC + # artifact; the mssql-python binding legs seed it for their validation + # solve, and ConsolidateConda collects it into the published conda set. + $condaOut = Join-Path "$(ob_outputDirectory)" 'conda' + $built = Get-ChildItem -Path (Join-Path "$(Agent.TempDirectory)/odbc-conda-bld" 'bld') -Recurse -Include *.conda, *.tar.bz2 | + Where-Object { $_.Name -like 'mssql-python-odbc-*' } + if (-not $built) { Write-Error 'No companion conda package produced'; exit 1 } + foreach ($p in $built) { + $subdir = Split-Path -Leaf $p.DirectoryName + $dest = Join-Path $condaOut $subdir + New-Item -ItemType Directory -Force -Path $dest | Out-Null + Copy-Item $p.FullName -Destination $dest -Force + Write-Host " staged $subdir/$($p.Name)" + } + displayName: 'Build mssql-python-odbc conda (win-64, once)' + # OneBranch requires artifact naming: drop__. # ConsolidateOdbc picks this up via its 'drop_ODBC_*' item pattern. - task: PublishPipelineArtifact@1 diff --git a/OneBranchPipelines/stages/build-windows-single-stage.yml b/OneBranchPipelines/stages/build-windows-single-stage.yml index db1c4b1eb..cd06bdb57 100644 --- a/OneBranchPipelines/stages/build-windows-single-stage.yml +++ b/OneBranchPipelines/stages/build-windows-single-stage.yml @@ -42,6 +42,11 @@ parameters: - name: signWindowsBinaries type: boolean default: true + # Conda: build + validate conda packages from this leg's wheels (x64 only). + # Requires installOdbcWheel (the external odbc wheel is repackaged/solved too). + - name: buildConda + type: boolean + default: false stages: - stage: ${{ parameters.stageName }} @@ -394,7 +399,30 @@ stages: set ARCHITECTURE=$(targetArch) python setup.py bdist_wheel displayName: 'Build wheel package' - + + # ========================= + # CONDA PACKAGES (x64 only) + # ========================= + # Repackage the freshly built mssql-python wheel into the per-Python + # mssql-python (binding) conda package and validate it on THIS agent (which + # already has the wheel and a live LocalDB). The Python-AGNOSTIC companion + # (mssql-python-odbc) is NOT rebuilt here -- it is built ONCE in ODBC_BuildAll + # and SEEDED into the local channel so the binding's `mssql-python-odbc + # ==` pin resolves for the solve/import. Conda must be built per matching + # -platform leg: conda-build provisions a real win-64 host env that cannot be + # created on any other OS. The step is x64-gated (the ARM64 host can neither + # run pytest nor provision a win-arm64 conda env). + - ${{ if and(parameters.buildConda, parameters.installOdbcWheel) }}: + - template: /OneBranchPipelines/steps/conda-build-validate-step.yml@self + parameters: + pythonVersion: ${{ parameters.pythonVersion }} + condaSubdir: 'win-64' + targetArch: ${{ parameters.architecture }} + odbcWheelFilter: 'mssql_python_odbc-*win_amd64.whl' + # Build ONLY the per-Python binding; seed the companion built once in + # ODBC_BuildAll (removes the previous per-Python companion redundancy). + package: 'binding' + # ========================= # SIGNED-WHEEL EVIDENCE (verification only) # ========================= diff --git a/OneBranchPipelines/steps/conda-build-validate-step-posix.yml b/OneBranchPipelines/steps/conda-build-validate-step-posix.yml new file mode 100644 index 000000000..e5bed1f21 --- /dev/null +++ b/OneBranchPipelines/steps/conda-build-validate-step-posix.yml @@ -0,0 +1,130 @@ +# Conda Build + Validate Step Template (POSIX / bash) +# ============================================================================ +# Bash twin of conda-build-validate-step.yml for the macOS (osx-arm64) and Linux +# (linux-64) build legs. Repackages THIS leg's mssql-python wheel(s) + the +# external mssql-python-odbc wheel into conda packages and validates solve+import +# on the SAME native agent. conda-build provisions a real per-subdir host env, so +# this only runs on the matching native platform (no cross-build, no musl target, +# no aarch64 here — the aarch64 host is x86_64 + QEMU). +# +# This step ONLY builds + validates + stages conda packages as an artifact. It +# does NOT publish anything (publishing happens in the release pipeline), and it is +# BLOCKING by default (continueOnError=false): if conda cannot build/validate the +# packages the leg FAILS, so a broken conda package can never hide behind a green +# build. The one intentionally best-effort exception is the emulated linux-aarch64 +# leg (QEMU flakiness), which overrides continueOnError to true at its call site. +parameters: + # conda subdir this leg targets: 'osx-arm64' or 'linux-64' (display + staging). + - name: condaSubdir + type: string + # Directory holding the freshly built mssql-python wheel(s) for this platform. + - name: mssqlWheelDir + type: string + default: '$(Build.SourcesDirectory)/dist' + # Glob selecting this platform's mssql-python wheel(s) (odbc excluded in-script). + - name: mssqlWheelGlob + type: string + default: 'mssql_python-*.whl' + # Directory holding the downloaded external mssql-python-odbc wheel(s). + # macOS downloads to $(Pipeline.Workspace)/odbc_wheels; Linux flattens them into + # $(Build.SourcesDirectory)/odbc_wheels — pass the right one per leg. + - name: odbcWheelDir + type: string + default: '$(Pipeline.Workspace)/odbc_wheels' + # find -name filter selecting THIS platform's odbc wheel from the consolidated + # odbc drop (which contains ALL 7 platforms). MUST match the leg's OS/arch, + # else conda-build's pip install fails with DistributionNotFound. + - name: odbcWheelFilter + type: string + # Repo conda/ recipe root (contains mssql-python/ and mssql-python-odbc/). + - name: recipeRoot + type: string + default: '$(Build.SourcesDirectory)/conda' + # Space-free working dir for the conda croot + Miniforge + built packages. + - name: outputDir + type: string + default: '$(Agent.TempDirectory)/conda-bld' + # Optional comma-separated Python versions; empty = auto-detect from the wheels. + - name: pythonVersions + type: string + default: '' + # Optional target subdir to CROSS-build via CONDA_SUBDIR (e.g. 'osx-64' on an + # Apple-Silicon agent, 'linux-aarch64' on an x86_64 host). Empty = build the + # host's native subdir (osx-arm64 / linux-64). Cross-targeting relies on the host + # being able to RUN the target's Python for the import validation (Rosetta 2 / + # QEMU binfmt); the caller is responsible for that being available on the leg. + - name: condaTargetSubdir + type: string + default: '' + # The shared bash build+validate script. + - name: scriptPath + type: string + default: '$(Build.SourcesDirectory)/OneBranchPipelines/scripts/build-conda-packages.sh' + # BLOCKING by default: if conda cannot build/validate the packages, FAIL the leg + # instead of letting a green build hide a broken conda package. Callers running an + # intentionally best-effort emulated leg (e.g. linux-aarch64 under QEMU) may override + # this to true. + - name: continueOnError + type: boolean + default: false + +steps: + - bash: | + set -euo pipefail + + MSSQL_WHEEL_DIR="${{ parameters.mssqlWheelDir }}" + ODBC_WHEEL_DIR="${{ parameters.odbcWheelDir }}" + OUT="${{ parameters.outputDir }}" + LINKS="$OUT/wheels" + rm -rf "$LINKS"; mkdir -p "$LINKS" + + # Gather this platform's mssql-python wheel(s) into ONE find-links dir, + # excluding the odbc package (its filename also starts with mssql_python). + shopt -s nullglob + mssql_found=0 + for w in "$MSSQL_WHEEL_DIR"/${{ parameters.mssqlWheelGlob }}; do + case "$(basename "$w")" in mssql_python_odbc-*) continue ;; esac + cp -f "$w" "$LINKS/"; mssql_found=1 + done + [ "$mssql_found" = "1" ] || { echo "ERROR: no mssql-python wheel in $MSSQL_WHEEL_DIR" >&2; exit 1; } + + # Gather THIS platform's odbc wheel (must match the leg's OS/arch). + odbc_whl="$(find "$ODBC_WHEEL_DIR" -name '${{ parameters.odbcWheelFilter }}' 2>/dev/null | head -1)" + [ -n "$odbc_whl" ] || { echo "ERROR: no wheel matching '${{ parameters.odbcWheelFilter }}' in $ODBC_WHEEL_DIR" >&2; exit 1; } + cp -f "$odbc_whl" "$LINKS/" + echo "find-links wheels:"; ls -1 "$LINKS" + + # Derive versions from the wheel filenames (single source of truth: the + # ESRP-signed wheels), so the conda package version can NEVER drift. + mssql_whl="$(ls "$LINKS"/mssql_python-*.whl | grep -v mssql_python_odbc | head -1)" + MSSQL_VER="$(basename "$mssql_whl" | sed -nE 's/^mssql_python-([^-]+)-.*/\1/p')" + ODBC_VER="$(basename "$odbc_whl" | sed -nE 's/^mssql_python_odbc-([^-]+)-.*/\1/p')" + [ -n "$MSSQL_VER" ] && [ -n "$ODBC_VER" ] || { echo "ERROR: could not derive versions from wheel filenames" >&2; exit 1; } + echo "Derived versions -> mssql-python=$MSSQL_VER mssql-python-odbc=$ODBC_VER" + + # Build + validate the conda packages for this leg's Python version(s). + chmod +x "${{ parameters.scriptPath }}" + bash "${{ parameters.scriptPath }}" \ + "$LINKS" \ + "${{ parameters.recipeRoot }}" \ + "$OUT" \ + "$MSSQL_VER" \ + "$ODBC_VER" \ + "${{ parameters.pythonVersions }}" \ + "${{ parameters.condaTargetSubdir }}" + + # Stage the built conda packages onto the leg artifact so the consolidate + # stage can collect them (preserve the conda subdir folder layout). + CONDA_OUT="$(ob_outputDirectory)/conda" + mkdir -p "$CONDA_OUT" + staged=0 + while IFS= read -r p; do + subdir="$(basename "$(dirname "$p")")" + mkdir -p "$CONDA_OUT/$subdir" + cp -f "$p" "$CONDA_OUT/$subdir/" + echo " staged $subdir/$(basename "$p")" + staged=1 + done < <(find "$OUT/bld" -type f \( -name 'mssql-python*.conda' -o -name 'mssql-python*.tar.bz2' \)) + [ "$staged" = "1" ] || { echo "ERROR: no conda packages were produced" >&2; exit 1; } + displayName: 'Conda build + validate (${{ parameters.condaSubdir }})' + continueOnError: ${{ parameters.continueOnError }} diff --git a/OneBranchPipelines/steps/conda-build-validate-step.yml b/OneBranchPipelines/steps/conda-build-validate-step.yml new file mode 100644 index 000000000..a3a31970f --- /dev/null +++ b/OneBranchPipelines/steps/conda-build-validate-step.yml @@ -0,0 +1,181 @@ +# Conda Build + Validate Step Template +# ============================================================================ +# Repackages the prebuilt, ESRP-signed wheels produced by THIS build leg into +# conda packages, then validates them on the SAME native agent (which already +# has the matching wheel, the external mssql-python-odbc wheel, and a live +# SQL Server for pytest). Include this AFTER the wheel is built on a build leg. +# +# WHY THIS RUNS PER-PLATFORM (not on a single host like ODBC_BuildAll): +# `ODBC_BuildAll` cross-produces every wheel on one host because setup_odbc.py +# only RE-TAGS a data zip. conda-build is different: it provisions a real host +# environment for the target subdir and `pip install`s the matching wheel +# (see conda/*/bld.bat|build.sh). A linux-64 / osx-* host env cannot be created +# on a Windows agent, so — exactly like the wheels and like the conda-forge +# pyodbc-feedstock — each conda package must be built on its matching platform. +# +# SCOPE / LIMITATIONS (first cut, intentionally conservative): +# * x64 / native only. Skipped on cross-arch legs (e.g. Windows ARM64) because +# the import validation cannot execute on the x64 host AND conda cannot +# provision a win-arm64 host env there — same reason pytest is skipped there. +# * musllinux has NO conda target (conda linux-* is glibc), so this step is +# never included on the musllinux legs. +# +# This step ONLY builds + validates + stages conda packages as an artifact. It +# does NOT publish anything — publishing (anaconda upload / ESRP) happens in the +# release pipeline, exactly like the wheels. +parameters: + # Python version this leg builds, X.Y (e.g. '3.13'). One conda build per leg. + - name: pythonVersion + type: string + # conda subdir to stamp on the packages (win-64, osx-64, osx-arm64, + # linux-64, linux-aarch64). Must match the platform of THIS agent. + - name: condaSubdir + type: string + # Target architecture of the wheel build; the step is skipped unless 'x64' + # (or a native arch) so cross-compiled legs don't attempt a conda build. + - name: targetArch + type: string + default: 'x64' + # Directory holding the freshly built mssql-python wheel (setup.py bdist_wheel). + - name: mssqlWheelDir + type: string + default: '$(Build.SourcesDirectory)/dist' + # Directory holding the downloaded external mssql-python-odbc wheel(s) + # (populated by the leg's `installOdbcWheel` download step). + - name: odbcWheelDir + type: string + default: '$(Pipeline.Workspace)/odbc_wheels' + # Filename filter selecting THIS platform's mssql-python-odbc wheel from the + # consolidated odbc drop (which contains ALL 7 platforms). MUST match the leg's + # OS/arch, otherwise conda-build's `pip install` on this host fails with + # DistributionNotFound (a macOS/linux wheel is not installable on win-64, etc.). + - name: odbcWheelFilter + type: string + default: 'mssql_python_odbc-*win_amd64.whl' + # Repo conda/ recipe root (contains mssql-python/ and mssql-python-odbc/). + - name: recipeRoot + type: string + default: '$(Build.SourcesDirectory)/conda' + # Space-free working dir for the conda croot + Miniforge + built packages. + - name: outputDir + type: string + default: '$(Agent.TempDirectory)/conda-bld' + # The shared build+validate script (installs Miniforge/conda-build, builds the + # companion then the binding, indexes a local channel, solves + imports both). + - name: scriptPath + type: string + default: '$(Build.SourcesDirectory)/OneBranchPipelines/scripts/build-conda-packages.ps1' + # BLOCKING by default: if conda cannot build/validate the packages, FAIL the leg + # instead of letting a green build hide a broken conda package. Callers running an + # intentionally best-effort emulated leg (e.g. linux-aarch64 under QEMU) may override + # this to true. + - name: continueOnError + type: boolean + default: false + # Which package(s) this step builds. Windows legs build ONLY the per-Python binding + # ('binding'); the Python-agnostic companion (mssql-python-odbc) is built ONCE in the + # ODBC_BuildAll stage and SEEDED here so the binding's version-locked + # `mssql-python-odbc ==` pin resolves in the validation solve WITHOUT rebuilding + # the companion per Python. Use 'all' to also build the companion (legacy behaviour). + - name: package + type: string + default: 'binding' + # Artifact holding the prebuilt companion conda, staged under conda// by the + # ODBC_BuildAll stage. Downloaded + seeded in 'binding' mode. + - name: driverCondaArtifact + type: string + default: 'drop_ODBC_BuildAll_BuildWheel' + # Local folder the prebuilt companion conda is downloaded to (binding mode only). + - name: driverCondaDownloadDir + type: string + default: '$(Agent.TempDirectory)/driver-conda' + +steps: + # binding mode: fetch the Python-agnostic companion conda built once in ODBC_BuildAll + # so the binding's `mssql-python-odbc ==` pin resolves in the validation solve. + - ${{ if eq(parameters.package, 'binding') }}: + - task: DownloadPipelineArtifact@2 + displayName: 'Download prebuilt companion conda (ODBC_BuildAll)' + condition: ne('${{ parameters.targetArch }}', 'arm64') + inputs: + buildType: 'current' + artifactName: '${{ parameters.driverCondaArtifact }}' + itemPattern: '**/conda/**/*.conda' + targetPath: '${{ parameters.driverCondaDownloadDir }}' + + - powershell: | + $ErrorActionPreference = 'Stop' + + $mssqlWheelDir = "${{ parameters.mssqlWheelDir }}" + $odbcWheelDir = "${{ parameters.odbcWheelDir }}" + + # Gather both packages' wheels into ONE find-links dir the recipes install from. + $links = Join-Path "${{ parameters.outputDir }}" 'wheels' + New-Item -ItemType Directory -Force -Path $links | Out-Null + + $mssqlWheel = Get-ChildItem -Path $mssqlWheelDir -Filter 'mssql_python-*.whl' -ErrorAction SilentlyContinue | + Where-Object { $_.Name -notlike 'mssql_python_odbc-*' } | Select-Object -First 1 + if (-not $mssqlWheel) { Write-Error "No mssql_python-*.whl found in $mssqlWheelDir"; exit 1 } + + $odbcWheel = Get-ChildItem -Path $odbcWheelDir -Recurse -Filter '${{ parameters.odbcWheelFilter }}' -ErrorAction SilentlyContinue | + Select-Object -First 1 + if (-not $odbcWheel) { Write-Error "No wheel matching '${{ parameters.odbcWheelFilter }}' found in $odbcWheelDir"; exit 1 } + + Copy-Item $mssqlWheel.FullName -Destination $links -Force + Copy-Item $odbcWheel.FullName -Destination $links -Force + Write-Host "find-links wheels:" + Get-ChildItem $links | ForEach-Object { Write-Host " - $($_.Name)" } + + # Derive the versions from the wheel filenames (single source of truth: the + # ESRP-signed wheels themselves) so the conda package version can NEVER drift + # from the wheel. Filenames: mssql_python--cp3X-...whl and + # mssql_python_odbc--py3-none-...whl. + if ($mssqlWheel.Name -notmatch '^mssql_python-([^-]+)-') { Write-Error "Cannot parse version from $($mssqlWheel.Name)"; exit 1 } + $mssqlVer = $Matches[1] + if ($odbcWheel.Name -notmatch '^mssql_python_odbc-([^-]+)-') { Write-Error "Cannot parse version from $($odbcWheel.Name)"; exit 1 } + $odbcVer = $Matches[1] + Write-Host "Derived versions -> mssql-python=$mssqlVer mssql-python-odbc=$odbcVer" + + # Build + validate the conda packages for THIS leg's single Python version. + # 'binding' mode seeds the companion from the ODBC_BuildAll download (splatted + # only when present) instead of rebuilding it per Python. + # + # MUST be a HASHTABLE, not an array: array splatting binds POSITIONALLY, so + # @('-DriverCondaDir', ) passed '-DriverCondaDir' as a literal value into + # the first free positional param ($CondaSubdir). That set CONDA_SUBDIR=-DriverCondaDir + # and conda-build then parsed the platform as '' -> `KeyError: ''`. Hashtable + # splatting binds by NAME, so -DriverCondaDir is bound to $DriverCondaDir correctly. + $driverArg = @{} + if ('${{ parameters.package }}' -eq 'binding') { + $driverArg = @{ DriverCondaDir = "${{ parameters.driverCondaDownloadDir }}" } + } + & "${{ parameters.scriptPath }}" ` + -WheelsDir $links ` + -RecipeRoot "${{ parameters.recipeRoot }}" ` + -OutputDir "${{ parameters.outputDir }}" ` + -MssqlPythonVersion $mssqlVer ` + -OdbcVersion $odbcVer ` + -PythonVersions "${{ parameters.pythonVersion }}" ` + -Package '${{ parameters.package }}' ` + @driverArg + if ($LASTEXITCODE -ne 0) { Write-Error "conda build+validate failed (exit $LASTEXITCODE)"; exit 1 } + + # Stage the built conda packages onto the leg's artifact so the consolidate + # stage can collect them (mirrors how the wheels ride the same artifact). + $condaOut = Join-Path "$(ob_outputDirectory)" 'conda' + New-Item -ItemType Directory -Force -Path $condaOut | Out-Null + $built = Get-ChildItem -Path (Join-Path "${{ parameters.outputDir }}" 'bld') -Recurse -Include *.conda, *.tar.bz2 | + Where-Object { $_.Name -like 'mssql-python*' } + if (-not $built) { Write-Error "No conda packages were produced under $($links)"; exit 1 } + foreach ($p in $built) { + # Preserve the conda subdir folder layout (e.g. win-64/) so the channel + # is valid when consolidated and indexed downstream. + $subdir = Split-Path -Leaf (Split-Path -Parent $p.FullName) + $dest = Join-Path $condaOut $subdir + New-Item -ItemType Directory -Force -Path $dest | Out-Null + Copy-Item $p.FullName -Destination $dest -Force + Write-Host " staged $subdir/$($p.Name)" + } + displayName: 'Conda build + validate (${{ parameters.condaSubdir }} py${{ parameters.pythonVersion }})' + condition: ne('${{ parameters.targetArch }}', 'arm64') + continueOnError: ${{ parameters.continueOnError }} diff --git a/OneBranchPipelines/steps/conda-publish-step.yml b/OneBranchPipelines/steps/conda-publish-step.yml new file mode 100644 index 000000000..e0fe0accb --- /dev/null +++ b/OneBranchPipelines/steps/conda-publish-step.yml @@ -0,0 +1,212 @@ +# Conda Publish Step Template +# ============================================================================ +# Publishes the consolidated conda packages produced by build definition 2199 +# (artifact drop_ConsolidateConda_ConsolidateArtifacts) to an Anaconda.org channel +# using anaconda-client (`anaconda upload`). +# +# Reference pattern: azure-sdk-for-python's conda publish +# (eng/pipelines/templates/stages/archetype-conda-release.yml), which runs +# `anaconda upload --user Microsoft --skip-existing` inside a 1ES releaseJob and +# authenticates via the ANACONDA_API_TOKEN env var. ESRP has NO Conda ContentType, +# so anaconda-client is the sanctioned publish path. +# +# Two adaptations vs. azure-sdk: +# 1. Our packages are NON-noarch (native binding + driver), so they live under +# per-platform subdirs (win-64 / osx-arm64 / linux-64), NOT a single noarch +# folder. We recurse every subdir. +# 2. #706 ordering: the version-locked companion (mssql-python-odbc) is uploaded +# BEFORE the binding (mssql-python) so the binding's pinned dependency is +# always resolvable on the channel. We also refuse to publish a mismatched or +# incomplete set. +# +# The caller MUST: +# - run this ONLY after the conda-release-step.yml readiness gate has passed, and +# - supply ANACONDA_API_TOKEN (variable group 'Anaconda Publishing') to the job. +# This template never puts the token on the command line (anaconda-client reads it +# from the env), so it never appears in the logs. +parameters: + # Build pipeline definition id that produced the consolidated conda artifact. + - name: buildDefinitionId + type: number + default: 2199 + # Consolidated conda artifact name (see consolidate-conda-artifacts-job.yml). + - name: condaArtifactName + type: string + default: 'drop_ConsolidateConda_ConsolidateArtifacts' + # Target Anaconda.org channel/org (e.g. 'microsoft'). Empty is rejected so a + # test pipeline can never accidentally push to the production channel. + - name: condaChannel + type: string + default: '' + # Channel label to publish under (production packages go to 'main'). + - name: condaLabel + type: string + default: 'main' + # Comma-separated subdirs a complete release MUST contain (PyPI parity minus + # win-arm64 and musllinux). Guards against publishing an incomplete set even if + # this step is run standalone. Keep in sync with conda-release-step.yml. + - name: requiredSubdirs + type: string + default: 'win-64,osx-64,osx-arm64,linux-64,linux-aarch64' + # Comma-separated superset of subdirs allowed to appear; anything else fails. + - name: allowedSubdirs + type: string + default: 'win-64,win-arm64,osx-64,osx-arm64,linux-64,linux-aarch64' + # Python used to install/run anaconda-client in the release job. + - name: pythonVersion + type: string + default: '3.12' + # Optional display-name prefix (e.g. '[TEST] ' for the dummy pipeline). + - name: labelPrefix + type: string + default: '' + +steps: + - task: DownloadPipelineArtifact@2 + displayName: '${{ parameters.labelPrefix }}Download consolidated conda packages (publish)' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: ${{ parameters.buildDefinitionId }} + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: '${{ parameters.condaArtifactName }}' + targetPath: '$(Build.SourcesDirectory)/conda-artifacts' + + - task: UsePythonVersion@0 + displayName: '${{ parameters.labelPrefix }}Use Python ${{ parameters.pythonVersion }}' + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + addToPath: true + + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Install anaconda-client' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + python -m pip install --upgrade pip + python -m pip install anaconda-client + # anaconda-client installs the `anaconda` console script onto PATH. + anaconda --version + + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Publish conda packages to anaconda.org/${{ parameters.condaChannel }}' + env: + # anaconda-client reads ANACONDA_API_TOKEN automatically, so the token is + # never passed on the command line and never appears in the logs. Supplied by + # the caller from the 'Anaconda Publishing' variable group. + ANACONDA_API_TOKEN: $(ANACONDA_API_TOKEN) + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + + # Refuse to publish without an explicit target channel (protects the dummy + # pipeline, whose default channel is empty, from ever hitting production). + if ([string]::IsNullOrWhiteSpace("${{ parameters.condaChannel }}")) { + Write-Error "condaChannel is empty. Supply the target Anaconda.org channel/org (production = 'microsoft')." + exit 1 + } + + if ([string]::IsNullOrWhiteSpace($env:ANACONDA_API_TOKEN)) { + Write-Error "ANACONDA_API_TOKEN is not set. Add the 'Anaconda Publishing' variable group to the job." + exit 1 + } + + $root = "$(Build.SourcesDirectory)/conda-artifacts/conda" + if (-not (Test-Path $root)) { + Write-Error "Consolidated conda tree not found at $root. Did ConsolidateConda run in the selected build?" + exit 1 + } + + $required = '${{ parameters.requiredSubdirs }}'.Split(',') | ForEach-Object { $_.Trim() } | Where-Object { $_ } + $allowed = '${{ parameters.allowedSubdirs }}'.Split(',') | ForEach-Object { $_.Trim() } | Where-Object { $_ } + + $pkgs = @(Get-ChildItem -Path $root -Recurse -Include *.conda, *.tar.bz2) + if ($pkgs.Count -eq 0) { Write-Error "No conda packages found under $root. Refusing to publish an empty set."; exit 1 } + + $bySubdir = $pkgs | Group-Object { $_.Directory.Name } + function Get-Binding($grp) { @($grp | Where-Object { $_.Name -like 'mssql-python-*' -and $_.Name -notlike 'mssql-python-odbc-*' }) } + function Get-Companion($grp) { @($grp | Where-Object { $_.Name -like 'mssql-python-odbc-*' }) } + + Write-Host "Discovered $($pkgs.Count) conda package(s) across $($bySubdir.Count) subdir(s):" + foreach ($g in ($bySubdir | Sort-Object Name)) { + Write-Host (" {0,-14} binding={1} companion={2}" -f $g.Name, (Get-Binding $g.Group).Count, (Get-Companion $g.Group).Count) + } + + $failed = $false + + # No mis-stamped subdir. + foreach ($g in $bySubdir) { + if ($allowed -notcontains $g.Name) { + Write-Host "##vso[task.logissue type=error]Unexpected conda subdir '$($g.Name)' (not in allowedSubdirs). Refusing to publish." + $failed = $true + } + } + + # Every required subdir present and internally #706-paired. + $foundSubdirs = @($bySubdir | ForEach-Object { $_.Name }) + foreach ($req in $required) { + if ($foundSubdirs -notcontains $req) { + Write-Host "##vso[task.logissue type=error]Required conda subdir '$req' is MISSING. Refusing to publish an incomplete set." + $failed = $true + continue + } + $grp = ($bySubdir | Where-Object { $_.Name -eq $req }).Group + $b = (Get-Binding $grp).Count; $c = (Get-Companion $grp).Count + # PRESENCE, not equal counts: the Windows companion is a SINGLE Python- + # agnostic package (built once in ODBC_BuildAll) serving all per-Python + # win-64 bindings, so b(5) != c(1) is EXPECTED there and not a violation. + # macOS/Linux still build the companion per-Python (c > 1) and keep 1:1. + if ($b -eq 0) { Write-Host "##vso[task.logissue type=error]Subdir '$req' has no binding (mssql-python) package."; $failed = $true } + if ($c -eq 0) { Write-Host "##vso[task.logissue type=error]Subdir '$req' has no companion (mssql-python-odbc) package."; $failed = $true } + if ($c -gt 1 -and $b -ne $c) { Write-Host "##vso[task.logissue type=error]#706 violation in '$req': $b binding vs $c per-Python companion (must pair 1:1)."; $failed = $true } + } + + # #706: split into companion (mssql-python-odbc) and binding (mssql-python). + $companion = Get-Companion $pkgs + $binding = Get-Binding $pkgs + # #706 pairing per subdir (both-or-neither): a companion without ANY binding is + # the companion-only bump behind #706; a binding without a companion cannot + # resolve its version-locked `mssql-python-odbc ==` pin. + foreach ($g in $bySubdir) { + $gb = (Get-Binding $g.Group).Count; $gc = (Get-Companion $g.Group).Count + if (($gb -gt 0) -ne ($gc -gt 0)) { + Write-Host "##vso[task.logissue type=error]#706 violation in '$($g.Name)': binding=$gb companion=$gc (a binding and its companion must ship together)." + $failed = $true + } + } + if ($companion.Count -gt 0 -and $binding.Count -eq 0) { + Write-Host "##vso[task.logissue type=error]#706 violation (global): $($companion.Count) companion package(s) with NO binding (companion-only release)." + $failed = $true + } + if ($binding.Count -gt 0 -and $companion.Count -eq 0) { + Write-Host "##vso[task.logissue type=error]#706 violation (global): $($binding.Count) binding package(s) with NO companion." + $failed = $true + } + + if ($failed) { Write-Error "Conda publish pre-check FAILED. Refusing to publish an incomplete/mis-paired set."; exit 1 } + + function Publish-One($file) { + for ($attempt = 1; $attempt -le 3; $attempt++) { + Write-Host "Uploading $($file.Directory.Name)/$($file.Name) (attempt $attempt) ..." + anaconda upload --user "${{ parameters.condaChannel }}" --label "${{ parameters.condaLabel }}" --skip-existing "$($file.FullName)" + if ($LASTEXITCODE -eq 0) { return } + Write-Host "Attempt $attempt failed; retrying in 5s ..." + Start-Sleep -Seconds 5 + } + Write-Error "Failed to upload $($file.Name) after 3 attempts." + exit 1 + } + + # #706 ordering: companion FIRST so the binding's pinned dependency is + # resolvable the instant the binding lands on the channel. + Write-Host "==== Publishing companion (mssql-python-odbc) packages ====" + foreach ($p in ($companion | Sort-Object FullName)) { Publish-One $p } + + Write-Host "==== Publishing binding (mssql-python) packages ====" + foreach ($p in ($binding | Sort-Object FullName)) { Publish-One $p } + + Write-Host "" + Write-Host "Uploaded $($companion.Count) companion + $($binding.Count) binding conda packages to anaconda.org/${{ parameters.condaChannel }} (label ${{ parameters.condaLabel }})." diff --git a/OneBranchPipelines/steps/conda-release-step.yml b/OneBranchPipelines/steps/conda-release-step.yml new file mode 100644 index 000000000..e4e54be67 --- /dev/null +++ b/OneBranchPipelines/steps/conda-release-step.yml @@ -0,0 +1,142 @@ +# Conda Release Readiness Step Template +# ============================================================================ +# Downloads the consolidated conda packages produced by the build pipeline +# (definition 2199, artifact drop_ConsolidateConda_ConsolidateArtifacts) and +# enforces the RELEASE-TIME hard gate that the build pipeline intentionally does +# NOT enforce. +# +# Why the gate lives HERE and not in the build: +# - BUILD pipeline: conda is collected BEST-EFFORT (warn-only) so a conda hiccup +# on any leg can never fail the build or block the primary wheel release. +# - RELEASE pipeline: conda completeness is GATED — an incomplete or mis-paired +# conda set must never be shipped. +# +# What it enforces (all from each package's AUTHORITATIVE info/index.json, never +# folder names or bare counts -- so a mis-stamped subdir or a dropped Python +# variant cannot slip through): +# - real subdir: every package's info/index.json `subdir` is in `allowedSubdirs` +# AND equals the folder it was staged into (catches an osx-64 package copied +# into osx-arm64/, which a folder-name check cannot). +# - required subdirs: every subdir in `requiredSubdirs` (the PyPI-parity set) +# is present. +# - Python matrix: on every required subdir the mssql-python binding covers +# EVERY expected Python (`pythonVersions`) -- catches e.g. 3 of 5 win-64 +# bindings shipping against the single companion. +# - versions: all packages of a name share one version (and match the expected +# version when `mssqlPythonVersion` / `odbcVersion` are supplied). +# - #706 pairing: a binding and its companion SHIP TOGETHER (both present or +# both absent) -- never a companion-only bump. The Windows companion is a +# SINGLE Python-agnostic package built once in ODBC_BuildAll that serves all +# per-Python win-64 bindings, so pairing is PRESENCE-based; where the +# companion is per-Python (macOS/Linux, count > 1) it must pair 1:1. +# +# The check is implemented in conda/validate_conda_release.py (unit-tested by +# tests/test_027_conda_release_metadata.py), which reads the zstd-compressed +# info/index.json embedded in every .conda. +# +# win-arm64 is intentionally NOT in the parity set: an x64 agent cannot import- +# validate an arm64 conda package (same reason its wheel skips pytest), so it needs +# a native win-arm64 agent before it can be gated. musllinux has no conda subdir at +# all (conda Linux is glibc-only), so it is correctly absent. +# +# Publishing to anaconda.org is a SEPARATE, still-to-be-finalized step (ESRP Conda +# ContentType vs anaconda-client upload); this template only proves the artifact is +# complete and correctly paired so publishing can proceed safely. +parameters: + # Build pipeline definition id that produced the conda artifact. + - name: buildDefinitionId + type: number + default: 2199 + # Consolidated conda artifact name (see consolidate-conda-artifacts-job.yml). + - name: condaArtifactName + type: string + default: 'drop_ConsolidateConda_ConsolidateArtifacts' + # Comma-separated subdirs that a complete release MUST contain (PyPI parity minus + # win-arm64 and musllinux, which have no validated conda build). If an emulated + # leg (osx-64 / linux-aarch64) ever proves too flaky to gate on, drop it here — + # no code change needed. + - name: requiredSubdirs + type: string + default: 'win-64,osx-64,osx-arm64,linux-64,linux-aarch64' + # Comma-separated superset of subdirs that are ALLOWED to appear. Any discovered + # subdir outside this set fails the gate (guards against a mis-stamped subdir). + # win-arm64 is allowed-but-not-required so a future native-agent build can land + # without tripping the gate. + - name: allowedSubdirs + type: string + default: 'win-64,win-arm64,osx-64,osx-arm64,linux-64,linux-aarch64' + # Comma-separated Python versions the binding matrix MUST cover on every required + # subdir. The gate reads each binding's pyXY build tag from info/index.json. + - name: pythonVersions + type: string + default: '3.10,3.11,3.12,3.13,3.14' + # Optional EXACT expected versions. When set, the gate asserts every package's + # info/index.json version matches; when empty it still enforces one-version-per- + # package consistency plus the subdir / matrix / pairing checks. + - name: mssqlPythonVersion + type: string + default: '' + - name: odbcVersion + type: string + default: '' + # Optional display-name prefix (e.g. '[TEST] ' for the dummy pipeline). + - name: labelPrefix + type: string + default: '' + +steps: + - task: DownloadPipelineArtifact@2 + displayName: '${{ parameters.labelPrefix }}Download consolidated conda packages' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: ${{ parameters.buildDefinitionId }} + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: '${{ parameters.condaArtifactName }}' + targetPath: '$(Build.SourcesDirectory)/conda-artifacts' + + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Validate conda release readiness (metadata: subdirs + Python matrix + #706 pairing)' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $root = "$(Build.SourcesDirectory)/conda-artifacts/conda" + if (-not (Test-Path $root)) { + Write-Error "Consolidated conda tree not found at $root. Was ConsolidateConda produced by the selected build run?" + exit 1 + } + + $gate = "$(Build.SourcesDirectory)/conda/validate_conda_release.py" + if (-not (Test-Path $gate)) { + Write-Error "Metadata gate script not found at $gate." + exit 1 + } + + # The gate reads each package's AUTHORITATIVE info/index.json (a zstd tar + # inside every .conda) rather than trusting folder names or counts. zstd is + # stdlib on py3.14+; install the `zstandard` backend so the reader always works. + python -m pip install --quiet --disable-pip-version-check zstandard + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to install the 'zstandard' backend needed to read .conda metadata." + exit 1 + } + + # Pass EXACT version expectations only when supplied; the gate still enforces + # one-version-per-package consistency when they are empty. + $extra = @() + if ('${{ parameters.mssqlPythonVersion }}'.Trim()) { $extra += @('--mssql-python-version', ('${{ parameters.mssqlPythonVersion }}'.Trim())) } + if ('${{ parameters.odbcVersion }}'.Trim()) { $extra += @('--mssql-python-odbc-version', ('${{ parameters.odbcVersion }}'.Trim())) } + + python "$gate" ` + --root "$root" ` + --required-subdirs '${{ parameters.requiredSubdirs }}' ` + --allowed-subdirs '${{ parameters.allowedSubdirs }}' ` + --pythons '${{ parameters.pythonVersions }}' ` + @extra + if ($LASTEXITCODE -ne 0) { + Write-Error "Conda release readiness FAILED. Refusing to proceed with an incomplete/mis-labeled/mis-paired conda set." + exit 1 + } + Write-Host "Conda set is release-ready (the publish step is gated separately)." diff --git a/conda/.gitattributes b/conda/.gitattributes new file mode 100644 index 000000000..68446d138 --- /dev/null +++ b/conda/.gitattributes @@ -0,0 +1,3 @@ +# Keep shell build scripts LF so conda-build works on Linux/macOS agents, +# regardless of the checkout host's core.autocrlf setting. +*.sh text eol=lf diff --git a/conda/driver_load_probe.py b/conda/driver_load_probe.py new file mode 100644 index 000000000..c9fd618c9 --- /dev/null +++ b/conda/driver_load_probe.py @@ -0,0 +1,138 @@ +"""DB-less ODBC driver-load proof for the conda test-before-live gate. + +Importing ``mssql_python`` and issuing the first ``connect()`` triggers the +one-time native ODBC driver load (``std::call_once`` in the C++ binding). To +prove the driver payload is present AND architecture-correct WITHOUT a live SQL +Server, we attempt a connection to an unreachable local port and classify the +failure. + +FAIL-CLOSED classification (this is the whole point of the probe): + +* We treat the outcome as PASS **only** when there is positive proof the native + driver loaded -- either a clean connect, or a *connection-stage* diagnostic + that only the loaded ``msodbcsql`` driver can emit (its ``[Microsoft][ODBC + Driver 18 for SQL Server]`` branding, a SQL Server network provider error, a + TLS handshake error, or a login / auth outcome). See ``_DRIVER_LOADED_MARKERS``. +* Every other exception is treated as a load failure -> non-zero exit. This + includes the C++ ``LoadDriverOrThrowException`` family + ("Failed to load the driver...", "Failed to load library: ", + "Failed to load required function pointers...", "ODBC driver not found...", + the ``mssql-auth.dll`` errors) and the macOS ``dlopen`` / ``dlerror`` detail -- + none of which contain a loaded-driver marker, so a broken / missing / + mis-architecture driver can never report PASS. + +This gates on the actual DRIVER, not just the tiny ``mssql_python_odbc`` Python +shim, and needs no ``DB_CONNECTION_STRING`` secret. A real live ``SELECT 1`` still +runs separately whenever a server is wired. + +Exit code 0 = driver loaded; non-zero = driver did not load (blocks publish). +""" + +import sys + +# Positive signals: the native ODBC driver LOADED and reached the network / TLS +# / auth stage (or connected). These are the ONLY outcomes that count as PASS. +# All markers are matched case-insensitively. +_DRIVER_LOADED_MARKERS = ( + # The loaded msodbcsql driver brands every diagnostic it emits; a driver + # that failed to load / link / resolve its symbols never gets far enough to + # print this, so it is the strongest single proof of a successful load. + "odbc driver 18 for sql server", + "microsoft][odbc", + # SQL Server network / transport providers -- reached only after load. + "tcp provider", + "named pipes provider", + "shared memory provider", + "sql server network interfaces", + # Connection / login outcomes that prove the handshake was attempted. + "login timeout expired", + "a network-related or instance-specific error", + "server was not found", + "server is not found", + "actively refused", # Windows WSAECONNREFUSED (target port closed) + "connection refused", # posix ECONNREFUSED (target port closed) + "communication link failure", + "unable to establish", + "login failed for user", # authentication stage reached + "cannot open database", # server reached, database validation + # TLS handshake reached -> both the driver and its crypto backend loaded. + "ssl provider", + "ssl security error", + "certificate", +) + +# Negative signals: the native driver did NOT load / link / resolve. Listed only +# to produce a clearer FAIL message -- classification is allowlist-based, so an +# unrecognized exception still fails closed even if it matches nothing here. +_DRIVER_LOAD_FAILURE_MARKERS = ( + "failed to load the driver", + "failed to load library", + "failed to load required function pointers", + "odbc driver not found", + "mssql-auth.dll", + "mssql-python-odbc", + "cannot open shared object", # linux dlopen failure + "image not found", # macOS dlopen failure + "no such file or directory", # driver binary absent + "can't open lib", # unixODBC could not open the driver + "unsupported architecture", + "unsupported platform", +) + + +def driver_loaded(exc): + """FAIL-CLOSED classifier for the connect outcome. + + Returns ``True`` only when there is positive proof the native ODBC driver + loaded: a clean connect (``exc is None``) or a connection-stage diagnostic + that the loaded driver alone can emit. Every other exception -- including the + C++ "Failed to load the driver..." family and anything unrecognized -- + returns ``False`` so the probe exits non-zero. + """ + if exc is None: + return True + msg = str(exc).lower() + return any(marker in msg for marker in _DRIVER_LOADED_MARKERS) + + +def describe(exc): + """Short, human-readable reason string for the probe's stdout / exit line.""" + if exc is None: + return "clean connect" + msg = str(exc) + low = msg.lower() + for marker in _DRIVER_LOAD_FAILURE_MARKERS: + if marker in low: + return "driver load failure -> " + msg[:300] + return msg[:300] + + +def main(): + # Deferred so this module can be imported (and ``driver_loaded`` unit-tested) + # WITHOUT triggering the native ``mssql_python`` import, which needs the + # compiled extension + driver payload. + import mssql_python + + # Unreachable endpoint (nothing listens on TCP port 1) -> the driver loads, + # attempts the socket, and fails fast at the network stage. + conn_str = "Server=127.0.0.1,1;Database=x;Uid=x;Pwd=x;Encrypt=no;TrustServerCertificate=yes;" + outcome = None + try: + conn = mssql_python.connect(conn_str) + # Reaching a real server on 127.0.0.1:1 is not expected, but a successful + # connect still proves the driver loaded. Close it and pass. + try: + conn.close() + except Exception: # noqa: BLE001 - best-effort cleanup only + pass + except Exception as exc: # noqa: BLE001 - deliberately classified below + outcome = exc + + if driver_loaded(outcome): + print("DRIVER_LOADED (" + describe(outcome) + ")") + return + sys.exit("DRIVER DID NOT LOAD / wrong arch / missing companion: " + describe(outcome)) + + +if __name__ == "__main__": + main() diff --git a/conda/mssql-python/MICROSOFT_ODBC_DRIVER_FOR_SQL_SERVER_LICENSE.txt b/conda/mssql-python/MICROSOFT_ODBC_DRIVER_FOR_SQL_SERVER_LICENSE.txt new file mode 100644 index 000000000..ccd8cc2b8 --- /dev/null +++ b/conda/mssql-python/MICROSOFT_ODBC_DRIVER_FOR_SQL_SERVER_LICENSE.txt @@ -0,0 +1,100 @@ +MICROSOFT SOFTWARE LICENSE TERMS +MICROSOFT ODBC DRIVER 18 FOR SQL SERVER + +These license terms are an agreement between you and Microsoft Corporation (or one of its affiliates). They apply to the software named above and any Microsoft services or software updates (except to the extent such services or updates are accompanied by new or additional terms, in which case those different terms apply prospectively and do not alter your or Microsoft’s rights relating to pre-updated software or services). IF YOU COMPLY WITH THESE LICENSE TERMS, YOU HAVE THE RIGHTS BELOW. BY USING THE SOFTWARE, YOU ACCEPT THESE TERMS. + +1. INSTALLATION AND USE RIGHTS. + + a) General. You may install and use any number of copies of the software to develop and test your applications. + + b) Third Party Software. The software may include third party applications that Microsoft, not the third party, licenses to you under this agreement. Any included notices for third party applications are for your information only. + +2. DISTRIBUTABLE CODE. The software may contain code you are permitted to distribute (i.e. make available for third parties) in applications you develop, as described in this Section. + + a) Distribution Rights. The code and test files described below are distributable if included with the software. + + i. REDIST.TXT Files. You may copy and distribute the object code form of code listed on the REDIST list in the software, if any, or listed at REDIST (https://aka.ms/odbc18eularedist); + + ii. Image Library. You may copy and distribute images, graphics, and animations in the Image Library as described in the software documentation; + + iii. Sample Code, Templates, and Styles. You may copy, modify, and distribute the source and object code form of code marked as “sample”, “template”, “simple styles”, and “sketch styles”; and + + iv. Third Party Distribution. You may permit distributors of your applications to copy and distribute any of this distributable code you elect to distribute with your applications. + + b) Distribution Requirements. For any code you distribute, you must: + + i. add significant primary functionality to it in your applications; + + ii. require distributors and external end users to agree to terms that protect it and Microsoft at least as much as this agreement; and + + iii. indemnify, defend, and hold harmless Microsoft from any claims, including attorneys’ fees, related to the distribution or use of your applications, except to the extent that any claim is based solely on the unmodified distributable code. + + c) Distribution Restrictions. You may not: + + i. use Microsoft’s trademarks or trade dress in your application in any way that suggests your application comes from or is endorsed by Microsoft; or + + ii. modify or distribute the source code of any distributable code so that any part of it becomes subject to any license that requires that the distributable code, any other part of the software, or any of Microsoft’s other intellectual property be disclosed or distributed in source code form, or that others have the right to modify it. + +3. DATA COLLECTION. Some features in the software may enable collection of data from users of your applications that access or use the software. If you use these features to enable data collection in your applications, you must comply with applicable law, including getting any required user consent, and maintain a prominent privacy policy that accurately informs users about how you use, collect, and share their data. You agree to comply with all applicable provisions of the Microsoft Privacy Statement at [https://go.microsoft.com/fwlink/?LinkId=521839]. + +4. SCOPE OF LICENSE. The software is licensed, not sold. Microsoft reserves all other rights. Unless applicable law gives you more rights despite this limitation, you will not (and have no right to): + + a) work around any technical limitations in the software that only allow you to use it in certain ways; + + b) reverse engineer, decompile or disassemble the software; + + c) remove, minimize, block, or modify any notices of Microsoft or its suppliers in the software; + + d) use the software in any way that is against the law or to create or propagate malware; or + + e) share, publish, distribute, or lend the software (except for any distributable code, subject to the terms above), provide the software as a stand-alone hosted solution for others to use, or transfer the software or this agreement to any third party. + +5. EXPORT RESTRICTIONS. You must comply with all domestic and international export laws and regulations that apply to the software, which include restrictions on destinations, end users, and end use. For further information on export restrictions, visit http://aka.ms/exporting. + +6. SUPPORT SERVICES. Microsoft is not obligated under this agreement to provide any support services for the software. Any support provided is “as is”, “with all faults”, and without warranty of any kind. + +7. UPDATES. The software may periodically check for updates, and download and install them for you. You may obtain updates only from Microsoft or authorized sources. Microsoft may need to update your system to provide you with updates. You agree to receive these automatic updates without any additional notice. Updates may not include or support all existing software features, services, or peripheral devices. + +8. ENTIRE AGREEMENT. This agreement, and any other terms Microsoft may provide for supplements, updates, or third-party applications, is the entire agreement for the software. + +9. APPLICABLE LAW AND PLACE TO RESOLVE DISPUTES. If you acquired the software in the United States or Canada, the laws of the state or province where you live (or, if a business, where your principal place of business is located) govern the interpretation of this agreement, claims for its breach, and all other claims (including consumer protection, unfair competition, and tort claims), regardless of conflict of laws principles. If you acquired the software in any other country, its laws apply. If U.S. federal jurisdiction exists, you and Microsoft consent to exclusive jurisdiction and venue in the federal court in King County, Washington for all disputes heard in court. If not, you and Microsoft consent to exclusive jurisdiction and venue in the Superior Court of King County, Washington for all disputes heard in court. + +10. CONSUMER RIGHTS; REGIONAL VARIATIONS. This agreement describes certain legal rights. You may have other rights, including consumer rights, under the laws of your state or country. Separate and apart from your relationship with Microsoft, you may also have rights with respect to the party from which you acquired the software. This agreement does not change those other rights if the laws of your state or country do not permit it to do so. For example, if you acquired the software in one of the below regions, or mandatory country law applies, then the following provisions apply to you: + + a) Australia. You have statutory guarantees under the Australian Consumer Law and nothing in this agreement is intended to affect those rights. + + b) Canada. If you acquired this software in Canada, you may stop receiving updates by turning off the automatic update feature, disconnecting your device from the Internet (if and when you re-connect to the Internet, however, the software will resume checking for and installing updates), or uninstalling the software. The product documentation, if any, may also specify how to turn off updates for your specific device or software. + + c) Germany and Austria. + + i. Warranty. The properly licensed software will perform substantially as described in any Microsoft materials that accompany the software. However, Microsoft gives no contractual guarantee in relation to the licensed software. + + ii. Limitation of Liability. In case of intentional conduct, gross negligence, claims based on the Product Liability Act, as well as, in case of death or personal or physical injury, Microsoft is liable according to the statutory law. + + Subject to the foregoing clause ii., Microsoft will only be liable for slight negligence if Microsoft is in breach of such material contractual obligations, the fulfillment of which facilitate the due performance of this agreement, the breach of which would endanger the purpose of this agreement and the compliance with which a party may constantly trust in (so-called "cardinal obligations"). In other cases of slight negligence, Microsoft will not be liable for slight negligence. + +11. DISCLAIMER OF WARRANTY. THE SOFTWARE IS LICENSED “AS IS.” YOU BEAR THE RISK OF USING IT. MICROSOFT GIVES NO EXPRESS WARRANTIES, GUARANTEES, OR CONDITIONS. TO THE EXTENT PERMITTED UNDER APPLICABLE LAWS, MICROSOFT EXCLUDES ALL IMPLIED WARRANTIES, INCLUDING MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. + +12. LIMITATION ON AND EXCLUSION OF DAMAGES. IF YOU HAVE ANY BASIS FOR RECOVERING DAMAGES DESPITE THE PRECEDING DISCLAIMER OF WARRANTY, YOU CAN RECOVER FROM MICROSOFT AND ITS SUPPLIERS ONLY DIRECT DAMAGES UP TO U.S. $5.00. YOU CANNOT RECOVER ANY OTHER DAMAGES, INCLUDING CONSEQUENTIAL, LOST PROFITS, SPECIAL, INDIRECT, OR INCIDENTAL DAMAGES. + + This limitation applies to (a) anything related to the software, services, content (including code) on third party Internet sites, or third party applications; and (b) claims for breach of contract, warranty, guarantee, or condition; strict liability, negligence, or other tort; or any other claim; in each case to the extent permitted by applicable law. + + It also applies even if Microsoft knew or should have known about the possibility of the damages. The above limitation or exclusion may not apply to you because your state, province, or country may not allow the exclusion or limitation of incidental, consequential, or other damages. + +Please note: As this software is distributed in Canada, some of the clauses in this agreement are provided below in French. + +Remarque: Ce logiciel étant distribué au Canada, certaines des clauses dans ce contrat sont fournies ci-dessous en français. + +EXONÉRATION DE GARANTIE. Le logiciel visé par une licence est offert « tel quel ». Toute utilisation de ce logiciel est à votre seule risque et péril. Microsoft n’accorde aucune autre garantie expresse. Vous pouvez bénéficier de droits additionnels en vertu du droit local sur la protection des consommateurs, que ce contrat ne peut modifier. La ou elles sont permises par le droit locale, les garanties implicites de qualité marchande, d’adéquation à un usage particulier et d’absence de contrefaçon sont exclues. + +LIMITATION DES DOMMAGES-INTÉRÊTS ET EXCLUSION DE RESPONSABILITÉ POUR LES DOMMAGES. Vous pouvez obtenir de Microsoft et de ses fournisseurs une indemnisation en cas de dommages directs uniquement à hauteur de 5,00 $ US. Vous ne pouvez prétendre à aucune indemnisation pour les autres dommages, y compris les dommages spéciaux, indirects ou accessoires et pertes de bénéfices. + +Cette limitation concerne: + + • tout ce qui est relié au logiciel, aux services ou au contenu (y compris le code) figurant sur des sites Internet tiers ou dans des programmes tiers; et + + • les réclamations au titre de violation de contrat ou de garantie, ou au titre de responsabilité stricte, de négligence ou d’une autre faute dans la limite autorisée par la loi en vigueur. + +Elle s’applique également, même si Microsoft connaissait ou devrait connaître l’éventualité d’un tel dommage. Si votre pays n’autorise pas l’exclusion ou la limitation de responsabilité pour les dommages indirects, accessoires ou de quelque nature que ce soit, il se peut que la limitation ou l’exclusion ci-dessus ne s’appliquera pas à votre égard. + +EFFET JURIDIQUE. Le présent contrat décrit certains droits juridiques. Vous pourriez avoir d’autres droits prévus par les lois de votre pays. Le présent contrat ne modifie pas les droits que vous confèrent les lois de votre pays si celles-ci ne le permettent pas. \ No newline at end of file diff --git a/conda/mssql-python/MICROSOFT_VISUAL_STUDIO_LICENSE.txt b/conda/mssql-python/MICROSOFT_VISUAL_STUDIO_LICENSE.txt new file mode 100644 index 000000000..79c580d8f --- /dev/null +++ b/conda/mssql-python/MICROSOFT_VISUAL_STUDIO_LICENSE.txt @@ -0,0 +1,54 @@ +MICROSOFT SOFTWARE LICENSE TERMS +MICROSOFT VISUAL STUDIO 2017 TOOLS, ADD-ONs and EXTENSIONS + +These license terms are an agreement between Microsoft Corporation (or based on where you live, one of its affiliates) and you. +They apply to the software named above. The terms also apply to any Microsoft services or updates for the +software, except to the extent those have different terms. + +IF YOU COMPLY WITH THESE LICENSE TERMS, YOU HAVE THE RIGHTS BELOW. + +1. INSTALLATION AND USE RIGHTS. + You may install and use any number of copies of the software. + +2. TERMS FOR SPECIFIC COMPONENTS. + a. Utilities. The software may contain some items on the Utilities List at https://go.microsoft.com/fwlink/?linkid=823097. You may copy and install these Utilities, if included with the software, onto devices to debug and deploy your applications and databases you developed with the software. Please note that Utilities are designed for temporary use, that Microsoft may not be able to patch or update Utilities separately from the rest of the software, and that some Utilities by their nature may make it possible for others to access devices on which the Utilities are installed. As a result, you should delete all Utilities you have installed after you finish debugging or deploying your applications and databases. Microsoft is not responsible for any third party use or access of Utilities you install on any device. + b. Build Tools. The software may include build tools which have specific use terms. For build tools, you may copy and install files from the software onto your build devices, including physical devices and virtual machines or containers on those machines, whether on-premises or remote machines that are owned by you, hosted on Azure for you, or dedicated solely to your use (collectively, “Build Devices”). You and others in your organization may use these files on your Build Devices solely to compile, build, and verify applications or run quality or performance tests of those applications as part of the build process. For clarity, “applications” means applications developed by you and others in your organization who are each licensed to use the software. + c. Microsoft Platforms. The software may include components from Microsoft Windows; Microsoft Windows Server; Microsoft SQL Server; Microsoft Exchange; Microsoft Office; and Microsoft SharePoint. These components are governed by separate agreements and their own product support policies, as described in the Microsoft “Licenses” folder accompanying the software, except that, if license terms for those components are also included in the associated installation directory, those license terms control. + d. Third Party Components. The software may include third party components with separate legal notices or governed by other agreements, as may be described in the ThirdPartyNotices file(s) accompanying the software. + +3. DATA. + a. Data Collection. The software may collect information about you and your use of the software, and send that to Microsoft. Microsoft may use this information to provide services and improve our products and services. You may opt-out of many of these scenarios, but not all, as described in the product documentation. There are also some features in the software that may enable you and Microsoft to collect data from users of your applications. If you use these features, you must comply with applicable law, including providing appropriate notices to users of your applications and you should provide a copy of Microsoft’s privacy statement to your users. The Microsoft privacy statement is located here https://go.microsoft.com/fwlink/?LinkID=824704. You can learn more about data collection and use in the help documentation and our privacy statement. Your use of the software operates as your consent to these practices. + b. Processing of Personal Data. To the extent Microsoft is a processor or subprocessor of personal data in connection with the software, Microsoft makes the commitments in the European Union General Data Protection Regulation Terms of the Online Services Terms to all customers effective May 25, 2018, at http://go.microsoft.com/?linkid=9840733. + +4. SCOPE OF LICENSE. The software is licensed, not sold. This agreement only gives you some rights to use the software. Microsoft reserves all other rights. Unless applicable law gives you more rights despite this limitation, you may use the software only as expressly permitted in this agreement. In doing so, you must comply with any technical limitations in the software that only allow you to use it in certain ways. You may not + · work around any technical limitations in the software; + · reverse engineer, decompile or disassemble the software, or otherwise attempt to derive the source code for the software except, and only to the extent required by third party licensing terms governing the use of certain open source components that may be included in the software; + · remove, minimize, block or modify any notices of Microsoft or its suppliers in the software; + · use the software in any way that is against the law; or + · share, publish, rent or lease the software, or provide the software as a stand-alone hosted as solution for others to use, or transfer the software or this agreement to any third party. + +5. EXPORT RESTRICTIONS. You must comply with all domestic and international export laws and regulations that apply to the software, which include restrictions on destinations, end users, and end use. For further information on export restrictions, visit www.microsoft.com/exporting. + +6. SUPPORT SERVICES. Because this software is “as is,” we may not provide support services for it. + +7. ENTIRE AGREEMENT. This agreement, and the terms for supplements, updates, Internet-based services and support services that you use, are the entire agreement for the software and support services. + +8. APPLICABLE LAW. If you acquired the software in the United States, Washington law applies to interpretation of and claims for breach of this agreement, and the laws of the state where you live apply to all other claims. If you acquired the software in any other country, its laws apply. + +9. CONSUMER RIGHTS; REGIONAL VARIATIONS. This agreement describes certain legal rights. You may have other rights, including consumer rights, under the laws of your state or country. Separate and apart from your relationship with Microsoft, you may also have rights with respect to the party from which you acquired the software. This agreement does not change those other rights if the laws of your state or country do not permit it to do so. For example, if you acquired the software in one of the below regions, or mandatory country law applies, then the following provisions apply to you: + a. Australia. You have statutory guarantees under the Australian Consumer Law and nothing in this agreement is intended to affect those rights. + b. Canada. If you acquired this software in Canada, you may stop receiving updates by turning off the automatic update feature, disconnecting your device from the Internet (if and when you re-connect to the Internet, however, the software will resume checking for and installing updates), or uninstalling the software. The product documentation, if any, may also specify how to turn off updates for your specific device or software. + c. Germany and Austria. + (i) Warranty. The properly licensed software will perform substantially as described in any Microsoft materials that accompany the software. However, Microsoft gives no contractual guarantee in relation to the licensed software. + (ii) Limitation of Liability. In case of intentional conduct, gross negligence, claims based on the Product Liability Act, as well as, in case of death or personal or physical injury, Microsoft is liable according to the statutory law. + + Subject to the foregoing clause (ii), Microsoft will only be liable for slight negligence if Microsoft is in breach of such material contractual obligations, the fulfillment of which facilitate the due performance of this agreement, the breach of which would endanger the purpose of this agreement and the compliance with which a party may constantly trust in (so-called "cardinal obligations"). In other cases of slight negligence, Microsoft will not be liable for slight negligence. + +10. DISCLAIMER OF WARRANTY. THE SOFTWARE IS LICENSED “AS-IS.” YOU BEAR THE RISK OF USING IT. MICROSOFT GIVES NO EXPRESS WARRANTIES, GUARANTEES OR CONDITIONS. TO THE EXTENT PERMITTED UNDER YOUR LOCAL LAWS, MICROSOFT EXCLUDES THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. + +11. LIMITATION ON AND EXCLUSION OF DAMAGES. YOU CAN RECOVER FROM MICROSOFT AND ITS SUPPLIERS ONLY DIRECT DAMAGES UP TO U.S. $5.00. YOU CANNOT RECOVER ANY OTHER DAMAGES, INCLUDING CONSEQUENTIAL, LOST PROFITS, SPECIAL, INDIRECT OR INCIDENTAL DAMAGES. +This limitation applies to (a) anything related to the software, services, content (including code) on third party Internet sites, or third party applications; and (b) claims for breach of contract, breach of warranty, guarantee or condition, strict liability, negligence, or other tort to the extent permitted by applicable law. + +It also applies even if Microsoft knew or should have known about the possibility of the damages. The above limitation or exclusion may not apply to you because your country may not allow the exclusion or limitation of incidental, consequential or other damages. + +EULA ID: VS 2017_TOOLS_ADDONs_C++_RTW.3_ENU diff --git a/conda/mssql-python/bld.bat b/conda/mssql-python/bld.bat new file mode 100644 index 000000000..d73fbb16f --- /dev/null +++ b/conda/mssql-python/bld.bat @@ -0,0 +1,23 @@ +@echo on +REM Repackage the prebuilt, ESRP-signed wheel into a conda package (offline), and +REM vendor the ODBC Driver 18 payload INTO it (v1.11.0 model: libs ship inside). +REM PKG_NAME / PKG_VERSION are exported by conda-build; WHEELS_DIR + MSSQL_ODBC_VERSION +REM by the pipeline. +setlocal enabledelayedexpansion +"%PYTHON%" -m pip install --no-deps --no-index --find-links "%WHEELS_DIR%" %PKG_NAME%==%PKG_VERSION% -vv +if errorlevel 1 exit 1 + +REM Extract the python-agnostic py3-none-win odbc wheel into the SAME site-packages +REM so mssql_python_odbc\libs\ sits beside mssql_python\ (the loader finds the driver +REM there). WHEELS_DIR is staged per-target, so a single matching odbc wheel is present. +set "SP=%PREFIX%\Lib\site-packages" +if not exist "%SP%" mkdir "%SP%" +set "ODBC_WHL=" +for %%W in ("%WHEELS_DIR%\mssql_python_odbc-%MSSQL_ODBC_VERSION%-py3-none-win_*.whl") do set "ODBC_WHL=%%~fW" +if not defined ODBC_WHL ( + echo ERROR: no mssql_python_odbc==%MSSQL_ODBC_VERSION% py3-none-win wheel in "%WHEELS_DIR%" + exit 1 +) +echo Extracting "!ODBC_WHL!" into "%SP%" +tar -xf "!ODBC_WHL!" -C "%SP%" +if errorlevel 1 exit 1 diff --git a/conda/mssql-python/build.sh b/conda/mssql-python/build.sh new file mode 100644 index 000000000..c5ab3af0e --- /dev/null +++ b/conda/mssql-python/build.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Repackage the prebuilt, ESRP-signed wheel into a conda package (offline). +# PKG_NAME / PKG_VERSION are exported by conda-build; WHEELS_DIR by the pipeline/harness. +set -euo pipefail +# Cross-arch (emulated) build: when repackaging the aarch64 wheel on an x86_64 host, +# $PYTHON is the target-arch interpreter and runs under qemu-user. Point qemu at the +# aarch64 glibc loader/libs (installed via libc6-arm64-cross) so it can find +# /lib/ld-linux-aarch64.so.1 instead of aborting with "Could not open". The dir only +# exists on the emulated aarch64 leg; setting the var elsewhere is a harmless no-op. +[ -d /usr/aarch64-linux-gnu ] && export QEMU_LD_PREFIX="${QEMU_LD_PREFIX:-/usr/aarch64-linux-gnu}" + +# This package is SELF-CONTAINED (v1.11.0 model): the ODBC Driver 18 payload ships +# INSIDE it, so there is NO separate mssql-python-odbc conda package. We land BOTH +# the code wheel AND the python-agnostic py3-none- odbc wheel in the SAME +# site-packages, so mssql_python_odbc/libs/ sits beside mssql_python/ and the C++ +# loader resolves the driver there. WHEELS_DIR is staged per-target by the pipeline, +# so exactly one matching odbc wheel is present. +odbc_ver="${MSSQL_ODBC_VERSION:?MSSQL_ODBC_VERSION not set}" + +# The normal path installs with the host-env Python -- native builds, and the +# QEMU-emulated linux-aarch64 leg where the aarch64 Python runs under binfmt. pip +# resolves the correct site-packages for BOTH wheels, so no unzip is needed there. +# The osx-arm64 conda package is CROSS-built on an Intel macOS agent (no reverse +# Rosetta): the arm64 host Python CANNOT execute and pip would abort, so extract +# both wheels (zips) WITHOUT Python -- the same approach the Windows bld.bat uses +# with `tar`. macOS ships `unzip`. The arm64 slice comes from the universal2 wheel; +# conda-build still stamps osx-arm64. +if "$PYTHON" -c "import sys" >/dev/null 2>&1; then + "$PYTHON" -m pip install --no-deps --no-index --find-links "$WHEELS_DIR" "$PKG_NAME==$PKG_VERSION" -vv + "$PYTHON" -m pip install --no-deps --no-index --find-links "$WHEELS_DIR" "mssql-python-odbc==$odbc_ver" -vv +else + echo "Host Python '$PYTHON' is not executable on this agent (non-emulated cross-build);" + echo "extracting both wheels into \$SP_DIR without running Python." + mkdir -p "$SP_DIR" + pkg_underscore="${PKG_NAME//-/_}" + code_whl="" + for w in "$WHEELS_DIR/${pkg_underscore}-${PKG_VERSION}-"*.whl; do + [ -e "$w" ] && { code_whl="$w"; break; } + done + [ -n "$code_whl" ] || { echo "ERROR: no ${PKG_NAME}==${PKG_VERSION} wheel in '$WHEELS_DIR'" >&2; exit 1; } + odbc_whl="" + for w in "$WHEELS_DIR"/mssql_python_odbc-"$odbc_ver"-py3-none-*.whl; do + [ -e "$w" ] && { odbc_whl="$w"; break; } + done + [ -n "$odbc_whl" ] || { echo "ERROR: no mssql_python_odbc==$odbc_ver py3-none wheel in '$WHEELS_DIR'" >&2; exit 1; } + echo "Extracting '$code_whl' -> '$SP_DIR'" + unzip -oq "$code_whl" -d "$SP_DIR" + echo "Extracting '$odbc_whl' -> '$SP_DIR'" + unzip -oq "$odbc_whl" -d "$SP_DIR" +fi diff --git a/conda/mssql-python/meta.yaml b/conda/mssql-python/meta.yaml new file mode 100644 index 000000000..75ee3a0ca --- /dev/null +++ b/conda/mssql-python/meta.yaml @@ -0,0 +1,100 @@ +{% set version = environ.get('MSSQL_PYTHON_VERSION', '1.13.0') %} +{% set odbc_version = environ.get('MSSQL_ODBC_VERSION', '18.6.2.1') %} + +package: + name: mssql-python + version: "{{ version }}" + +build: + number: 0 + # Like the companion, this recipe REPACKAGES a prebuilt wheel (the compiled + # ddbc_bindings extension + bundled runtime); it compiles nothing. conda-build's + # overlinking/overdepending checks target from-source builds and mis-fire on + # vendored binaries (e.g. ddbc_bindings linking the driver that lives in the + # separate companion package), so downgrade both from errors to warnings -- CI + # enables them as errors by default. + error_overlinking: false + error_overdepending: false + # Like the companion, this recipe vendors PREBUILT, signed binaries (the compiled + # ddbc_bindings extension + bundled VC++ runtime); conda-build must neither rewrite + # nor scan them: + # - binary_relocation: rewriting RPATH / install-name in a signed binary corrupts + # the signature. + # - detect_binary_files_with_prefix: the build-prefix scan over the signed native + # binaries is meaningless for a pure repackage and is the packaging step that + # fails right after "Fixing permissions" on these recipes. + binary_relocation: false + detect_binary_files_with_prefix: false + # macOS builds this recipe for BOTH osx-64 (native on the Intel agent) and + # osx-arm64 (CROSS-built there). conda-build's .pyc byte-compilation runs the + # TARGET Python, which for osx-arm64 cannot execute on Intel -- so skip pyc on + # macOS (Python regenerates it at import). Linux/Windows legs are unaffected. + skip_compile_pyc: + - "**/*.py" # [osx] + # WHEELS_DIR is exported by the pipeline (or a local harness) and passed into the + # isolated conda-build environment so bld.bat / build.sh can install the prebuilt, + # ESRP-signed wheel from --find-links, fully offline. This mirrors CI: no PyPI. + script_env: + - WHEELS_DIR + # MSSQL_ODBC_VERSION lets build.sh / bld.bat locate the matching mssql-python-odbc + # wheel to vendor INTO this package (the driver payload now ships inside). + - MSSQL_ODBC_VERSION + +requirements: + host: + - python + - pip + run: + - python + # No version floor: on the `microsoft` channel azure-identity / azure-core / msal + # ship as CalVer (e.g. 2026.06.01), so a semver floor like `>=1.12.0` is a + # misleading no-op there (every published build already satisfies it). + - azure-identity + # --- ODBC Driver 18 payload deps (folded in from the former companion) -------- + # The proprietary driver libs now ship INSIDE this package (the v1.11.0 model: + # libs bundled in the wheel), so there is NO separate `mssql-python-odbc` conda + # package and its declared, security-serviced deps live here instead. + # + # OpenSSL: the driver dlopen's libssl/libcrypto for TLS (Encrypt=yes). Because it + # is dlopen'd (not an ELF NEEDED) conda-build's overlinking can't see it, so it + # must be declared. Linux-only, pinned >=3,<4 (Driver 18 supports the OpenSSL + # 1.1/3.0 ABI only; conda-forge has begun shipping openssl 4). macOS is excluded + # (the signed dylib dlopen's OpenSSL from a hardcoded Homebrew path -- users + # `brew install openssl`); Windows uses SChannel. + - openssl >=3,<4 # [linux] + # Kerberos: libmsodbcsql NEEDs libkrb5.so.3 + libgssapi_krb5.so.2 on Linux. macOS + # uses Kerberos.framework and Windows uses SSPI, so krb5 is Linux-only. + - krb5 # [linux] + # Windows VC++ runtime: msodbcsql18.dll imports VCRUNTIME140.dll, but the vendored + # vcredist ships only msvcp140.dll. Declare the security-serviced conda runtime. + - vc14_runtime # [win] + # conda drops the wheel's platform tag (manylinux_2_28 / macosx_15_0), so + # re-assert that floor as a virtual-package run constraint. The bundled wheels + # already required these, so this is never stricter than what shipped. + - __glibc >=2.28 # [linux] + - __osx >=15.0 # [osx] + +test: + imports: + - mssql_python + +about: + home: https://github.com/microsoft/mssql-python + # This package ships BOTH the MIT-licensed mssql-python code AND the proprietary + # Microsoft ODBC Driver 18 payload (+ the bundled VC++ runtime on Windows), so the + # license is the MIT code license AND the Microsoft proprietary EULA. + license: MIT AND LicenseRef-Microsoft-Proprietary + license_file: + - ../../LICENSE + - MICROSOFT_ODBC_DRIVER_FOR_SQL_SERVER_LICENSE.txt + - MICROSOFT_VISUAL_STUDIO_LICENSE.txt + summary: Microsoft driver for Python to interact with SQL Server and Azure SQL. + description: | + mssql-python is a DB API 2.0 (PEP 249) compliant driver for SQL Server, + Azure SQL, and Azure Synapse. This conda package is self-contained: the + proprietary Microsoft ODBC Driver 18 payload ships inside it (the same model as + the v1.11.0 wheel), so no separate driver package is required. + +extra: + recipe-maintainers: + - jahnvithakkar diff --git a/conda/tls_connect_probe.py b/conda/tls_connect_probe.py new file mode 100644 index 000000000..3005958b0 --- /dev/null +++ b/conda/tls_connect_probe.py @@ -0,0 +1,157 @@ +"""Live ``Encrypt=yes`` TLS gate: prove the driver's OpenSSL backend is REACHABLE. + +Why this exists (and why ``driver_load_probe.py`` is not enough): the Linux +``libmsodbcsql`` links ``libkrb5``/``libgssapi_krb5`` at load time but resolves +its OpenSSL backend (``libssl``/``libcrypto``) by **dlopen at TLS time** -- there +is no ``libssl``/``libcrypto`` ``DT_NEEDED`` or soname string in the binary, so +the crypto libraries are only touched when an actual encrypted handshake runs. +An ``Encrypt=no`` connect (what ``driver_load_probe.py`` does) NEVER exercises +that path, so it cannot reveal an unreachable OpenSSL -- e.g. a conda env where +the declared ``openssl`` lives in ``/lib`` that the vendored driver's +RUNPATH does not reach. Only a real ``Encrypt=yes`` handshake forces the dlopen. + +FAIL-CLOSED contract: + +* ``Encrypt`` is forced to ``yes`` (mandatory encryption), so the pre-login TLS + handshake MUST complete before any LOGIN7 packet is sent. Therefore ANY outcome + that reaches the authentication / database stage -- a clean connect, a + ``Login failed for user`` (18456), or a ``Cannot open database`` -- is POSITIVE + proof that OpenSSL loaded, negotiated, and established the encrypted channel. + These are the only PASS outcomes (see ``_TLS_COMPLETED_MARKERS``). +* Every other outcome fails closed (non-zero exit). In particular an OpenSSL that + could not be loaded surfaces BEFORE login as an ``SSL Provider`` / + ``libssl``/``libcrypto`` / ``cannot open shared object`` error -- classified + here as ``OPENSSL BACKEND UNREACHABLE`` (see ``_OPENSSL_UNREACHABLE_MARKERS``), + which is exactly the conda RUNPATH bug this gate is meant to catch. + +IMPORTANT -- masking caveat: this gate is only CONCLUSIVE on a minimal base with +NO system OpenSSL on the default loader path. On a full agent (or any host with a +system ``libssl``) the driver's dlopen can fall through to the system copy and the +handshake succeeds even when the conda ``/lib`` copy is unreachable -- +masking the very bug, just like the hosted CI agents do today. Run it in a +minimal container (no system OpenSSL) against a reachable server to make it +meaningful. The masking-IMMUNE static guard is +``eng/scripts/audit_bundled_binaries.py`` (it reads the RUNPATH bytes and requires +an ``$ORIGIN/..`` climb regardless of what system libs exist); this live gate is +the complementary end-to-end backstop. + +Config: set ``CONDA_TLS_PROBE_CONN`` to a reachable SQL Server connection string +(creds may be wrong -- reaching ``Login failed`` still proves TLS). If it is not +set the gate SKIPS loudly (exit 0) -- it never silently passes. + +Exit code 0 = TLS handshake completed (OpenSSL reachable) OR skipped; non-zero = +OpenSSL backend unreachable / handshake did not complete (blocks publish). +""" + +import os +import re +import sys + +# Outcomes that can ONLY occur AFTER a mandatory (Encrypt=yes) TLS handshake has +# completed -- i.e. positive proof the dlopen'd OpenSSL backend loaded and +# negotiated the encrypted channel. Matched case-insensitively. +_TLS_COMPLETED_MARKERS = ( + "login failed for user", # LOGIN7 rejected -> handshake already done + "18456", # SQL Server login-failed error number + "cannot open database", # authenticated, database validation stage + "changed database context", # connected successfully + "password did not match", +) + +# Markers that mean the crypto backend could NOT be loaded / the handshake never +# ran. Listed for a crisp FAIL message -- classification is allowlist-based, so an +# unrecognized outcome fails closed even if it matches nothing here. +_OPENSSL_UNREACHABLE_MARKERS = ( + "libssl", + "libcrypto", + "cannot open shared object", # linux dlopen failure of the crypto backend + "image not found", # macOS dlopen failure + "openssl", + "ssl provider", # an SSL Provider error before login = crypto/handshake fail + "ssl routines", + "encryption not supported", + "unable to load", + "cannot load", +) + + +def tls_completed(exc): + """FAIL-CLOSED classifier: True only when the TLS handshake provably completed. + + ``exc is None`` (clean connect) or a post-handshake authentication/database + diagnostic returns True; every other outcome -- including an OpenSSL-load + failure or anything unrecognized -- returns False so the gate exits non-zero. + """ + if exc is None: + return True + msg = str(exc).lower() + return any(marker in msg for marker in _TLS_COMPLETED_MARKERS) + + +def describe(exc): + """Short, human-readable reason string for the gate's stdout / exit line.""" + if exc is None: + return "clean connect (TLS handshake completed)" + msg = str(exc) + low = msg.lower() + for marker in _OPENSSL_UNREACHABLE_MARKERS: + if marker in low: + return "OpenSSL backend unreachable -> " + msg[:300] + return msg[:300] + + +def force_tls(conn): + """Force ``Encrypt=yes`` and ``TrustServerCertificate=yes`` on the string. + + Encrypt=yes makes the pre-login TLS handshake mandatory (the whole point of + the gate). TrustServerCertificate=yes lets it reach the auth stage against a + local dev server's self-signed cert -- this is a local connectivity gate, NOT + a security assertion, and must never be copied into a production connection. + """ + + def set_kv(s, key, val): + pat = re.compile(r"(?i)(^|;)\s*" + re.escape(key) + r"\s*=\s*[^;]*") + if pat.search(s): + return pat.sub(lambda m: (m.group(1) or "") + key + "=" + val, s, count=1) + return s + ";" + key + "=" + val + + conn = conn.strip().rstrip(";") + conn = set_kv(conn, "Encrypt", "yes") + conn = set_kv(conn, "TrustServerCertificate", "yes") + return conn + + +def main(): + raw = os.environ.get("CONDA_TLS_PROBE_CONN", "").strip() + if not raw: + print( + "TLS_PROBE_SKIPPED: set CONDA_TLS_PROBE_CONN to a reachable SQL Server " + "connection string (on a minimal base with no system OpenSSL) to run " + "this Encrypt=yes gate." + ) + return + + conn_str = force_tls(raw) + + # Deferred so this module can be imported (and the classifier unit-tested) + # WITHOUT the compiled extension / driver payload. + import mssql_python + + outcome = None + try: + conn = mssql_python.connect(conn_str) + try: + conn.close() + except Exception: # noqa: BLE001 - best-effort cleanup only + pass + except Exception as exc: # noqa: BLE001 - deliberately classified below + outcome = exc + + if tls_completed(outcome): + print("TLS_OK (OpenSSL backend reachable; " + describe(outcome) + ")") + return + sys.exit("TLS/OPENSSL BACKEND UNREACHABLE: " + describe(outcome)) + + +if __name__ == "__main__": + main() diff --git a/conda/validate_conda_release.py b/conda/validate_conda_release.py new file mode 100644 index 000000000..1baffb55a --- /dev/null +++ b/conda/validate_conda_release.py @@ -0,0 +1,255 @@ +"""Metadata-based conda release-readiness gate. + +The release pipeline must never ship an incomplete conda set. This module reads +the AUTHORITATIVE ``info/index.json`` embedded in every ``.conda`` / ``.tar.bz2`` +(never folder names or bare counts) and validates the self-contained +``mssql-python`` package -- which vendors the ODBC Driver 18 payload, so there is +NO separate companion package: + +* every package's real ``subdir`` is in the allowed set AND matches its folder + (catches a mislabeled / mis-stamped leg); +* the only package name is ``mssql-python`` and its version matches the expected + release version (or, if none supplied, is internally consistent -- one version); +* the full (required-subdir x Python) matrix is complete -- every required + platform ships a package for every expected Python. + +Exit code 0 = release-ready; non-zero = a violation was found (blocks publish). +""" + +from __future__ import annotations + +import argparse +import io +import json +import re +import sys +import tarfile +import zipfile +from collections import defaultdict + +_BINDING_NAME = "mssql-python" + +_PY_TAG_RE = re.compile(r"py(\d)(\d{1,2})") +_PY_DEP_RE = re.compile(r"python\s+(\d+)\.(\d+)") + + +def _zstd_decompress(raw: bytes) -> bytes: + """Decompress a zstandard blob, preferring the 3.14+ stdlib backend.""" + try: # Python 3.14+ + from compression import zstd # type: ignore + + return zstd.decompress(raw) + except Exception: # pragma: no cover - exercised via the third-party path + pass + import zstandard # third-party fallback + + return zstandard.ZstdDecompressor().decompress(raw) + + +def read_index_json(path: str) -> dict: + """Return the parsed ``info/index.json`` from a ``.conda`` / ``.tar.bz2``.""" + if path.endswith(".conda"): + with zipfile.ZipFile(path) as zf: + info_name = next( + n for n in zf.namelist() if n.startswith("info-") and n.endswith(".tar.zst") + ) + info_blob = zf.read(info_name) + with tarfile.open(fileobj=io.BytesIO(_zstd_decompress(info_blob))) as tf: + member = tf.extractfile("info/index.json") + if member is None: # pragma: no cover - malformed package + raise ValueError(f"{path}: info/index.json missing") + return json.load(member) + if path.endswith(".tar.bz2"): + with tarfile.open(path, "r:bz2") as tf: + member = tf.extractfile("info/index.json") + if member is None: # pragma: no cover - malformed package + raise ValueError(f"{path}: info/index.json missing") + return json.load(member) + raise ValueError(f"{path}: unrecognized conda package extension") + + +def python_tag_from_index(index: dict) -> str: + """Extract the ``X.Y`` Python version a package is built for, or ``''``. + + Uses the build string's ``pyXY`` token first (authoritative for conda-build + Python packages), then falls back to a ``python X.Y`` run dependency. A + Python-agnostic package (build string ``0``) has neither and returns ``''``. + """ + match = _PY_TAG_RE.search(str(index.get("build", ""))) + if match: + return f"{match.group(1)}.{match.group(2)}" + for dep in index.get("depends", []) or []: + match = _PY_DEP_RE.match(str(dep)) + if match: + return f"{match.group(1)}.{match.group(2)}" + return "" + + +def validate( + packages: list[dict], + required_subdirs: list[str], + allowed_subdirs: list[str], + expected_pythons: list[str], + expected_versions: dict | None = None, +) -> list[str]: + """Return a list of human-readable violation strings (empty == release-ready). + + ``packages`` is a list of dicts with keys: ``folder`` (staged subdir folder), + ``subdir`` (real info/index.json subdir), ``name``, ``version``, ``build``, + ``python`` (``X.Y`` or ``''``). + """ + errors: list[str] = [] + expected_versions = expected_versions or {} + + # 1. Authoritative subdir must be allowed AND match the folder it was staged in. + for p in packages: + ident = f"{p['name']}-{p['version']}-{p['build']}" + if p["subdir"] not in allowed_subdirs: + errors.append( + f"{ident}: real subdir '{p['subdir']}' is not in allowed set {allowed_subdirs}." + ) + if p["subdir"] != p["folder"]: + errors.append( + f"MISLABELED: {ident} is staged in folder '{p['folder']}' but its " + f"info/index.json subdir is '{p['subdir']}'." + ) + + # 2. Only the self-contained mssql-python package may appear; versions match + # expected (or are internally consistent -- one version per package). + seen_versions: dict = defaultdict(set) + for p in packages: + if p["name"] != _BINDING_NAME: + errors.append( + f"unexpected package name '{p['name']}' ({p['version']}); the " + f"self-contained conda package ships only '{_BINDING_NAME}'." + ) + continue + seen_versions[p["name"]].add(p["version"]) + for name, versions in seen_versions.items(): + if len(versions) > 1: + errors.append( + f"{name}: multiple versions present {sorted(versions)} " + f"(a release must ship exactly one version per package)." + ) + exp = expected_versions.get(name) + if exp is not None: + for v in versions: + if v != exp: + errors.append(f"{name}: version '{v}' != expected '{exp}'.") + + # Group by the REAL (metadata) subdir, never the folder name. + by_subdir: dict = defaultdict(list) + for p in packages: + by_subdir[p["subdir"]].append(p) + + # 3. Required subdirs: present with a COMPLETE per-Python matrix. + for sub in required_subdirs: + grp = by_subdir.get(sub, []) + if not grp: + errors.append(f"required subdir '{sub}' is MISSING.") + continue + bindings = [p for p in grp if p["name"] == _BINDING_NAME] + if not bindings: + errors.append(f"subdir '{sub}': no {_BINDING_NAME} package.") + + for p in bindings: + if not p["python"]: + errors.append( + f"{p['name']}-{p['version']}-{p['build']} in '{sub}' has no " + f"detectable Python tag (build string should carry pyXY)." + ) + got_pythons = sorted({p["python"] for p in bindings if p["python"]}) + missing = [py for py in expected_pythons if py not in got_pythons] + if missing: + errors.append( + f"subdir '{sub}': matrix INCOMPLETE -- missing Python {missing} " + f"(present: {got_pythons or 'none'})." + ) + + return errors + + +def collect_packages(root: str) -> list[dict]: + """Read every ``.conda`` / ``.tar.bz2`` under ``root`` into package dicts.""" + import glob + import os + + paths = sorted( + glob.glob(os.path.join(root, "**", "*.conda"), recursive=True) + + glob.glob(os.path.join(root, "**", "*.tar.bz2"), recursive=True) + ) + packages = [] + for path in paths: + index = read_index_json(path) + packages.append( + { + "folder": os.path.basename(os.path.dirname(path)), + "subdir": str(index.get("subdir", "")), + "name": str(index.get("name", "")), + "version": str(index.get("version", "")), + "build": str(index.get("build", "")), + "python": python_tag_from_index(index), + "path": path, + } + ) + return packages + + +def _split(value: str) -> list[str]: + return [x.strip() for x in value.split(",") if x.strip()] + + +def main(argv: list | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", required=True, help="Root of the consolidated conda tree.") + parser.add_argument( + "--required-subdirs", default="win-64,osx-64,osx-arm64,linux-64,linux-aarch64" + ) + parser.add_argument( + "--allowed-subdirs", + default="win-64,win-arm64,osx-64,osx-arm64,linux-64,linux-aarch64", + ) + parser.add_argument("--pythons", default="3.10,3.11,3.12,3.13,3.14") + parser.add_argument("--mssql-python-version", default=None) + parser.add_argument("--mssql-python-odbc-version", default=None) + args = parser.parse_args(argv) + + packages = collect_packages(args.root) + if not packages: + print(f"ERROR: no conda packages found under {args.root}.", file=sys.stderr) + return 1 + + expected_versions = {} + if args.mssql_python_version: + expected_versions[_BINDING_NAME] = args.mssql_python_version + # --mssql-python-odbc-version is accepted for back-compat but ignored: the + # self-contained mssql-python package vendors the ODBC payload, so there is no + # separate companion package to version. + + print(f"Discovered {len(packages)} conda package(s):") + for p in sorted(packages, key=lambda x: (x["subdir"], x["name"], x["python"])): + print( + f" {p['subdir']:<14} {p['name']:<18} {p['version']:<12} " + f"py={p['python'] or '-':<5} build={p['build']}" + ) + + errors = validate( + packages, + required_subdirs=_split(args.required_subdirs), + allowed_subdirs=_split(args.allowed_subdirs), + expected_pythons=_split(args.pythons), + expected_versions=expected_versions, + ) + + if errors: + print("\nConda release readiness FAILED:", file=sys.stderr) + for e in errors: + print(f" - {e}", file=sys.stderr) + return 1 + + print("\nOK: metadata-validated conda set is release-ready (subdirs, Python matrix, pairing).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_026_driver_load_probe.py b/tests/test_026_driver_load_probe.py new file mode 100644 index 000000000..71feceb5d --- /dev/null +++ b/tests/test_026_driver_load_probe.py @@ -0,0 +1,144 @@ +"""Fail-closed classification tests for ``conda/driver_load_probe.py``. + +The conda test-before-publish gate runs ``conda/driver_load_probe.py`` to prove +the repackaged native ODBC driver actually loads (not just the tiny +``mssql_python_odbc`` shim). The probe MUST fail closed: a broken / missing / +mis-architecture driver -- whose failure surfaces as the C++ +``LoadDriverOrThrowException`` family ("Failed to load the driver...", "Failed +to load library: ", "Failed to load required function pointers...") -- has +to make the probe exit non-zero, while a genuine connection-stage failure +(driver loaded, TCP/TLS/auth attempted) has to pass. + +These are pure, no-DB unit tests: the probe's native ``import mssql_python`` is +deferred into ``main()``, so the classifier can be loaded and exercised with a +stubbed connector without the compiled extension or a live SQL Server. +""" + +import importlib.util +import sys +import types +from pathlib import Path + +import pytest + +_PROBE_PATH = Path(__file__).resolve().parent.parent / "conda" / "driver_load_probe.py" + +# The conda/ sources are not shipped inside the built wheel, so the installed-wheel +# test leg copies only tests/ into an isolated dir. Skip the whole module (rather than +# erroring at collection/run) when the conda source it exercises is absent. +if not _PROBE_PATH.is_file(): + pytest.skip( + f"conda source not present ({_PROBE_PATH}); skipping conda driver-load probe tests", + allow_module_level=True, + ) + + +def _load_probe(): + """Import ``conda/driver_load_probe.py`` as a standalone module.""" + spec = importlib.util.spec_from_file_location("driver_load_probe_under_test", _PROBE_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# Messages the loaded msodbcsql driver emits once it has reached the network / +# TLS / auth stage. Every one of these MUST classify as "driver loaded" (PASS). +_LOADED_MESSAGES = [ + "Driver Error: Connection operation failed; DDBC Error: [Microsoft][ODBC Driver 18 for " + "SQL Server]TCP Provider: No connection could be made because the target machine actively " + "refused it.", + "[Microsoft][ODBC Driver 18 for SQL Server]Login timeout expired", + "[Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: Error code 0x2726", + "[Microsoft][ODBC Driver 18 for SQL Server]A network-related or instance-specific error " + "has occurred", + "[Microsoft][ODBC Driver 18 for SQL Server]SSL Provider: certificate verify failed", + "[Microsoft][ODBC Driver 18 for SQL Server]Login failed for user 'x'.", + "connection refused", +] + +# Messages that mean the native driver did NOT load / link / resolve. Every one +# of these MUST classify as "not loaded" (FAIL / non-zero exit). +_LOAD_FAILURE_MESSAGES = [ + "Failed to load the driver. Please read the documentation " + "(https://github.com/microsoft/mssql-python#installation) to install the required " + "dependencies.", + "Failed to load library: C:\\x\\msodbcsql18.dll", + "Failed to load required function pointers from driver.", + "ODBC driver not found at: /x/libmsodbcsql-18.5.so.2.1", + "Failed to load mssql-auth.dll. Please ensure it is present in the expected directory.", + "mssql-auth.dll not found. If you are using Entra ID, please ensure it is present.", + "The mssql-python-odbc package (which ships the ODBC driver binaries) is not installed.", + "dlopen(...): image not found", + "libcrypto.so.3: cannot open shared object file: No such file or directory", + "Unsupported architecture", + # Fail-closed default: an unexpected / unrelated error is NOT proof of load. + "some totally unexpected internal error", +] + + +@pytest.mark.parametrize("msg", _LOADED_MESSAGES) +def test_driver_loaded_true_for_connection_stage_errors(msg): + probe = _load_probe() + assert probe.driver_loaded(RuntimeError(msg)) is True + + +@pytest.mark.parametrize("msg", _LOAD_FAILURE_MESSAGES) +def test_driver_loaded_false_for_load_failures(msg): + probe = _load_probe() + assert probe.driver_loaded(RuntimeError(msg)) is False + + +def test_driver_loaded_true_for_clean_connect(): + probe = _load_probe() + assert probe.driver_loaded(None) is True + + +def _run_main_with_stub(monkeypatch, connect): + """Run ``probe.main()`` with a stubbed ``mssql_python`` module.""" + probe = _load_probe() + stub = types.ModuleType("mssql_python") + stub.connect = connect + monkeypatch.setitem(sys.modules, "mssql_python", stub) + return probe + + +def test_main_exits_nonzero_on_simulated_load_failure(monkeypatch): + def connect(_conn_str): + raise RuntimeError( + "Failed to load the driver. Please read the documentation to install the " + "required dependencies." + ) + + probe = _run_main_with_stub(monkeypatch, connect) + with pytest.raises(SystemExit) as excinfo: + probe.main() + # sys.exit() -> non-zero (truthy) exit code carrying the reason. + assert excinfo.value.code + assert "DRIVER DID NOT LOAD" in str(excinfo.value.code) + + +def test_main_passes_on_simulated_network_failure(monkeypatch): + def connect(_conn_str): + raise RuntimeError( + "[Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: No connection could be " + "made because the target machine actively refused it." + ) + + probe = _run_main_with_stub(monkeypatch, connect) + # A genuine connection-stage failure must NOT raise SystemExit (exit 0). + probe.main() + + +def test_main_passes_on_clean_connect(monkeypatch): + closed = {"value": False} + + class _Conn: + def close(self): + closed["value"] = True + + def connect(_conn_str): + return _Conn() + + probe = _run_main_with_stub(monkeypatch, connect) + probe.main() + assert closed["value"] is True diff --git a/tests/test_027_conda_release_metadata.py b/tests/test_027_conda_release_metadata.py new file mode 100644 index 000000000..c8f59bc74 --- /dev/null +++ b/tests/test_027_conda_release_metadata.py @@ -0,0 +1,200 @@ +"""Unit tests for the metadata-based conda release gate. + +``conda/validate_conda_release.py`` reads each package's authoritative +``info/index.json`` and enforces: real-subdir == folder, allowed subdirs, the +full (subdir x Python) matrix, and exact versions for the self-contained +``mssql-python`` package (which vendors the ODBC payload -- no companion). These +tests exercise the pure ``validate()`` logic with synthetic package records (no +real ``.conda`` needed) plus one optional round-trip through the metadata reader. +""" + +import importlib.util +import io +import json +import tarfile +from pathlib import Path + +import pytest + +_MODULE_PATH = Path(__file__).resolve().parent.parent / "conda" / "validate_conda_release.py" + +# The conda/ sources are not shipped inside the built wheel, so the installed-wheel +# test leg copies only tests/ into an isolated dir. Skip the whole module (rather than +# erroring at collection) when the conda source it exercises is absent. +if not _MODULE_PATH.is_file(): + pytest.skip( + f"conda source not present ({_MODULE_PATH}); skipping conda release metadata tests", + allow_module_level=True, + ) + + +def _load_module(): + spec = importlib.util.spec_from_file_location("validate_conda_release_under_test", _MODULE_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +vcr = _load_module() + +_REQUIRED = ["win-64", "osx-64", "osx-arm64", "linux-64", "linux-aarch64"] +_ALLOWED = ["win-64", "win-arm64", "osx-64", "osx-arm64", "linux-64", "linux-aarch64"] +_PYTHONS = ["3.10", "3.11", "3.12", "3.13", "3.14"] +_MP_VER = "1.13.0" + + +def _binding(subdir, py, folder=None, version=_MP_VER): + return { + "folder": folder or subdir, + "subdir": subdir, + "name": "mssql-python", + "version": version, + "build": f"py{py.replace('.', '')}_0", + "python": py, + } + + +def _healthy_set(): + """A complete release: the self-contained mssql-python package for every + (required subdir x Python).""" + pkgs = [] + for sub in _REQUIRED: + for py in _PYTHONS: + pkgs.append(_binding(sub, py)) + return pkgs + + +def _run(pkgs, expected_versions=None): + return vcr.validate( + pkgs, + required_subdirs=_REQUIRED, + allowed_subdirs=_ALLOWED, + expected_pythons=_PYTHONS, + expected_versions=( + expected_versions if expected_versions is not None else {"mssql-python": _MP_VER} + ), + ) + + +def test_healthy_set_passes(): + assert _run(_healthy_set()) == [] + + +def test_mislabeled_subdir_fails(): + pkgs = _healthy_set() + # An osx-64 package physically staged into the osx-arm64 folder. + pkgs.append(_binding("osx-64", "3.12", folder="osx-arm64")) + errors = _run(pkgs) + assert any("MISLABELED" in e for e in errors) + + +def test_missing_python_variant_on_win64_fails(): + # This is the exact 8e7f217f regression: drop a win-64 binding; presence-pairing + # against the single companion used to pass, metadata matrix must now fail. + pkgs = [p for p in _healthy_set() if not (p["subdir"] == "win-64" and p["python"] == "3.12")] + errors = _run(pkgs) + assert any("win-64" in e and "INCOMPLETE" in e and "3.12" in e for e in errors) + + +def test_stray_companion_package_fails(): + # The self-contained model ships ONLY mssql-python; a stray companion package + # (the old separate mssql-python-odbc) must now be rejected as unexpected. + pkgs = _healthy_set() + pkgs.append( + { + "folder": "linux-64", + "subdir": "linux-64", + "name": "mssql-python-odbc", + "version": "18.6.2.1", + "build": "0", + "python": "", + } + ) + errors = _run(pkgs) + assert any("unexpected package name" in e and "mssql-python-odbc" in e for e in errors) + + +def test_unexpected_subdir_fails(): + pkgs = _healthy_set() + pkgs.append(_binding("linux-ppc64le", "3.12")) + errors = _run(pkgs) + assert any("linux-ppc64le" in e and "allowed" in e for e in errors) + + +def test_version_mismatch_fails(): + pkgs = _healthy_set() + pkgs.append(_binding("linux-64", "3.14", version="9.9.9")) # stray wrong-version binding + # remove the correct 3.14 to avoid duplicate-python noise masking the version check + pkgs = [ + p + for p in pkgs + if not (p["subdir"] == "linux-64" and p["python"] == "3.14" and p["version"] == _MP_VER) + ] + errors = _run(pkgs) + assert any("version" in e.lower() for e in errors) + + +def test_multiple_versions_same_package_fails(): + pkgs = _healthy_set() + pkgs.append(_binding("linux-64", "3.10", version="1.12.0", folder="linux-64")) + errors = _run(pkgs, expected_versions={}) # no expected -> consistency check must still fail + assert any("multiple versions" in e for e in errors) + + +def test_missing_required_subdir_fails(): + pkgs = [p for p in _healthy_set() if p["subdir"] != "linux-aarch64"] + errors = _run(pkgs) + assert any("linux-aarch64" in e and "MISSING" in e for e in errors) + + +def test_python_tag_from_index(): + assert vcr.python_tag_from_index({"build": "py311_0"}) == "3.11" + assert vcr.python_tag_from_index({"build": "py310h1a2b3c_0"}) == "3.10" + assert ( + vcr.python_tag_from_index({"build": "0", "depends": ["python 3.12.* *_cpython"]}) == "3.12" + ) + assert vcr.python_tag_from_index({"build": "0"}) == "" + + +def _zstd_available(): + try: + from compression import zstd # noqa: F401 # py3.14+ + + return True + except Exception: + try: + import zstandard # noqa: F401 + + return True + except Exception: + return False + + +@pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") +def test_read_index_json_roundtrip(tmp_path): + import zipfile + + index = {"name": "mssql-python", "version": _MP_VER, "build": "py312_0", "subdir": "win-64"} + # Build info/index.json -> tar -> zstd -> .conda zip, then read it back. + tar_buf = io.BytesIO() + with tarfile.open(fileobj=tar_buf, mode="w") as tf: + data = json.dumps(index).encode() + ti = tarfile.TarInfo("info/index.json") + ti.size = len(data) + tf.addfile(ti, io.BytesIO(data)) + try: + from compression import zstd # py3.14+ + + compressed = zstd.compress(tar_buf.getvalue()) + except Exception: + import zstandard + + compressed = zstandard.ZstdCompressor().compress(tar_buf.getvalue()) + + conda_path = tmp_path / "mssql-python-1.13.0-py312_0.conda" + with zipfile.ZipFile(conda_path, "w") as zf: + zf.writestr("info-mssql-python-1.13.0-py312_0.tar.zst", compressed) + + got = vcr.read_index_json(str(conda_path)) + assert got["subdir"] == "win-64" + assert vcr.python_tag_from_index(got) == "3.12" diff --git a/tests/test_028_tls_connect_probe.py b/tests/test_028_tls_connect_probe.py new file mode 100644 index 000000000..fefb18bcf --- /dev/null +++ b/tests/test_028_tls_connect_probe.py @@ -0,0 +1,107 @@ +"""Fail-closed classification tests for ``conda/tls_connect_probe.py``. + +The live ``Encrypt=yes`` conda gate runs ``conda/tls_connect_probe.py`` to prove +the driver's dlopen'd OpenSSL backend (``libssl``/``libcrypto``) is REACHABLE -- +something the DB-less ``Encrypt=no`` ``driver_load_probe.py`` cannot show, because +the crypto libraries are only touched by a real TLS handshake. The classifier MUST +fail closed: only an outcome that provably means the mandatory pre-login TLS +handshake completed (a clean connect, a ``Login failed`` / 18456, or a +``Cannot open database``) may PASS; an OpenSSL-load failure or anything +unrecognized MUST fail. + +These are pure, no-DB unit tests: the probe's native ``import mssql_python`` is +deferred into ``main()``, so the classifier + ``force_tls`` can be exercised +without the compiled extension or a live SQL Server. +""" + +import importlib.util +from pathlib import Path + +import pytest + +_PROBE_PATH = Path(__file__).resolve().parent.parent / "conda" / "tls_connect_probe.py" + +# The conda/ sources are not shipped inside the built wheel, so the installed-wheel +# test leg copies only tests/ into an isolated dir. Skip the whole module (rather than +# erroring at collection/run) when the conda source it exercises is absent. +if not _PROBE_PATH.is_file(): + pytest.skip( + f"conda source not present ({_PROBE_PATH}); skipping conda TLS-probe tests", + allow_module_level=True, + ) + + +def _load_probe(): + """Import ``conda/tls_connect_probe.py`` as a standalone module.""" + spec = importlib.util.spec_from_file_location("tls_connect_probe_under_test", _PROBE_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# Outcomes that can only occur AFTER a mandatory Encrypt=yes handshake completes; +# every one MUST classify as "TLS completed" (PASS -> OpenSSL was reachable). +_TLS_COMPLETED_MESSAGES = [ + "[Microsoft][ODBC Driver 18 for SQL Server]Login failed for user 'x'.", + "Login failed for user 'sa'. (18456)", + '[Microsoft][ODBC Driver 18 for SQL Server]Cannot open database "X" requested by the ' + "login. The login failed.", + "Changed database context to 'master'.", +] + +# Outcomes that mean the crypto backend never loaded / the handshake never +# completed; every one MUST classify as "not completed" (FAIL / non-zero exit). +_TLS_FAILURE_MESSAGES = [ + "[Microsoft][ODBC Driver 18 for SQL Server]SSL Provider: The certificate chain was issued " + "by an authority that is not trusted.", + "libssl.so.3: cannot open shared object file: No such file or directory", + "libcrypto.so.3: cannot open shared object file: No such file or directory", + "[Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: Error code 0x2726", + "[Microsoft][ODBC Driver 18 for SQL Server]Login timeout expired", + "dlopen(libssl.dylib): image not found", + # Fail-closed default: an unexpected / unrelated error is NOT proof of a + # completed handshake. + "some totally unexpected internal error", +] + + +@pytest.mark.parametrize("msg", _TLS_COMPLETED_MESSAGES) +def test_tls_completed_true_for_post_handshake_outcomes(msg): + probe = _load_probe() + assert probe.tls_completed(RuntimeError(msg)) is True + + +@pytest.mark.parametrize("msg", _TLS_FAILURE_MESSAGES) +def test_tls_completed_false_for_pre_handshake_failures(msg): + probe = _load_probe() + assert probe.tls_completed(RuntimeError(msg)) is False + + +def test_tls_completed_true_for_clean_connect(): + probe = _load_probe() + assert probe.tls_completed(None) is True + + +def test_force_tls_appends_when_absent(): + probe = _load_probe() + out = probe.force_tls("Server=localhost;Database=x;Uid=x;Pwd=x;") + assert "Encrypt=yes" in out + assert "TrustServerCertificate=yes" in out + + +def test_force_tls_overrides_encrypt_no(): + probe = _load_probe() + out = probe.force_tls("Server=localhost;Encrypt=no;Database=x") + low = out.lower() + assert "encrypt=yes" in low + assert "encrypt=no" not in low + + +def test_force_tls_is_idempotent(): + probe = _load_probe() + once = probe.force_tls("Server=localhost;Database=x") + twice = probe.force_tls(once) + assert once == twice + # Exactly one Encrypt= and one TrustServerCertificate= key. + assert twice.lower().count("encrypt=") == 1 + assert twice.lower().count("trustservercertificate=") == 1