From 2ddf66bb21ba91e27e1f4f88265b6e163350d482 Mon Sep 17 00:00:00 2001 From: mwaisaida-13 Date: Tue, 18 Aug 2026 17:57:58 +0100 Subject: [PATCH 1/4] Add PyPI release workflow and Dependabot config python-release.yml publishes to PyPI on python-v* GitHub Releases via Trusted Publishing (OIDC) + PEP 740 attestations. A full 3.10-3.14 test matrix gates build/publish; the build job verifies tag==version and refuses while the 'Private :: Do Not Upload' guard is set. All actions are SHA-pinned, build/twine pinned and installed with --isolated, and a post-publish smoke test installs from PyPI and runs the CLI. dependabot.yml adds weekly grouped updates for github-actions SHAs and the Python (pip) dependencies. --- .github/dependabot.yml | 22 +++++ .github/workflows/python-release.yml | 143 +++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/python-release.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..e6fee30 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,22 @@ +# Keeps pinned dependencies current. Takes effect only on the default branch. +# https://docs.github.com/code-security/dependabot/dependabot-version-updates +version: 2 +updates: + # GitHub Actions: bumps the SHA pins and their `# vX.Y.Z` comments in workflows. + # Grouped into one PR to avoid five separate bumps. + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + actions: + patterns: ["*"] + + # Python deps in python/pyproject.toml. + - package-ecosystem: pip + directory: /python + schedule: + interval: weekly + groups: + python: + patterns: ["*"] diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml new file mode 100644 index 0000000..7538e44 --- /dev/null +++ b/.github/workflows/python-release.yml @@ -0,0 +1,143 @@ +name: Publish Python package to PyPI + +# Trusted publisher on PyPI must reference this exact filename and the `pypi` +# environment, or publishing is rejected. + +on: + release: + types: [published] + +permissions: + contents: read + +# cancel-in-progress: false — cancelling mid-publish can corrupt a release. +concurrency: + group: python-release-${{ github.ref }} + cancel-in-progress: false + +jobs: + # Full support matrix gates the release; build/publish depend on it. + # Re-runs the ubuntu CI tests here as one of the release gates. + # python-v* tags only + test: + name: test (py${{ matrix.python-version }}) + if: startsWith(github.ref_name, 'python-v') + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false # report every version, not just the first failure + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + defaults: + run: + working-directory: python + steps: + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "${{ matrix.python-version }}" + + - name: Install and run the test suite + timeout-minutes: 10 + run: | + python -m pip install -e ".[dev]" + python -m pytest -q + + build: + name: build and verify + needs: test + if: startsWith(github.ref_name, 'python-v') + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: python + steps: + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + # Tag must match pyproject.toml version, and the `Private :: Do Not Upload` + # guard must be gone — fail the release loudly otherwise. + - name: Verify the tag matches the packaged version and the package is publishable + run: | + set -euo pipefail + tag_version="${GITHUB_REF_NAME#python-v}" + pkg_version="$(python -c 'import tomllib, pathlib; print(tomllib.loads(pathlib.Path("pyproject.toml").read_text())["project"]["version"])')" + if [ "$tag_version" != "$pkg_version" ]; then + echo "::error::Release tag is python-v${tag_version} but pyproject.toml declares ${pkg_version}. Bump the version and re-tag." + exit 1 + fi + if grep -q "Private :: Do Not Upload" pyproject.toml; then + echo "::error::The 'Private :: Do Not Upload' classifier is still set. Remove it in the release commit once the open-source review has cleared." + exit 1 + fi + echo "Publishing version ${pkg_version}." + + - name: Build wheel and sdist + timeout-minutes: 5 + run: | + python -m pip install --isolated build==1.5.0 + python -m build + + - name: Check artifact metadata + timeout-minutes: 5 + run: | + python -m pip install --isolated twine==7.0.0 + python -m twine check --strict dist/* + + - name: Save built distributions for the publish job + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: python-dist + path: python/dist/ + if-no-files-found: error # an empty artifact silently breaks the publish job + + publish: + name: publish + needs: build + if: startsWith(github.ref_name, 'python-v') + runs-on: ubuntu-latest + timeout-minutes: 10 + # Must match the environment name on the trusted publisher; carries the + # required-reviewer gate so a run alone cannot publish. + environment: pypi + permissions: + # Job permsissions REPLACE the top-level block, so restate contents: read. + contents: read + id-token: write # OIDC token for PyPI auth + PEP 740 attestation signing + steps: + - name: Retrieve built distributions from the build job + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: python-dist + path: dist/ + + - name: Publish to PyPI with attestations + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + attestations: true + + # Prove the uploaded artifact installs and runs from PyPI. Retry: the index + # can lag the upload. + - name: Smoke-test the published package + run: | + set -euo pipefail + version="$(python3 -c "import pathlib; print(next(pathlib.Path('dist').glob('*.whl')).name.split('-')[1])")" + python3 -m venv /tmp/smoke + for attempt in $(seq 1 5); do + if /tmp/smoke/bin/pip install "developer-toolkit-elasticache==${version}"; then + break + fi + echo "install attempt ${attempt} failed (index may lag the upload); retrying in 15s" + sleep 15 + done + /tmp/smoke/bin/developer-toolkit-elasticache --help + /tmp/smoke/bin/python -c "import developer_toolkit_elasticache" From ed972960fba51eea6d71958c53c3ff55f3d36519 Mon Sep 17 00:00:00 2001 From: mwaisaida-13 Date: Thu, 20 Aug 2026 11:24:09 +0100 Subject: [PATCH 2/4] ci/cd: extract build scripts from YAML, centralize Python versions Address review feedback: - Extract all shell logic from workflow YAML into python/scripts/ (build.sh, lint.sh, test.sh, verify_release.sh, smoke_test.sh, load_python_versions.sh). Zero embedded shell in workflows. - Centralize CI Python version matrix in .github/python-versions.json, shared by both workflows via load_python_versions.sh. - Pin build tools (build==1.5.0, twine==7.0.0) in a dedicated python/requirements/build.txt tracked by Dependabot. - Update dependabot.yml with pip ecosystem entry for build tools. - Use actions/setup-python directly (SHA-pinned) in all jobs. --- .github/dependabot.yml | 12 ++++- .github/python-versions.json | 1 + .github/workflows/python-release.yml | 74 ++++++++++---------------- .github/workflows/python.yml | 67 +++++++++++++---------- python/requirements/build.txt | 4 ++ python/scripts/build.sh | 8 +++ python/scripts/lint.sh | 7 +++ python/scripts/load_python_versions.sh | 6 +++ python/scripts/smoke_test.sh | 25 +++++++++ python/scripts/test.sh | 6 +++ python/scripts/verify_release.sh | 19 +++++++ 11 files changed, 155 insertions(+), 74 deletions(-) create mode 100644 .github/python-versions.json create mode 100644 python/requirements/build.txt create mode 100755 python/scripts/build.sh create mode 100755 python/scripts/lint.sh create mode 100755 python/scripts/load_python_versions.sh create mode 100755 python/scripts/smoke_test.sh create mode 100755 python/scripts/test.sh create mode 100755 python/scripts/verify_release.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e6fee30..91ba83c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,7 +12,7 @@ updates: actions: patterns: ["*"] - # Python deps in python/pyproject.toml. + # Python runtime/dev deps in python/pyproject.toml. - package-ecosystem: pip directory: /python schedule: @@ -20,3 +20,13 @@ updates: groups: python: patterns: ["*"] + + # Pinned build tooling (build, twine) in python/requirements/build.txt. + # Dependabot discovers *.txt files in this directory. + - package-ecosystem: pip + directory: /python/requirements + schedule: + interval: weekly + groups: + build-tools: + patterns: ["*"] diff --git a/.github/python-versions.json b/.github/python-versions.json new file mode 100644 index 0000000..17d71fa --- /dev/null +++ b/.github/python-versions.json @@ -0,0 +1 @@ +["3.10", "3.11", "3.12", "3.13", "3.14"] diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 7538e44..4592bcf 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -16,18 +16,32 @@ concurrency: cancel-in-progress: false jobs: + # Load supported Python versions from the shared JSON file. + versions: + name: load python versions + if: startsWith(github.ref_name, 'python-v') + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set.outputs.matrix }} + steps: + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - id: set + run: bash python/scripts/load_python_versions.sh + # Full support matrix gates the release; build/publish depend on it. # Re-runs the ubuntu CI tests here as one of the release gates. # python-v* tags only test: name: test (py${{ matrix.python-version }}) - if: startsWith(github.ref_name, 'python-v') + needs: versions runs-on: ubuntu-latest timeout-minutes: 15 strategy: fail-fast: false # report every version, not just the first failure matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ${{ fromJson(needs.versions.outputs.matrix) }} defaults: run: working-directory: python @@ -38,13 +52,11 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: "${{ matrix.python-version }}" + python-version: ${{ matrix.python-version }} - name: Install and run the test suite timeout-minutes: 10 - run: | - python -m pip install -e ".[dev]" - python -m pytest -q + run: bash scripts/test.sh build: name: build and verify @@ -64,34 +76,14 @@ jobs: with: python-version: "3.12" - # Tag must match pyproject.toml version, and the `Private :: Do Not Upload` - # guard must be gone — fail the release loudly otherwise. + # Tag must match pyproject.toml version, and the publish guard `Private :: Do Not Upload` + # must be gone. - name: Verify the tag matches the packaged version and the package is publishable - run: | - set -euo pipefail - tag_version="${GITHUB_REF_NAME#python-v}" - pkg_version="$(python -c 'import tomllib, pathlib; print(tomllib.loads(pathlib.Path("pyproject.toml").read_text())["project"]["version"])')" - if [ "$tag_version" != "$pkg_version" ]; then - echo "::error::Release tag is python-v${tag_version} but pyproject.toml declares ${pkg_version}. Bump the version and re-tag." - exit 1 - fi - if grep -q "Private :: Do Not Upload" pyproject.toml; then - echo "::error::The 'Private :: Do Not Upload' classifier is still set. Remove it in the release commit once the open-source review has cleared." - exit 1 - fi - echo "Publishing version ${pkg_version}." - - - name: Build wheel and sdist - timeout-minutes: 5 - run: | - python -m pip install --isolated build==1.5.0 - python -m build + run: bash scripts/verify_release.sh - - name: Check artifact metadata + - name: Build wheel and sdist, check metadata timeout-minutes: 5 - run: | - python -m pip install --isolated twine==7.0.0 - python -m twine check --strict dist/* + run: bash scripts/build.sh - name: Save built distributions for the publish job uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -110,10 +102,14 @@ jobs: # required-reviewer gate so a run alone cannot publish. environment: pypi permissions: - # Job permsissions REPLACE the top-level block, so restate contents: read. + # Job permissions REPLACE the top-level block, so restate contents: read. contents: read id-token: write # OIDC token for PyPI auth + PEP 740 attestation signing steps: + # Checkout so the smoke-test script is on the runner. + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Retrieve built distributions from the build job uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -128,16 +124,4 @@ jobs: # Prove the uploaded artifact installs and runs from PyPI. Retry: the index # can lag the upload. - name: Smoke-test the published package - run: | - set -euo pipefail - version="$(python3 -c "import pathlib; print(next(pathlib.Path('dist').glob('*.whl')).name.split('-')[1])")" - python3 -m venv /tmp/smoke - for attempt in $(seq 1 5); do - if /tmp/smoke/bin/pip install "developer-toolkit-elasticache==${version}"; then - break - fi - echo "install attempt ${attempt} failed (index may lag the upload); retrying in 15s" - sleep 15 - done - /tmp/smoke/bin/developer-toolkit-elasticache --help - /tmp/smoke/bin/python -c "import developer_toolkit_elasticache" + run: bash python/scripts/smoke_test.sh diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 9a4e3d9..c6f3834 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -8,10 +8,12 @@ on: paths: - "python/**" - ".github/workflows/python.yml" + - ".github/python-versions.json" pull_request: paths: - "python/**" - ".github/workflows/python.yml" + - ".github/python-versions.json" defaults: run: @@ -22,52 +24,61 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@v5 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - - name: Install package with dev dependencies - run: python -m pip install -e ".[dev]" + - name: Lint and format check + run: bash scripts/lint.sh - # Lint and format are interpreter-independent, so they run once rather - # than across the whole test matrix. - - name: Ruff lint - run: python -m ruff check . + # Load supported Python versions from the shared JSON file. + versions: + name: load python versions + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set.outputs.matrix }} + steps: + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Ruff format - run: python -m ruff format --check . + - id: set + working-directory: ${{ github.workspace }} + run: bash python/scripts/load_python_versions.sh test: name: py${{ matrix.python-version }} on ${{ matrix.os }} + needs: versions runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - # The floor and the ceiling on each OS, plus Apple Silicon. Every version - # in requires-python is covered on Linux; the other platforms test the - # ends of the range, which is where interpreter differences show up. - os: [ubuntu-latest, windows-latest, macos-14] - python-version: ["3.10", "3.14"] + # Every supported version runs on Linux; + # the other platforms test the ends of the range, which is where + # interpreter differences show up. + os: [ubuntu-latest] + python-version: ${{ fromJson(needs.versions.outputs.matrix) }} include: - - os: ubuntu-latest - python-version: "3.11" - - os: ubuntu-latest - python-version: "3.12" - - os: ubuntu-latest - python-version: "3.13" + - os: windows-latest + python-version: "3.10" + - os: windows-latest + python-version: "3.14" + - os: macos-14 + python-version: "3.10" + - os: macos-14 + python-version: "3.14" steps: - - uses: actions/checkout@v4 + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} - - name: Install package with dev dependencies - run: python -m pip install -e ".[dev]" - - - name: Run tests - run: python -m pytest -q + - name: Install and run tests + run: bash scripts/test.sh diff --git a/python/requirements/build.txt b/python/requirements/build.txt new file mode 100644 index 0000000..28e5036 --- /dev/null +++ b/python/requirements/build.txt @@ -0,0 +1,4 @@ +# Build and packaging tools pinned for reproducible releases. +# Dependabot keeps these current via .github/dependabot.yml. +build==1.5.0 +twine==7.0.0 diff --git a/python/scripts/build.sh b/python/scripts/build.sh new file mode 100755 index 0000000..75e8464 --- /dev/null +++ b/python/scripts/build.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Build the wheel and sdist, then validate metadata with twine. +# Called from the release workflow's build job. +set -euo pipefail + +python3 -m pip install --isolated -r requirements/build.txt +python3 -m build +python3 -m twine check --strict dist/* diff --git a/python/scripts/lint.sh b/python/scripts/lint.sh new file mode 100755 index 0000000..5f22751 --- /dev/null +++ b/python/scripts/lint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Install dependencies and run linting checks. +set -euo pipefail + +python3 -m pip install -e ".[dev]" +python3 -m ruff check . +python3 -m ruff format --check . diff --git a/python/scripts/load_python_versions.sh b/python/scripts/load_python_versions.sh new file mode 100755 index 0000000..537f464 --- /dev/null +++ b/python/scripts/load_python_versions.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Load supported Python versions from the shared JSON file and export as a +# GitHub Actions output for use in downstream matrix jobs. +set -euo pipefail + +echo "matrix=$(cat .github/python-versions.json)" >> "$GITHUB_OUTPUT" diff --git a/python/scripts/smoke_test.sh b/python/scripts/smoke_test.sh new file mode 100755 index 0000000..48423dd --- /dev/null +++ b/python/scripts/smoke_test.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Smoke-test the published package from PyPI. +# Retries because the index can lag the upload. +set -euo pipefail + +version="$(python3 -c "import pathlib; print(next(pathlib.Path('dist').glob('*.whl')).name.split('-')[1])")" +python3 -m venv /tmp/smoke + +installed=false +for attempt in $(seq 1 5); do + if /tmp/smoke/bin/pip install "developer-toolkit-elasticache==${version}"; then + installed=true + break + fi + echo "Install attempt ${attempt} failed (index may lag the upload); retrying in 15s" + sleep 15 +done + +if [ "$installed" = false ]; then + echo "::error::Failed to install developer-toolkit-elasticache==${version} from PyPI after 5 attempts." + exit 1 +fi + +/tmp/smoke/bin/developer-toolkit-elasticache --help +/tmp/smoke/bin/python -c "import developer_toolkit_elasticache" diff --git a/python/scripts/test.sh b/python/scripts/test.sh new file mode 100755 index 0000000..95bbe80 --- /dev/null +++ b/python/scripts/test.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Install the package with dev dependencies and run the test suite. +set -euo pipefail + +python3 -m pip install -e ".[dev]" +python3 -m pytest -q diff --git a/python/scripts/verify_release.sh b/python/scripts/verify_release.sh new file mode 100755 index 0000000..d0c14f7 --- /dev/null +++ b/python/scripts/verify_release.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Verify the release tag matches pyproject.toml and the package is publishable. +# Called from the release workflow's build job. +set -euo pipefail + +tag_version="${GITHUB_REF_NAME#python-v}" +pkg_version="$(python3 -c 'import tomllib, pathlib; print(tomllib.loads(pathlib.Path("pyproject.toml").read_text())["project"]["version"])')" + +if [ "$tag_version" != "$pkg_version" ]; then + echo "::error::Release tag is python-v${tag_version} but pyproject.toml declares ${pkg_version}. Bump the version and re-tag." + exit 1 +fi + +if grep -q "Private :: Do Not Upload" pyproject.toml; then + echo "::error::The 'Private :: Do Not Upload' classifier is still set. Remove it in the release commit once the open-source review has cleared." + exit 1 +fi + +echo "Publishing version ${pkg_version}." From fe3600054c52579dd2d8001bda82923756138a6a Mon Sep 17 00:00:00 2001 From: mwaisaida-13 Date: Fri, 21 Aug 2026 17:18:08 +0100 Subject: [PATCH 3/4] ci/cd: replace shell scripts with a dev.py task runner Supersede the shell-script approach with a single Python task runner that runs the same locally and in the workflows. - python/dev.py: one entry point for lint/test/build/verify/versions/smoke/ci. It creates and reuses a local .venv and shells out to the pinned tools, so contributors run `./dev.py ci` with no manual venv setup. - Supported Python versions live once in [tool.python-support] (pyproject); `dev.py versions` generates the range both workflow matrices consume. Removes .github/python-versions.json. - Dev/build/publish tooling pinned in python/requirements/dev.txt (ruff, pytest, build, twine), tracked by Dependabot. Removes requirements/build.txt. - Workflow steps are now one-line `python python/dev.py ` calls (no embedded shell); verify checks tag==version and the Do-Not-Upload guard; smoke asserts the installed version. --- .github/dependabot.yml | 6 +- .github/python-versions.json | 1 - .github/workflows/python-release.yml | 42 ++--- .github/workflows/python.yml | 37 +++-- python/dev.py | 212 +++++++++++++++++++++++++ python/pyproject.toml | 14 +- python/requirements/build.txt | 4 - python/requirements/dev.txt | 13 ++ python/scripts/build.sh | 8 - python/scripts/lint.sh | 7 - python/scripts/load_python_versions.sh | 6 - python/scripts/smoke_test.sh | 25 --- python/scripts/test.sh | 6 - python/scripts/verify_release.sh | 19 --- 14 files changed, 278 insertions(+), 122 deletions(-) delete mode 100644 .github/python-versions.json create mode 100755 python/dev.py delete mode 100644 python/requirements/build.txt create mode 100644 python/requirements/dev.txt delete mode 100755 python/scripts/build.sh delete mode 100755 python/scripts/lint.sh delete mode 100755 python/scripts/load_python_versions.sh delete mode 100755 python/scripts/smoke_test.sh delete mode 100755 python/scripts/test.sh delete mode 100755 python/scripts/verify_release.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 91ba83c..c181f1d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,12 +21,12 @@ updates: python: patterns: ["*"] - # Pinned build tooling (build, twine) in python/requirements/build.txt. - # Dependabot discovers *.txt files in this directory. + # Pinned dev/build/publish tooling in python/requirements/dev.txt (ruff, pytest, + # build, twine). Dependabot discovers the *.txt files in this directory. - package-ecosystem: pip directory: /python/requirements schedule: interval: weekly groups: - build-tools: + tooling: patterns: ["*"] diff --git a/.github/python-versions.json b/.github/python-versions.json deleted file mode 100644 index 17d71fa..0000000 --- a/.github/python-versions.json +++ /dev/null @@ -1 +0,0 @@ -["3.10", "3.11", "3.12", "3.13", "3.14"] diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 4592bcf..d6e55aa 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -16,23 +16,30 @@ concurrency: cancel-in-progress: false jobs: - # Load supported Python versions from the shared JSON file. + # Source of truth for the test matrix: generated from [tool.python-support]. + # python-v* tags only — test/build/publish inherit this via `needs`. versions: - name: load python versions + name: python versions if: startsWith(github.ref_name, 'python-v') runs-on: ubuntu-latest + timeout-minutes: 5 outputs: - matrix: ${{ steps.set.outputs.matrix }} + data: ${{ steps.set.outputs.data }} steps: - name: Check out the repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - id: set - run: bash python/scripts/load_python_versions.sh + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Generate supported versions from pyproject.toml + id: set + run: echo "data=$(python python/dev.py versions)" >> "$GITHUB_OUTPUT" # Full support matrix gates the release; build/publish depend on it. # Re-runs the ubuntu CI tests here as one of the release gates. - # python-v* tags only test: name: test (py${{ matrix.python-version }}) needs: versions @@ -41,10 +48,7 @@ jobs: strategy: fail-fast: false # report every version, not just the first failure matrix: - python-version: ${{ fromJson(needs.versions.outputs.matrix) }} - defaults: - run: - working-directory: python + python-version: ${{ fromJson(needs.versions.outputs.data).all }} steps: - name: Check out the repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -56,7 +60,7 @@ jobs: - name: Install and run the test suite timeout-minutes: 10 - run: bash scripts/test.sh + run: python python/dev.py test build: name: build and verify @@ -64,9 +68,6 @@ jobs: if: startsWith(github.ref_name, 'python-v') runs-on: ubuntu-latest timeout-minutes: 15 - defaults: - run: - working-directory: python steps: - name: Check out the repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -79,11 +80,11 @@ jobs: # Tag must match pyproject.toml version, and the publish guard `Private :: Do Not Upload` # must be gone. - name: Verify the tag matches the packaged version and the package is publishable - run: bash scripts/verify_release.sh + run: python python/dev.py verify "$GITHUB_REF_NAME" - name: Build wheel and sdist, check metadata timeout-minutes: 5 - run: bash scripts/build.sh + run: python python/dev.py build - name: Save built distributions for the publish job uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -110,6 +111,11 @@ jobs: - name: Check out the repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Retrieve built distributions from the build job uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -121,7 +127,5 @@ jobs: with: attestations: true - # Prove the uploaded artifact installs and runs from PyPI. Retry: the index - # can lag the upload. - name: Smoke-test the published package - run: bash python/scripts/smoke_test.sh + run: python python/dev.py smoke "${GITHUB_REF_NAME#python-v}" diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index c6f3834..f88df24 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -8,16 +8,10 @@ on: paths: - "python/**" - ".github/workflows/python.yml" - - ".github/python-versions.json" pull_request: paths: - "python/**" - ".github/workflows/python.yml" - - ".github/python-versions.json" - -defaults: - run: - working-directory: python jobs: lint: @@ -33,21 +27,26 @@ jobs: python-version: "3.12" - name: Lint and format check - run: bash scripts/lint.sh + run: python python/dev.py lint - # Load supported Python versions from the shared JSON file. + # Load Python versions generated from [tool.python-support]. versions: - name: load python versions + name: python versions runs-on: ubuntu-latest outputs: - matrix: ${{ steps.set.outputs.matrix }} + data: ${{ steps.set.outputs.data }} steps: - name: Check out the repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - id: set - working-directory: ${{ github.workspace }} - run: bash python/scripts/load_python_versions.sh + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Generate supported versions from pyproject.toml + id: set + run: echo "data=$(python python/dev.py versions)" >> "$GITHUB_OUTPUT" test: name: py${{ matrix.python-version }} on ${{ matrix.os }} @@ -60,16 +59,16 @@ jobs: # the other platforms test the ends of the range, which is where # interpreter differences show up. os: [ubuntu-latest] - python-version: ${{ fromJson(needs.versions.outputs.matrix) }} + python-version: ${{ fromJson(needs.versions.outputs.data).all }} include: - os: windows-latest - python-version: "3.10" + python-version: "${{ fromJson(needs.versions.outputs.data).min }}" - os: windows-latest - python-version: "3.14" + python-version: "${{ fromJson(needs.versions.outputs.data).max }}" - os: macos-14 - python-version: "3.10" + python-version: "${{ fromJson(needs.versions.outputs.data).min }}" - os: macos-14 - python-version: "3.14" + python-version: "${{ fromJson(needs.versions.outputs.data).max }}" steps: - name: Check out the repository @@ -81,4 +80,4 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install and run tests - run: bash scripts/test.sh + run: python python/dev.py test diff --git a/python/dev.py b/python/dev.py new file mode 100755 index 0000000..e72e475 --- /dev/null +++ b/python/dev.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Developer/CI task runner — one entry point for every CI/CD step, run locally or +from the workflows. Requires Python 3.11+ (tomllib) for `verify` and `versions` + + ./dev.py lint # ruff check + format --check + ./dev.py test # editable install + pytest + ./dev.py build # build wheel + sdist, twine check --strict + ./dev.py verify [TAG] # tag == pyproject version + publish guard + ./dev.py versions # print the supported-version matrix as JSON + ./dev.py smoke [VERSION] # install from PyPI + assert version + run CLI + ./dev.py ci # lint + test + build (the local pre-flight) + +lint/test/build run inside a .venv the runner creates and reuses, so there is no +manual `python -m venv` step; delete .venv to rebuild it against a different +interpreter. +""" + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import time +import venv +from pathlib import Path + +HERE = Path(__file__).resolve().parent +VENV = HERE / ".venv" +PYPROJECT = HERE / "pyproject.toml" +PACKAGE = "developer-toolkit-elasticache" +IMPORT_NAME = "developer_toolkit_elasticache" +DO_NOT_UPLOAD = "Private :: Do Not Upload" + + +_COMMANDS = { + "lint": None, + "test": None, + "build": None, + "versions": None, + "ci": None, + "verify": ("tag", "release tag, e.g. python-v1.2.3"), + "smoke": ("version", "published version, e.g. 1.2.3"), +} + + +def run(*args: object) -> None: + print(f"+ {' '.join(map(str, args))}") + subprocess.run([str(a) for a in args], check=True, cwd=HERE) + + +def venv_python() -> str: + """Create the managed .venv on first use and return its Python interpreter. + + Lets the runner own an isolated environment, so `./dev.py test` works with no + manual `python -m venv`. In CI each job is a fresh runner, so the .venv is built + from that job's interpreter — honouring the matrix version. + """ + if not VENV.exists(): + print(f"+ creating virtualenv at {VENV}") + venv.create(VENV, with_pip=True) + bindir = VENV / ("Scripts" if os.name == "nt" else "bin") + return str(bindir / "python") + + +def _project() -> dict: + import tomllib # 3.11+; imported lazily so lint/test/build work on 3.10 + + return tomllib.loads(PYPROJECT.read_text())["project"] + + +def _support() -> dict: + import tomllib + + return tomllib.loads(PYPROJECT.read_text())["tool"]["python-support"] + + +def lint(_args: argparse.Namespace) -> int: + py = venv_python() + run(py, "-m", "pip", "install", "-q", "-r", "requirements/dev.txt") + run(py, "-m", "ruff", "check", ".") + run(py, "-m", "ruff", "format", "--check", ".") + return 0 + + +def test(_args: argparse.Namespace) -> int: + py = venv_python() + run(py, "-m", "pip", "install", "-q", "-e", ".", "-r", "requirements/dev.txt") + run(py, "-m", "pytest", "-q") + return 0 + + +def build(_args: argparse.Namespace) -> int: + py = venv_python() + run(py, "-m", "pip", "install", "-q", "-r", "requirements/dev.txt") + run(py, "-m", "build") + dist = sorted(str(p) for p in (HERE / "dist").glob("*")) + if not dist: + raise SystemExit("build produced no artifacts in dist/") + run(py, "-m", "twine", "check", "--strict", *dist) + return 0 + + +def verify(args: argparse.Namespace) -> int: + tag = args.tag or os.environ.get("GITHUB_REF_NAME", "") + tag_version = tag.removeprefix("python-v") + project = _project() + pkg_version = project["version"] + if tag_version != pkg_version: + print( + "::error::Release tag does not match the version in pyproject.toml " + f"({pkg_version}). Bump the version and re-tag." + ) + return 1 + if DO_NOT_UPLOAD in project.get("classifiers", []): + print( + f"::error::The '{DO_NOT_UPLOAD}' classifier is still set. Remove it in the " + "release commit once the open-source review has cleared." + ) + return 1 + print(f"Publishing version {pkg_version}.") + return 0 + + +def versions(_args: argparse.Namespace) -> int: + support = _support() + low, high = int(support["min"].split(".")[1]), int(support["max"].split(".")[1]) + vers = [f"3.{minor}" for minor in range(low, high + 1)] + print(json.dumps({"all": vers, "min": vers[0], "max": vers[-1]})) + return 0 + + +def smoke(args: argparse.Namespace) -> int: + version = args.version or os.environ.get("GITHUB_REF_NAME", "").removeprefix( + "python-v" + ) + if not version: + wheels = sorted((HERE / "dist").glob("*.whl")) + if not wheels: + print("::error::No version given and no wheel in dist/ to derive it from.") + return 1 + version = wheels[0].name.split("-")[1] + + with tempfile.TemporaryDirectory() as tmp: + env = Path(tmp) / "venv" + venv.create(env, with_pip=True) + bindir = env / ("Scripts" if os.name == "nt" else "bin") + pip, python, cli = ( + str(bindir / "pip"), + str(bindir / "python"), + str(bindir / PACKAGE), + ) + + spec = f"{PACKAGE}=={version}" + for attempt in range(1, 6): + if subprocess.run([pip, "install", spec], check=False).returncode == 0: + break + print( + f"install attempt {attempt} failed (index may lag the upload); retrying in 15s" + ) + time.sleep(15) + else: + print(f"::error::Could not install {PACKAGE} from PyPI after 5 attempts.") + return 1 + + got = subprocess.run( + [ + python, + "-c", + f"import importlib.metadata as m; print(m.version('{PACKAGE}'))", + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if got != version: + print(f"::error::Installed the wrong version of {PACKAGE} ({got}).") + return 1 + + subprocess.run([cli, "--help"], check=True) + subprocess.run([python, "-c", f"import {IMPORT_NAME}"], check=True) + + print(f"Smoke test passed for {PACKAGE}.") + return 0 + + +def ci(args: argparse.Namespace) -> int: + lint(args) + test(args) + build(args) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + prog="dev.py", description="Developer/CI task runner." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + for name, arg in _COMMANDS.items(): + subparser = subparsers.add_parser(name) + if arg is not None: + subparser.add_argument(arg[0], nargs="?", help=arg[1]) + subparser.set_defaults(func=globals()[name]) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/pyproject.toml b/python/pyproject.toml index 033208f..0c65b03 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -33,11 +33,10 @@ classifiers = [ "Programming Language :: Python :: 3.14", ] -[project.optional-dependencies] -dev = [ - "pytest>=7.0", - "ruff>=0.6", -] +# Supported Python range — the source for the CI/release test matrices. +[tool.python-support] +min = "3.10" +max = "3.14" [tool.ruff] # The floor in requires-python. Keeps ruff from suggesting syntax that would not @@ -72,6 +71,11 @@ extend-select = [ # The examples are illustrative scripts whose job is to print, and they are not an # importable package. "examples/**/*" = ["T201", "INP001"] +# dev.py is the task runner (not shipped code), so exempt the rules that fight its job: +# T201 — it prints command output and ::error:: annotations +# S603 — it shells out to the build tools (subprocess) with fixed, trusted args +# PLC0415 — it imports tomllib lazily so lint/test/build still run on Python 3.10 +"dev.py" = ["T201", "S603", "PLC0415"] [tool.pytest.ini_options] # The examples/ scripts are runnable demos that need live AWS credentials and a diff --git a/python/requirements/build.txt b/python/requirements/build.txt deleted file mode 100644 index 28e5036..0000000 --- a/python/requirements/build.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Build and packaging tools pinned for reproducible releases. -# Dependabot keeps these current via .github/dependabot.yml. -build==1.5.0 -twine==7.0.0 diff --git a/python/requirements/dev.txt b/python/requirements/dev.txt new file mode 100644 index 0000000..812e21e --- /dev/null +++ b/python/requirements/dev.txt @@ -0,0 +1,13 @@ +# Dev/CI tooling, pinned so CI is reproducible and a new release of any of these +# can't silently break an unrelated PR. Dependabot bumps them (pip entry in +# dependabot.yml). Installed by dev.py (lint/test/build). + +# build +build==1.5.0 + +# dev +ruff==0.16.3 +pytest==9.1.1 + +# publish +twine==7.0.0 diff --git a/python/scripts/build.sh b/python/scripts/build.sh deleted file mode 100755 index 75e8464..0000000 --- a/python/scripts/build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -# Build the wheel and sdist, then validate metadata with twine. -# Called from the release workflow's build job. -set -euo pipefail - -python3 -m pip install --isolated -r requirements/build.txt -python3 -m build -python3 -m twine check --strict dist/* diff --git a/python/scripts/lint.sh b/python/scripts/lint.sh deleted file mode 100755 index 5f22751..0000000 --- a/python/scripts/lint.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -# Install dependencies and run linting checks. -set -euo pipefail - -python3 -m pip install -e ".[dev]" -python3 -m ruff check . -python3 -m ruff format --check . diff --git a/python/scripts/load_python_versions.sh b/python/scripts/load_python_versions.sh deleted file mode 100755 index 537f464..0000000 --- a/python/scripts/load_python_versions.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -# Load supported Python versions from the shared JSON file and export as a -# GitHub Actions output for use in downstream matrix jobs. -set -euo pipefail - -echo "matrix=$(cat .github/python-versions.json)" >> "$GITHUB_OUTPUT" diff --git a/python/scripts/smoke_test.sh b/python/scripts/smoke_test.sh deleted file mode 100755 index 48423dd..0000000 --- a/python/scripts/smoke_test.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Smoke-test the published package from PyPI. -# Retries because the index can lag the upload. -set -euo pipefail - -version="$(python3 -c "import pathlib; print(next(pathlib.Path('dist').glob('*.whl')).name.split('-')[1])")" -python3 -m venv /tmp/smoke - -installed=false -for attempt in $(seq 1 5); do - if /tmp/smoke/bin/pip install "developer-toolkit-elasticache==${version}"; then - installed=true - break - fi - echo "Install attempt ${attempt} failed (index may lag the upload); retrying in 15s" - sleep 15 -done - -if [ "$installed" = false ]; then - echo "::error::Failed to install developer-toolkit-elasticache==${version} from PyPI after 5 attempts." - exit 1 -fi - -/tmp/smoke/bin/developer-toolkit-elasticache --help -/tmp/smoke/bin/python -c "import developer_toolkit_elasticache" diff --git a/python/scripts/test.sh b/python/scripts/test.sh deleted file mode 100755 index 95bbe80..0000000 --- a/python/scripts/test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -# Install the package with dev dependencies and run the test suite. -set -euo pipefail - -python3 -m pip install -e ".[dev]" -python3 -m pytest -q diff --git a/python/scripts/verify_release.sh b/python/scripts/verify_release.sh deleted file mode 100755 index d0c14f7..0000000 --- a/python/scripts/verify_release.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# Verify the release tag matches pyproject.toml and the package is publishable. -# Called from the release workflow's build job. -set -euo pipefail - -tag_version="${GITHUB_REF_NAME#python-v}" -pkg_version="$(python3 -c 'import tomllib, pathlib; print(tomllib.loads(pathlib.Path("pyproject.toml").read_text())["project"]["version"])')" - -if [ "$tag_version" != "$pkg_version" ]; then - echo "::error::Release tag is python-v${tag_version} but pyproject.toml declares ${pkg_version}. Bump the version and re-tag." - exit 1 -fi - -if grep -q "Private :: Do Not Upload" pyproject.toml; then - echo "::error::The 'Private :: Do Not Upload' classifier is still set. Remove it in the release commit once the open-source review has cleared." - exit 1 -fi - -echo "Publishing version ${pkg_version}." From c1c8fa0061946583c54e48ab3ba84052bf2e7fdf Mon Sep 17 00:00:00 2001 From: mwaisaida-13 Date: Tue, 25 Aug 2026 18:02:58 +0100 Subject: [PATCH 4/4] ci/cd: move task runner to scripts/python_dev.py; add setup command - Rename python/dev.py -> scripts/python_dev.py: a repo-level runner decoupled from the Python package (not shipped in the wheel or sdist). - Add `setup` command that installs the package + pinned dev tooling; every task self-installs on run (pip is a no-op when satisfied), matching GLIDE's pattern. - Rename commands: `versions` -> `list-supported-versions`, `build` -> `package`; fold the [tool.python-support] drift check into `verify`. - Pin pip in requirements/dev.txt; drop the now-unneeded dev.py lint ignore (the runner no longer lives under python/). - Workflows call `python scripts/python_dev.py `; CI path filter adds scripts/**. --- .github/workflows/python-release.yml | 14 ++-- .github/workflows/python.yml | 8 +- python/pyproject.toml | 5 -- python/requirements/dev.txt | 6 +- python/dev.py => scripts/python_dev.py | 103 ++++++++++++++++++------- 5 files changed, 91 insertions(+), 45 deletions(-) rename python/dev.py => scripts/python_dev.py (61%) diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index d6e55aa..2f0e1a0 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -36,10 +36,10 @@ jobs: - name: Generate supported versions from pyproject.toml id: set - run: echo "data=$(python python/dev.py versions)" >> "$GITHUB_OUTPUT" + run: echo "data=$(python scripts/python_dev.py list-supported-versions)" >> "$GITHUB_OUTPUT" - # Full support matrix gates the release; build/publish depend on it. - # Re-runs the ubuntu CI tests here as one of the release gates. + # Full support matrix gates the release; build/publish depend on it. Sanity check with + # Ubuntu-only tests since package is pure-Python. test: name: test (py${{ matrix.python-version }}) needs: versions @@ -60,7 +60,7 @@ jobs: - name: Install and run the test suite timeout-minutes: 10 - run: python python/dev.py test + run: python scripts/python_dev.py test build: name: build and verify @@ -80,11 +80,11 @@ jobs: # Tag must match pyproject.toml version, and the publish guard `Private :: Do Not Upload` # must be gone. - name: Verify the tag matches the packaged version and the package is publishable - run: python python/dev.py verify "$GITHUB_REF_NAME" + run: python scripts/python_dev.py verify "$GITHUB_REF_NAME" - name: Build wheel and sdist, check metadata timeout-minutes: 5 - run: python python/dev.py build + run: python scripts/python_dev.py package - name: Save built distributions for the publish job uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -128,4 +128,4 @@ jobs: attestations: true - name: Smoke-test the published package - run: python python/dev.py smoke "${GITHUB_REF_NAME#python-v}" + run: python scripts/python_dev.py smoke "${GITHUB_REF_NAME#python-v}" diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index f88df24..7f94871 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -7,10 +7,12 @@ on: branches: [main] paths: - "python/**" + - "scripts/**" - ".github/workflows/python.yml" pull_request: paths: - "python/**" + - "scripts/**" - ".github/workflows/python.yml" jobs: @@ -27,7 +29,7 @@ jobs: python-version: "3.12" - name: Lint and format check - run: python python/dev.py lint + run: python scripts/python_dev.py lint # Load Python versions generated from [tool.python-support]. versions: @@ -46,7 +48,7 @@ jobs: - name: Generate supported versions from pyproject.toml id: set - run: echo "data=$(python python/dev.py versions)" >> "$GITHUB_OUTPUT" + run: echo "data=$(python scripts/python_dev.py list-supported-versions)" >> "$GITHUB_OUTPUT" test: name: py${{ matrix.python-version }} on ${{ matrix.os }} @@ -80,4 +82,4 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install and run tests - run: python python/dev.py test + run: python scripts/python_dev.py test diff --git a/python/pyproject.toml b/python/pyproject.toml index 0c65b03..14a8251 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -71,11 +71,6 @@ extend-select = [ # The examples are illustrative scripts whose job is to print, and they are not an # importable package. "examples/**/*" = ["T201", "INP001"] -# dev.py is the task runner (not shipped code), so exempt the rules that fight its job: -# T201 — it prints command output and ::error:: annotations -# S603 — it shells out to the build tools (subprocess) with fixed, trusted args -# PLC0415 — it imports tomllib lazily so lint/test/build still run on Python 3.10 -"dev.py" = ["T201", "S603", "PLC0415"] [tool.pytest.ini_options] # The examples/ scripts are runnable demos that need live AWS credentials and a diff --git a/python/requirements/dev.txt b/python/requirements/dev.txt index 812e21e..d36eb0b 100644 --- a/python/requirements/dev.txt +++ b/python/requirements/dev.txt @@ -1,6 +1,10 @@ # Dev/CI tooling, pinned so CI is reproducible and a new release of any of these # can't silently break an unrelated PR. Dependabot bumps them (pip entry in -# dependabot.yml). Installed by dev.py (lint/test/build). +# dependabot.yml). Installed by scripts/python_dev.py setup. +# Requires Python 3.11+ + +# install +pip==26.2.1 # build build==1.5.0 diff --git a/python/dev.py b/scripts/python_dev.py similarity index 61% rename from python/dev.py rename to scripts/python_dev.py index e72e475..86ccb35 100755 --- a/python/dev.py +++ b/scripts/python_dev.py @@ -1,16 +1,17 @@ #!/usr/bin/env python3 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Developer/CI task runner — one entry point for every CI/CD step, run locally or +"""Python task runner — one entry point for every Python CI/CD step, run locally or from the workflows. Requires Python 3.11+ (tomllib) for `verify` and `versions` - ./dev.py lint # ruff check + format --check - ./dev.py test # editable install + pytest - ./dev.py build # build wheel + sdist, twine check --strict - ./dev.py verify [TAG] # tag == pyproject version + publish guard - ./dev.py versions # print the supported-version matrix as JSON - ./dev.py smoke [VERSION] # install from PyPI + assert version + run CLI - ./dev.py ci # lint + test + build (the local pre-flight) + ./python_dev.py setup # install the package + all dev tooling + ./python_dev.py lint # ruff check + format --check + ./python_dev.py test # pytest + ./python_dev.py package # build wheel + sdist, twine check --strict + ./python_dev.py verify [TAG] # tag == pyproject version + publish guard + drift + ./python_dev.py list-supported-versions # print the supported-version matrix as JSON + ./python_dev.py smoke [VERSION] # install from PyPI + assert version + run CLI + ./python_dev.py ci # setup + lint + test + package (pre-flight) lint/test/build run inside a .venv the runner creates and reuses, so there is no manual `python -m venv` step; delete .venv to rebuild it against a different @@ -20,6 +21,7 @@ import argparse import json import os +import re import subprocess import sys import tempfile @@ -27,19 +29,21 @@ import venv from pathlib import Path -HERE = Path(__file__).resolve().parent -VENV = HERE / ".venv" -PYPROJECT = HERE / "pyproject.toml" +REPO_ROOT = Path(__file__).resolve().parent.parent +PYTHON_DIR = REPO_ROOT / "python" +VENV = PYTHON_DIR / ".venv" +PYPROJECT = PYTHON_DIR / "pyproject.toml" PACKAGE = "developer-toolkit-elasticache" IMPORT_NAME = "developer_toolkit_elasticache" DO_NOT_UPLOAD = "Private :: Do Not Upload" _COMMANDS = { + "setup": None, "lint": None, "test": None, - "build": None, - "versions": None, + "package": None, + "list-supported-versions": None, "ci": None, "verify": ("tag", "release tag, e.g. python-v1.2.3"), "smoke": ("version", "published version, e.g. 1.2.3"), @@ -48,7 +52,7 @@ def run(*args: object) -> None: print(f"+ {' '.join(map(str, args))}") - subprocess.run([str(a) for a in args], check=True, cwd=HERE) + subprocess.run([str(a) for a in args], check=True, cwd=PYTHON_DIR) def venv_python() -> str: @@ -66,7 +70,7 @@ def venv_python() -> str: def _project() -> dict: - import tomllib # 3.11+; imported lazily so lint/test/build work on 3.10 + import tomllib # 3.11+; imported lazily so setup/lint/test still load on 3.10 return tomllib.loads(PYPROJECT.read_text())["project"] @@ -77,32 +81,64 @@ def _support() -> dict: return tomllib.loads(PYPROJECT.read_text())["tool"]["python-support"] -def lint(_args: argparse.Namespace) -> int: +def _ensure_env() -> str: + """Create python/.venv and install the package + pinned dev tooling into it.""" py = venv_python() - run(py, "-m", "pip", "install", "-q", "-r", "requirements/dev.txt") + run(py, "-m", "pip", "install", "-q", "-e", ".", "-r", "requirements/dev.txt") + return py + + +def setup(_args: argparse.Namespace) -> int: + """Explicitly create python/.venv and install the package + pinned dev tooling.""" + _ensure_env() + print("Dev environment ready.") + return 0 + + +def lint(_args: argparse.Namespace) -> int: + py = _ensure_env() run(py, "-m", "ruff", "check", ".") run(py, "-m", "ruff", "format", "--check", ".") return 0 def test(_args: argparse.Namespace) -> int: - py = venv_python() - run(py, "-m", "pip", "install", "-q", "-e", ".", "-r", "requirements/dev.txt") + py = _ensure_env() run(py, "-m", "pytest", "-q") return 0 -def build(_args: argparse.Namespace) -> int: - py = venv_python() - run(py, "-m", "pip", "install", "-q", "-r", "requirements/dev.txt") +def package(_args: argparse.Namespace) -> int: + py = _ensure_env() run(py, "-m", "build") - dist = sorted(str(p) for p in (HERE / "dist").glob("*")) + dist = sorted(str(p) for p in (PYTHON_DIR / "dist").glob("*")) if not dist: raise SystemExit("build produced no artifacts in dist/") run(py, "-m", "twine", "check", "--strict", *dist) return 0 +def _support_drift(project: dict, support: dict) -> str | None: + """Report if [tool.python-support] disagrees with requires-python or the Python + classifiers, else None.""" + expected_requires = f">={support['min']}" + if project.get("requires-python") != expected_requires: + return ( + f"requires-python is {project.get('requires-python')!r} but " + f"[tool.python-support].min implies {expected_requires!r}." + ) + lo, hi = int(support["min"].split(".")[1]), int(support["max"].split(".")[1]) + expected = [f"3.{minor}" for minor in range(lo, hi + 1)] + declared = sorted( + m.group(1) + for c in project.get("classifiers", []) + if (m := re.match(r"Programming Language :: Python :: (3\.\d+)$", c)) + ) + if declared != expected: + return f"Python classifiers {declared} do not match python-support {expected}." + return None + + def verify(args: argparse.Namespace) -> int: tag = args.tag or os.environ.get("GITHUB_REF_NAME", "") tag_version = tag.removeprefix("python-v") @@ -120,14 +156,23 @@ def verify(args: argparse.Namespace) -> int: "release commit once the open-source review has cleared." ) return 1 + drift = _support_drift(project, _support()) + if drift: + print(f"::error::{drift}") + return 1 print(f"Publishing version {pkg_version}.") return 0 -def versions(_args: argparse.Namespace) -> int: +def list_supported_versions(_args: argparse.Namespace) -> int: support = _support() - low, high = int(support["min"].split(".")[1]), int(support["max"].split(".")[1]) - vers = [f"3.{minor}" for minor in range(low, high + 1)] + lo_major, lo_minor = (int(p) for p in support["min"].split(".")[:2]) + hi_major, hi_minor = (int(p) for p in support["max"].split(".")[:2]) + if lo_major != hi_major: + raise SystemExit("[tool.python-support] min and max must share a major version.") + vers = [f"{lo_major}.{minor}" for minor in range(lo_minor, hi_minor + 1)] + if not vers: + raise SystemExit("[tool.python-support] min is greater than max.") print(json.dumps({"all": vers, "min": vers[0], "max": vers[-1]})) return 0 @@ -137,7 +182,7 @@ def smoke(args: argparse.Namespace) -> int: "python-v" ) if not version: - wheels = sorted((HERE / "dist").glob("*.whl")) + wheels = sorted((PYTHON_DIR / "dist").glob("*.whl")) if not wheels: print("::error::No version given and no wheel in dist/ to derive it from.") return 1 @@ -189,7 +234,7 @@ def smoke(args: argparse.Namespace) -> int: def ci(args: argparse.Namespace) -> int: lint(args) test(args) - build(args) + package(args) return 0 @@ -202,7 +247,7 @@ def main() -> int: subparser = subparsers.add_parser(name) if arg is not None: subparser.add_argument(arg[0], nargs="?", help=arg[1]) - subparser.set_defaults(func=globals()[name]) + subparser.set_defaults(func=globals()[name.replace("-", "_")]) args = parser.parse_args() return args.func(args)