diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..c181f1d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,32 @@ +# Keeps pinned dependencies current. Takes effect only on the default branch. +# https://docs.github.com/code-security/dependabot/dependabot-version-updates +version: 2 +updates: + # GitHub Actions: bumps the SHA pins and their `# vX.Y.Z` comments in workflows. + # Grouped into one PR to avoid five separate bumps. + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + actions: + patterns: ["*"] + + # Python runtime/dev deps in python/pyproject.toml. + - package-ecosystem: pip + directory: /python + schedule: + interval: weekly + groups: + python: + patterns: ["*"] + + # Pinned dev/build/publish tooling in python/requirements/dev.txt (ruff, pytest, + # build, twine). Dependabot discovers the *.txt files in this directory. + - package-ecosystem: pip + directory: /python/requirements + schedule: + interval: weekly + groups: + tooling: + patterns: ["*"] diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml new file mode 100644 index 0000000..2f0e1a0 --- /dev/null +++ b/.github/workflows/python-release.yml @@ -0,0 +1,131 @@ +name: Publish Python package to PyPI + +# Trusted publisher on PyPI must reference this exact filename and the `pypi` +# environment, or publishing is rejected. + +on: + release: + types: [published] + +permissions: + contents: read + +# cancel-in-progress: false — cancelling mid-publish can corrupt a release. +concurrency: + group: python-release-${{ github.ref }} + cancel-in-progress: false + +jobs: + # Source of truth for the test matrix: generated from [tool.python-support]. + # python-v* tags only — test/build/publish inherit this via `needs`. + versions: + name: python versions + if: startsWith(github.ref_name, 'python-v') + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + data: ${{ steps.set.outputs.data }} + steps: + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Generate supported versions from pyproject.toml + id: set + run: echo "data=$(python scripts/python_dev.py list-supported-versions)" >> "$GITHUB_OUTPUT" + + # Full support matrix gates the release; build/publish depend on it. Sanity check with + # Ubuntu-only tests since package is pure-Python. + test: + name: test (py${{ matrix.python-version }}) + needs: versions + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false # report every version, not just the first failure + matrix: + python-version: ${{ fromJson(needs.versions.outputs.data).all }} + steps: + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install and run the test suite + timeout-minutes: 10 + run: python scripts/python_dev.py test + + build: + name: build and verify + needs: test + if: startsWith(github.ref_name, 'python-v') + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + # Tag must match pyproject.toml version, and the publish guard `Private :: Do Not Upload` + # must be gone. + - name: Verify the tag matches the packaged version and the package is publishable + run: python scripts/python_dev.py verify "$GITHUB_REF_NAME" + + - name: Build wheel and sdist, check metadata + timeout-minutes: 5 + run: python scripts/python_dev.py package + + - name: Save built distributions for the publish job + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: python-dist + path: python/dist/ + if-no-files-found: error # an empty artifact silently breaks the publish job + + publish: + name: publish + needs: build + if: startsWith(github.ref_name, 'python-v') + runs-on: ubuntu-latest + timeout-minutes: 10 + # Must match the environment name on the trusted publisher; carries the + # required-reviewer gate so a run alone cannot publish. + environment: pypi + permissions: + # Job permissions REPLACE the top-level block, so restate contents: read. + contents: read + id-token: write # OIDC token for PyPI auth + PEP 740 attestation signing + steps: + # Checkout so the smoke-test script is on the runner. + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Retrieve built distributions from the build job + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: python-dist + path: dist/ + + - name: Publish to PyPI with attestations + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + attestations: true + + - name: Smoke-test the published package + run: python scripts/python_dev.py smoke "${GITHUB_REF_NAME#python-v}" diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 9a4e3d9..7f94871 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -7,67 +7,79 @@ on: branches: [main] paths: - "python/**" + - "scripts/**" - ".github/workflows/python.yml" pull_request: paths: - "python/**" + - "scripts/**" - ".github/workflows/python.yml" -defaults: - run: - working-directory: python - jobs: lint: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@v5 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - - name: Install package with dev dependencies - run: python -m pip install -e ".[dev]" + - name: Lint and format check + run: python scripts/python_dev.py lint - # Lint and format are interpreter-independent, so they run once rather - # than across the whole test matrix. - - name: Ruff lint - run: python -m ruff check . + # Load Python versions generated from [tool.python-support]. + versions: + name: python versions + runs-on: ubuntu-latest + outputs: + data: ${{ steps.set.outputs.data }} + steps: + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Ruff format - run: python -m ruff format --check . + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Generate supported versions from pyproject.toml + id: set + run: echo "data=$(python scripts/python_dev.py list-supported-versions)" >> "$GITHUB_OUTPUT" test: name: py${{ matrix.python-version }} on ${{ matrix.os }} + needs: versions runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - # The floor and the ceiling on each OS, plus Apple Silicon. Every version - # in requires-python is covered on Linux; the other platforms test the - # ends of the range, which is where interpreter differences show up. - os: [ubuntu-latest, windows-latest, macos-14] - python-version: ["3.10", "3.14"] + # Every supported version runs on Linux; + # the other platforms test the ends of the range, which is where + # interpreter differences show up. + os: [ubuntu-latest] + python-version: ${{ fromJson(needs.versions.outputs.data).all }} include: - - os: ubuntu-latest - python-version: "3.11" - - os: ubuntu-latest - python-version: "3.12" - - os: ubuntu-latest - python-version: "3.13" + - os: windows-latest + python-version: "${{ fromJson(needs.versions.outputs.data).min }}" + - os: windows-latest + python-version: "${{ fromJson(needs.versions.outputs.data).max }}" + - os: macos-14 + python-version: "${{ fromJson(needs.versions.outputs.data).min }}" + - os: macos-14 + python-version: "${{ fromJson(needs.versions.outputs.data).max }}" steps: - - uses: actions/checkout@v4 + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} - - name: Install package with dev dependencies - run: python -m pip install -e ".[dev]" - - - name: Run tests - run: python -m pytest -q + - name: Install and run tests + run: python scripts/python_dev.py test diff --git a/python/pyproject.toml b/python/pyproject.toml index 033208f..14a8251 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -33,11 +33,10 @@ classifiers = [ "Programming Language :: Python :: 3.14", ] -[project.optional-dependencies] -dev = [ - "pytest>=7.0", - "ruff>=0.6", -] +# Supported Python range — the source for the CI/release test matrices. +[tool.python-support] +min = "3.10" +max = "3.14" [tool.ruff] # The floor in requires-python. Keeps ruff from suggesting syntax that would not diff --git a/python/requirements/dev.txt b/python/requirements/dev.txt new file mode 100644 index 0000000..d36eb0b --- /dev/null +++ b/python/requirements/dev.txt @@ -0,0 +1,17 @@ +# Dev/CI tooling, pinned so CI is reproducible and a new release of any of these +# can't silently break an unrelated PR. Dependabot bumps them (pip entry in +# dependabot.yml). Installed by scripts/python_dev.py setup. +# Requires Python 3.11+ + +# install +pip==26.2.1 + +# build +build==1.5.0 + +# dev +ruff==0.16.3 +pytest==9.1.1 + +# publish +twine==7.0.0 diff --git a/scripts/python_dev.py b/scripts/python_dev.py new file mode 100755 index 0000000..86ccb35 --- /dev/null +++ b/scripts/python_dev.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Python task runner — one entry point for every Python CI/CD step, run locally or +from the workflows. Requires Python 3.11+ (tomllib) for `verify` and `versions` + + ./python_dev.py setup # install the package + all dev tooling + ./python_dev.py lint # ruff check + format --check + ./python_dev.py test # pytest + ./python_dev.py package # build wheel + sdist, twine check --strict + ./python_dev.py verify [TAG] # tag == pyproject version + publish guard + drift + ./python_dev.py list-supported-versions # print the supported-version matrix as JSON + ./python_dev.py smoke [VERSION] # install from PyPI + assert version + run CLI + ./python_dev.py ci # setup + lint + test + package (pre-flight) + +lint/test/build run inside a .venv the runner creates and reuses, so there is no +manual `python -m venv` step; delete .venv to rebuild it against a different +interpreter. +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +import time +import venv +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +PYTHON_DIR = REPO_ROOT / "python" +VENV = PYTHON_DIR / ".venv" +PYPROJECT = PYTHON_DIR / "pyproject.toml" +PACKAGE = "developer-toolkit-elasticache" +IMPORT_NAME = "developer_toolkit_elasticache" +DO_NOT_UPLOAD = "Private :: Do Not Upload" + + +_COMMANDS = { + "setup": None, + "lint": None, + "test": None, + "package": None, + "list-supported-versions": None, + "ci": None, + "verify": ("tag", "release tag, e.g. python-v1.2.3"), + "smoke": ("version", "published version, e.g. 1.2.3"), +} + + +def run(*args: object) -> None: + print(f"+ {' '.join(map(str, args))}") + subprocess.run([str(a) for a in args], check=True, cwd=PYTHON_DIR) + + +def venv_python() -> str: + """Create the managed .venv on first use and return its Python interpreter. + + Lets the runner own an isolated environment, so `./dev.py test` works with no + manual `python -m venv`. In CI each job is a fresh runner, so the .venv is built + from that job's interpreter — honouring the matrix version. + """ + if not VENV.exists(): + print(f"+ creating virtualenv at {VENV}") + venv.create(VENV, with_pip=True) + bindir = VENV / ("Scripts" if os.name == "nt" else "bin") + return str(bindir / "python") + + +def _project() -> dict: + import tomllib # 3.11+; imported lazily so setup/lint/test still load on 3.10 + + return tomllib.loads(PYPROJECT.read_text())["project"] + + +def _support() -> dict: + import tomllib + + return tomllib.loads(PYPROJECT.read_text())["tool"]["python-support"] + + +def _ensure_env() -> str: + """Create python/.venv and install the package + pinned dev tooling into it.""" + py = venv_python() + run(py, "-m", "pip", "install", "-q", "-e", ".", "-r", "requirements/dev.txt") + return py + + +def setup(_args: argparse.Namespace) -> int: + """Explicitly create python/.venv and install the package + pinned dev tooling.""" + _ensure_env() + print("Dev environment ready.") + return 0 + + +def lint(_args: argparse.Namespace) -> int: + py = _ensure_env() + run(py, "-m", "ruff", "check", ".") + run(py, "-m", "ruff", "format", "--check", ".") + return 0 + + +def test(_args: argparse.Namespace) -> int: + py = _ensure_env() + run(py, "-m", "pytest", "-q") + return 0 + + +def package(_args: argparse.Namespace) -> int: + py = _ensure_env() + run(py, "-m", "build") + dist = sorted(str(p) for p in (PYTHON_DIR / "dist").glob("*")) + if not dist: + raise SystemExit("build produced no artifacts in dist/") + run(py, "-m", "twine", "check", "--strict", *dist) + return 0 + + +def _support_drift(project: dict, support: dict) -> str | None: + """Report if [tool.python-support] disagrees with requires-python or the Python + classifiers, else None.""" + expected_requires = f">={support['min']}" + if project.get("requires-python") != expected_requires: + return ( + f"requires-python is {project.get('requires-python')!r} but " + f"[tool.python-support].min implies {expected_requires!r}." + ) + lo, hi = int(support["min"].split(".")[1]), int(support["max"].split(".")[1]) + expected = [f"3.{minor}" for minor in range(lo, hi + 1)] + declared = sorted( + m.group(1) + for c in project.get("classifiers", []) + if (m := re.match(r"Programming Language :: Python :: (3\.\d+)$", c)) + ) + if declared != expected: + return f"Python classifiers {declared} do not match python-support {expected}." + return None + + +def verify(args: argparse.Namespace) -> int: + tag = args.tag or os.environ.get("GITHUB_REF_NAME", "") + tag_version = tag.removeprefix("python-v") + project = _project() + pkg_version = project["version"] + if tag_version != pkg_version: + print( + "::error::Release tag does not match the version in pyproject.toml " + f"({pkg_version}). Bump the version and re-tag." + ) + return 1 + if DO_NOT_UPLOAD in project.get("classifiers", []): + print( + f"::error::The '{DO_NOT_UPLOAD}' classifier is still set. Remove it in the " + "release commit once the open-source review has cleared." + ) + return 1 + drift = _support_drift(project, _support()) + if drift: + print(f"::error::{drift}") + return 1 + print(f"Publishing version {pkg_version}.") + return 0 + + +def list_supported_versions(_args: argparse.Namespace) -> int: + support = _support() + lo_major, lo_minor = (int(p) for p in support["min"].split(".")[:2]) + hi_major, hi_minor = (int(p) for p in support["max"].split(".")[:2]) + if lo_major != hi_major: + raise SystemExit("[tool.python-support] min and max must share a major version.") + vers = [f"{lo_major}.{minor}" for minor in range(lo_minor, hi_minor + 1)] + if not vers: + raise SystemExit("[tool.python-support] min is greater than max.") + print(json.dumps({"all": vers, "min": vers[0], "max": vers[-1]})) + return 0 + + +def smoke(args: argparse.Namespace) -> int: + version = args.version or os.environ.get("GITHUB_REF_NAME", "").removeprefix( + "python-v" + ) + if not version: + wheels = sorted((PYTHON_DIR / "dist").glob("*.whl")) + if not wheels: + print("::error::No version given and no wheel in dist/ to derive it from.") + return 1 + version = wheels[0].name.split("-")[1] + + with tempfile.TemporaryDirectory() as tmp: + env = Path(tmp) / "venv" + venv.create(env, with_pip=True) + bindir = env / ("Scripts" if os.name == "nt" else "bin") + pip, python, cli = ( + str(bindir / "pip"), + str(bindir / "python"), + str(bindir / PACKAGE), + ) + + spec = f"{PACKAGE}=={version}" + for attempt in range(1, 6): + if subprocess.run([pip, "install", spec], check=False).returncode == 0: + break + print( + f"install attempt {attempt} failed (index may lag the upload); retrying in 15s" + ) + time.sleep(15) + else: + print(f"::error::Could not install {PACKAGE} from PyPI after 5 attempts.") + return 1 + + got = subprocess.run( + [ + python, + "-c", + f"import importlib.metadata as m; print(m.version('{PACKAGE}'))", + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if got != version: + print(f"::error::Installed the wrong version of {PACKAGE} ({got}).") + return 1 + + subprocess.run([cli, "--help"], check=True) + subprocess.run([python, "-c", f"import {IMPORT_NAME}"], check=True) + + print(f"Smoke test passed for {PACKAGE}.") + return 0 + + +def ci(args: argparse.Namespace) -> int: + lint(args) + test(args) + package(args) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + prog="dev.py", description="Developer/CI task runner." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + for name, arg in _COMMANDS.items(): + subparser = subparsers.add_parser(name) + if arg is not None: + subparser.add_argument(arg[0], nargs="?", help=arg[1]) + subparser.set_defaults(func=globals()[name.replace("-", "_")]) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main())