diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml new file mode 100644 index 0000000..4c1406d --- /dev/null +++ b/.github/workflows/test-docker.yml @@ -0,0 +1,136 @@ +name: "Test Docker actions" + +on: + pull_request: + paths: + - "docker-*/**" + - "tests/**" + - ".github/workflows/test-docker.yml" + push: + branches: [main] + paths: + - "docker-*/**" + - "tests/**" + - ".github/workflows/test-docker.yml" + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: python3 -m unittest discover -s tests + - run: shellcheck docker-*/*.sh + + metadata: + name: metadata (${{ matrix.name }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: base release + mode: release + version: 1.2.3 + aliases: "" + tags: "1.2.3,1.2,latest" + label: "1.2.3" + - name: base prerelease + mode: release + version: 1.2.3rc1 + aliases: "" + tags: "1.2.3rc1" + label: "1.2.3rc1" + - name: base postrelease + mode: release + version: 1.2.3.post1 + aliases: "" + tags: "1.2.3.post1" + label: "1.2.3.post1" + - name: base development version + mode: release + version: 1.2.3.dev1 + aliases: "" + tags: "1.2.3.dev1" + label: "1.2.3.dev1" + - name: base development + mode: development + version: "" + aliases: "" + tags: "sha-$SHA" + label: "sha-$SHA" + - name: base dry-run + mode: dry-run + version: "" + aliases: "" + tags: "dry-run" + label: "dry-run" + - name: variant release + mode: release + version: 1.2.3 + aliases: "alpine3.23\nalpine" + tags: "1.2.3-alpine3.23,1.2-alpine3.23,alpine3.23,1.2.3-alpine,1.2-alpine,alpine" + label: "1.2.3-alpine3.23" + - name: variant prerelease + mode: release + version: 1.2.3rc1 + aliases: "alpine3.23\nalpine" + tags: "1.2.3rc1-alpine3.23,alpine3.23,1.2.3rc1-alpine,alpine" + label: "1.2.3rc1-alpine3.23" + - name: variant development + mode: development + version: "" + aliases: "alpine3.23\nalpine" + tags: "sha-$SHA-alpine3.23,sha-$SHA-alpine" + label: "sha-$SHA-alpine3.23" + - name: variant dry-run + mode: dry-run + version: "" + aliases: "alpine3.23\nalpine" + tags: "alpine3.23,alpine" + label: "alpine3.23" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Generate tag rules + id: rules + working-directory: ${{ runner.temp }} + env: + ACTION_PATH: ${{ github.workspace }}/docker-build + MODE: ${{ matrix.mode }} + VERSION: ${{ matrix.version }} + ALIASES: ${{ matrix.aliases }} + run: | + printf '[project]\nversion = "%s"\n' "$VERSION" > pyproject.toml + python3 "$ACTION_PATH/metadata.py" + - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + id: metadata + with: + images: ghcr.io/astral-sh/test + tags: ${{ steps.rules.outputs.tags }} + flavor: ${{ steps.rules.outputs.flavor }} + - name: Check tags and version label + env: + METADATA: ${{ steps.metadata.outputs.json }} + EXPECTED_TAGS: ${{ matrix.tags }} + EXPECTED_LABEL: ${{ matrix.label }} + run: | + python3 - <<'PY' + import json + import os + + metadata = json.loads(os.environ["METADATA"]) + sha = os.environ["GITHUB_SHA"][:7] + expected_tags = { + f"ghcr.io/astral-sh/test:{tag}" + for tag in os.environ["EXPECTED_TAGS"].replace("$SHA", sha).split(",") + } + assert set(metadata["tags"]) == expected_tags, metadata["tags"] + expected_label = os.environ["EXPECTED_LABEL"].replace("$SHA", sha) + assert metadata["labels"]["org.opencontainers.image.version"] == expected_label, metadata["labels"] + PY diff --git a/README.md b/README.md index 3bff049..be48a6d 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ Shared GitHub Actions for Astral projects. - [Create release pull request](create-release-pr/): commit prepared release files and create or update a pull request. +- [Docker build and publication](docker-build/README.md): plan, version-check, + build, and publish Astral images with Depot and standard tags. - [Release smoke test](release-smoke-test/): test release artifacts in disposable containers. - [PR security review](security-pr-review/): review pull requests with Codex Security. - [PR review findings](pull-request-review/): format and publish structured review findings. diff --git a/docker-build/README.md b/docker-build/README.md new file mode 100644 index 0000000..3492df6 --- /dev/null +++ b/docker-build/README.md @@ -0,0 +1,172 @@ +# Docker build and publication + +These actions implement Astral's Docker release conventions for uv, Ruff, and ty. +They interpret the `cargo-dist` plan, check release versions against +`pyproject.toml`, generate standard tags and OCI metadata, and build multi-platform +images with Depot. Release candidates are saved and published later without +rebuilding. + +The calling repository supplies its Depot project IDs, binary names, Dockerfile, +and image inventory. It owns registry credentials, job permissions, protected +environments, and release approval. + +Use Linux runners with Docker, Bash, Python 3.11 or newer, and `jq`. Grant build jobs +`contents: read` and `id-token: write`, and configure the selected Depot project's +[GitHub Actions OIDC trust](https://depot.dev/docs/container-builds/integrations/github-actions). +Production and development builds should use separate Depot projects. Pin each +action reference to the same reviewed commit. + +## Plan the build + +`docker-plan` selects the build mode, Depot project, registry destinations, and +variant matrix. It expects `astral-sh/` or `astral-sh/-dev` as the +calling repository. + +```yaml +- uses: astral-sh/github-actions/docker-plan@ + id: plan + with: + name: ruff + plan: ${{ inputs.plan }} + push-dev: ${{ inputs.push-dev }} + production-project: ${{ vars.DEPOT_PROJECT_PROD }} + development-project: ${{ vars.DEPOT_PROJECT_DEV }} +``` + +| Build | Mode | Save in Depot | Publication | +| ------------------------- | ------------- | ------------- | ----------------------------------- | +| Pull request | `dry-run` | No | No | +| Release dry-run | `dry-run` | Yes | No | +| Release | `release` | Yes | After release approval | +| Trusted development build | `development` | Yes | `ghcr.io/astral-sh/-dev` only | + +Release builds use `ghcr.io/astral-sh/`. Set `dockerhub: true` to also use +`docker.io/astral/`, as uv does. Development publication requires a +same-repository pull request in one of the two Astral repositories and cannot be +combined with a release plan. Each repository supplies its own development Depot +project; release plans in the primary repository, including dry-runs, use the +production project. + +The default inventory contains Alpine, Debian slim, and buildpack-deps variants. +Set `image-mappings` to a newline-separated inventory to include additional base +images, Python versions, or DHI images. The action excludes DHI images when +authenticated pulls are disabled, including fork PRs and development-repository +pull-request builds. Use `matrix` for the build matrix and `login` to gate read-only registry +authentication. `publish` is build intent; the publishing workflow must still +enforce its release gates. + +## Build the base image + +Check out the source and authenticate any private or rate-limited base-image +pulls before invoking `docker-build`. + +```yaml +- uses: astral-sh/github-actions/docker-build@ + id: build + with: + name: ruff + project: ${{ needs.plan.outputs.depot-project-id }} + images: ${{ needs.plan.outputs.images }} + mode: ${{ needs.plan.outputs.mode }} + version: ${{ needs.plan.outputs.version }} + save: ${{ needs.plan.outputs.save }} +``` + +`docker-build` returns `build-id` and `digest`. It builds `linux/amd64,linux/arm64` +from the repository root, using `Dockerfile` unless `file` names a variant file. +`save: false` builds without creating a release candidate or +uploading an image record. Saved builds include maximum-mode provenance and an +SBOM, and upload a `docker-image-` artifact containing the build ID, digest, +tags, and index annotations. + +The base image's release version must exactly match `[project].version` in +the repository's `pyproject.toml`. A mismatch fails before contacting +Depot. Release tags use PEP 440 full and major/minor versions, with the full +version first for the OCI version label. Stable base releases also get `latest`; +PEP 440 pre-, post-, and development-version releases keep their full version and +do not move the minor or `latest` tags. Development images use SHA tags, and +base-image dry-runs use `dry-run`. + +## Build variants + +`docker-variant` creates a Dockerfile that copies the selected binaries into +`/usr/local/bin`, clears the inherited entrypoint, and uses the first binary as +the default command. +It also returns the record name and ordered aliases. `docker-build` generates +their tags using the same mode and version as the base image. + +```yaml +- uses: astral-sh/github-actions/docker-variant@ + id: variant + with: + image-mapping: alpine:3.23,alpine3.23,alpine + source-image: registry.depot.dev/${{ needs.plan.outputs.depot-project-id }}@${{ needs.base.outputs.digest }} + binaries: ruff + +- uses: astral-sh/github-actions/docker-build@ + with: + name: ${{ steps.variant.outputs.name }} + project: ${{ needs.plan.outputs.depot-project-id }} + pull-build-id: ${{ needs.base.outputs.build-id }} + file: ${{ steps.variant.outputs.file }} + images: ${{ needs.plan.outputs.images }} + aliases: ${{ steps.variant.outputs.aliases }} + mode: ${{ needs.plan.outputs.mode }} + version: ${{ needs.plan.outputs.version }} + save: ${{ needs.plan.outputs.save }} +``` + +For uv, use `binaries: uv uvx` and set `dockerfile-lines` to +`ENV UV_TOOL_BIN_DIR="/usr/local/bin"`. For ty, use `binaries: ty`. +Stable release variants get full-version, major/minor, and unversioned alias +tags. Pre-, post-, and development-version releases get full-version and +unversioned alias tags. The first alias determines the OCI version label. Development +variants get SHA tags with alias suffixes. Dry-run variants use only their +unversioned aliases. Variants never receive an automatic `latest` tag and use the release version +already checked by their base-image build, without needing a source checkout. +In an unsaved pull-request build, omit `pull-build-id` and use the plan's +`ghcr-image` with the `latest` tag as the source image. + +## Collect the manifest + +After every build finishes, `docker-manifest` downloads the per-image artifacts +and requires exactly the expected base and variant names. It uploads +`docker-manifest.json` and returns its artifact ID. + +```yaml +- uses: astral-sh/github-actions/docker-manifest@ + id: manifest + with: + base-name: ruff + image-mappings: '["alpine:3.23,alpine3.23,alpine"]' +``` + +The manifest has a `base` record and an `extra` array. Per-image artifacts use +`docker-image-`; the assembled artifact is named `docker-manifest`. + +## Publish + +The publishing workflow must enforce its own trusted event, branch, repository, +and environment restrictions. Download the manifest by artifact ID, select the +Depot project and allowed destinations independently of its contents, and log in +to those registries. Grant `contents: read`, `id-token: write`, `packages: write`, +and `attestations: write` for GHCR publication and attestations. + +```yaml +- uses: astral-sh/github-actions/docker-publish@ + with: + project: ${{ vars.DEPOT_PROJECT_PROD }} + image: ${{ inputs.image }} + images: ghcr.io/astral-sh/ruff + attestation-image: ghcr.io/astral-sh/ruff +``` + +The publisher rejects tags outside the supplied destination list. It verifies +the saved Depot digest, copies the full index to each registry, verifies the +copy, and applies the recorded annotations and tags. It returns the annotated +digest for `attestation-image` and attests both that digest and the original. +Throttled Depot manifest pushes are retried with exponential backoff and jitter. +Publish variants before the base image to keep the base first in GHCR's package +listing. + +Run the local contract tests with `python3 -m unittest discover -s tests`. diff --git a/docker-build/action.yml b/docker-build/action.yml new file mode 100644 index 0000000..464c696 --- /dev/null +++ b/docker-build/action.yml @@ -0,0 +1,108 @@ +name: "Build Docker image" +description: "Build an Astral image with standard version checks and tags, and optionally save it in Depot Registry." + +inputs: + name: + description: "Unique name for this image in the build manifest." + required: true + project: + description: "Depot project ID." + required: true + images: + description: "Newline-separated destination image repositories." + required: true + mode: + description: "The tag mode from `docker-plan`: `release`, `development`, or `dry-run`." + default: "dry-run" + version: + description: "Release version, checked against `project.version` in the checkout's `pyproject.toml` for base-image builds." + default: "" + aliases: + description: "Newline-separated variant aliases from `docker-variant`; empty for the base image." + default: "" + file: + description: "Path to the Dockerfile." + default: "Dockerfile" + save: + description: "Save the image in Depot Registry and upload its record." + default: "false" + pull-build-id: + description: "Optional saved build in this Depot project to authenticate pulls from." + default: "" + +outputs: + build-id: + description: "Depot build ID." + value: ${{ steps.build.outputs.build-id }} + digest: + description: "Original multi-platform image digest." + value: ${{ steps.build.outputs.digest }} + +runs: + using: "composite" + steps: + - name: "Check tag consistency and prepare image tags" + id: tags + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + MODE: ${{ inputs.mode }} + VERSION: ${{ inputs.version }} + ALIASES: ${{ inputs.aliases }} + run: python3 "$ACTION_PATH/metadata.py" + + - uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1.7.2 + + - name: "Authenticate to Depot Registry" + if: inputs.pull-build-id != '' + shell: bash + env: + DEPOT_PROJECT_ID: ${{ inputs.project }} + BUILD_ID: ${{ inputs.pull-build-id }} + run: | + token="$(depot pull-token --project "$DEPOT_PROJECT_ID" "$BUILD_ID")" + echo "::add-mask::$token" + printf '%s' "$token" | docker login registry.depot.dev --username x-token --password-stdin + + - name: "Extract image metadata" + id: meta + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: index + with: + images: ${{ inputs.images }} + tags: ${{ steps.tags.outputs.tags }} + flavor: ${{ steps.tags.outputs.flavor }} + + - name: "Build and save image" + id: build + uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.18.0 + with: + project: ${{ inputs.project }} + context: . + file: ${{ inputs.file }} + platforms: linux/amd64,linux/arm64 + save: ${{ inputs.save }} + labels: ${{ steps.meta.outputs.labels }} + provenance: mode=max + sbom: true + + - name: "Record image" + id: record + if: inputs.save == 'true' + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + NAME: ${{ inputs.name }} + BUILD_ID: ${{ steps.build.outputs.build-id }} + DIGEST: ${{ steps.build.outputs.digest }} + TAGS: ${{ steps.meta.outputs.tags }} + ANNOTATIONS: ${{ steps.meta.outputs.annotations }} + run: bash "$ACTION_PATH/record.sh" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: inputs.save == 'true' + with: + name: docker-image-${{ inputs.name }} + path: ${{ steps.record.outputs.path }} + overwrite: true diff --git a/docker-build/metadata.py b/docker-build/metadata.py new file mode 100644 index 0000000..0c6a631 --- /dev/null +++ b/docker-build/metadata.py @@ -0,0 +1,62 @@ +"""Check the project release version and generate Astral Docker tag rules.""" + +import os +import re +import tomllib +from pathlib import Path + + +def main(): + mode = os.environ["MODE"] + version = os.environ["VERSION"] + aliases = os.environ["ALIASES"].splitlines() + if mode not in {"release", "development", "dry-run"}: + raise SystemExit(f"Unknown Docker build mode: {mode!r}") + if any( + not re.fullmatch(r"[a-zA-Z0-9_][a-zA-Z0-9_.-]*", alias) for alias in aliases + ): + raise SystemExit("Invalid Docker image alias") + + if mode == "release": + if not re.fullmatch(r"[0-9][a-zA-Z0-9.!+_-]*", version): + raise SystemExit("Release builds require a release version") + # Variants use the version checked by their base-image build and do not + # need a source checkout. + if not aliases: + pyproject = Path("pyproject.toml") + with pyproject.open("rb") as source: + project_version = tomllib.load(source)["project"]["version"] + if version != project_version: + raise SystemExit( + f"The release version {version!r} does not match " + f"project.version {project_version!r} in {pyproject}" + ) + print(f"Releasing {version}") + + tags = [] + for alias in aliases or [""]: + suffix = f",suffix=-{alias}" if alias else "" + if mode == "release": + # The full version and first alias determine the OCI version label. + tags.append( + f"type=pep440,pattern={{{{ version }}}}{suffix},value={version}" + ) + tags.append( + f"type=pep440,pattern={{{{ major }}}}.{{{{ minor }}}}{suffix},value={version}" + ) + elif mode == "development": + tags.append(f"type=sha{suffix}") + + if mode != "development": + if alias: + tags.append(f"type=raw,value={alias}") + elif mode == "dry-run": + tags.append("type=raw,value=dry-run") + + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + print("tags< "$RUNNER_TEMP/docker-image-record.json" +echo "path=$RUNNER_TEMP/docker-image-record.json" >> "$GITHUB_OUTPUT" diff --git a/docker-manifest/action.yml b/docker-manifest/action.yml new file mode 100644 index 0000000..5c194de --- /dev/null +++ b/docker-manifest/action.yml @@ -0,0 +1,41 @@ +name: "Collect Docker image manifest" +description: "Collect a complete set of saved image records for a separate publishing job." + +inputs: + base-name: + description: "Name of the binary-only base image." + required: true + image-mappings: + description: "JSON array of the expected variant image mappings." + default: "[]" + +outputs: + artifact-id: + description: "Artifact ID for the assembled Docker image manifest." + value: ${{ steps.upload.outputs.artifact-id }} + +runs: + using: "composite" + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: docker-image-* + path: ${{ runner.temp }}/docker-image-records + merge-multiple: false + + - name: "Assemble image manifest" + id: manifest + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + BASE_NAME: ${{ inputs.base-name }} + IMAGE_MAPPINGS: ${{ inputs.image-mappings }} + RECORDS_PATH: ${{ runner.temp }}/docker-image-records + run: bash "$ACTION_PATH/assemble.sh" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + id: upload + with: + name: docker-manifest + path: ${{ steps.manifest.outputs.path }} + overwrite: true diff --git a/docker-manifest/assemble.sh b/docker-manifest/assemble.sh new file mode 100644 index 0000000..5bd304b --- /dev/null +++ b/docker-manifest/assemble.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Require exactly the expected names, not only the expected number of records. +# The first alias in each mapping is its manifest record name. +jq -se --arg base "$BASE_NAME" --argjson mappings "$IMAGE_MAPPINGS" ' + ([$base] + ($mappings | map(split(",")[1]))) as $expected + | if ($expected | unique | length) != ($expected | length) + or (map(.name) | sort) != ($expected | sort) + then error("incomplete Docker image manifest") + else {base: map(select(.name == $base))[0], + extra: map(select(.name != $base))} + end +' "$RECORDS_PATH"/*/*.json > "$RUNNER_TEMP/docker-manifest.json" +echo "path=$RUNNER_TEMP/docker-manifest.json" >> "$GITHUB_OUTPUT" diff --git a/docker-plan/action.yml b/docker-plan/action.yml new file mode 100644 index 0000000..73951ba --- /dev/null +++ b/docker-plan/action.yml @@ -0,0 +1,74 @@ +name: "Plan Astral Docker images" +description: "Select build and publication settings for an Astral release, development build, or dry run." + +inputs: + name: + description: "Astral project name, such as `uv`, `ruff`, or `ty`." + required: true + plan: + description: "The `cargo-dist` release plan, or an empty string for a pull-request build." + default: "" + push-dev: + description: "Plan SHA-tagged development-image publication from a same-repository Astral pull request." + default: "false" + production-project: + description: "Depot project for release builds in the primary repository." + required: true + development-project: + description: "Depot project for development builds in the calling repository." + required: true + dockerhub: + description: "Include `docker.io/astral/` as a release-image destination." + default: "false" + image-mappings: + description: "One variant per line: a base image followed by comma-separated aliases." + default: | + alpine:3.23,alpine3.23,alpine + debian:trixie-slim,trixie-slim,debian-slim + buildpack-deps:trixie,trixie,debian + +outputs: + depot-project-id: + description: "Depot project for this build." + value: ${{ steps.plan.outputs.depot-project-id }} + images: + description: "Newline-separated Astral registry destinations." + value: ${{ steps.plan.outputs.images }} + ghcr-image: + description: "GHCR repository for the base image and its variants." + value: ${{ steps.plan.outputs.ghcr-image }} + login: + description: "Whether the caller should authenticate Docker Hub and DHI pulls." + value: ${{ steps.plan.outputs.login }} + publish: + description: "Whether this build is intended for publication after the caller's release gates." + value: ${{ steps.plan.outputs.publish }} + mode: + description: "The tag mode: `release`, `development`, or `dry-run`." + value: ${{ steps.plan.outputs.mode }} + save: + description: "Whether to save images in Depot Registry for the publication handoff." + value: ${{ steps.plan.outputs.save }} + version: + description: "Release version to check against `pyproject.toml`; empty for other builds." + value: ${{ steps.plan.outputs.version }} + matrix: + description: "Variant matrix, excluding DHI images when authenticated pulls are disabled." + value: ${{ steps.plan.outputs.matrix }} + +runs: + using: "composite" + steps: + - name: "Plan images" + id: plan + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + NAME: ${{ inputs.name }} + PLAN: ${{ inputs.plan }} + PUSH_DEV: ${{ inputs.push-dev }} + PRODUCTION_PROJECT: ${{ inputs.production-project }} + DEVELOPMENT_PROJECT: ${{ inputs.development-project }} + DOCKERHUB: ${{ inputs.dockerhub }} + IMAGE_MAPPINGS: ${{ inputs.image-mappings }} + run: python3 "$ACTION_PATH/plan.py" diff --git a/docker-plan/plan.py b/docker-plan/plan.py new file mode 100644 index 0000000..b260069 --- /dev/null +++ b/docker-plan/plan.py @@ -0,0 +1,91 @@ +"""Plan Astral Docker builds without granting permission to publish them.""" + +import json +import os +import re +from pathlib import Path + + +def main(): + name = os.environ["NAME"] + if not re.fullmatch(r"[a-z][a-z0-9-]*", name): + raise SystemExit(f"Invalid Astral project name: {name!r}") + + repository = os.environ["GITHUB_REPOSITORY"] + primary_repository = f"astral-sh/{name}" + if repository not in {primary_repository, f"{primary_repository}-dev"}: + raise SystemExit(f"No Astral Docker project configured for {repository}") + + event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) + head_repository = event.get("pull_request", {}).get("head", {}).get("repo") or {} + same_repository_pr = ( + os.environ["GITHUB_EVENT_NAME"] == "pull_request" + and head_repository.get("full_name") == repository + ) + primary_repository_pr = repository == primary_repository and same_repository_pr + release_plan = os.environ["PLAN"] + push_dev = os.environ["PUSH_DEV"] == "true" + + version = "" + if push_dev: + if not same_repository_pr or release_plan: + raise SystemExit( + "Development publication requires a same-repository Astral pull request " + "without a release plan" + ) + mode = "development" + save = True + publish = True + elif release_plan: + plan = json.loads(release_plan) + implicit = plan["announcement_tag_is_implicit"] + if not isinstance(implicit, bool): + raise SystemExit( + "The release plan must declare whether its tag is implicit" + ) + mode = "dry-run" if implicit else "release" + save = True + publish = not implicit + if publish: + version = plan["announcement_tag"] + if not isinstance(version, str) or not re.fullmatch( + r"[0-9][a-zA-Z0-9.!+_-]*", version + ): + raise SystemExit("The release plan must contain a release version") + else: + mode = "dry-run" + save = False + publish = False + + login = mode == "release" or primary_repository_pr + project = ( + os.environ["PRODUCTION_PROJECT"] + if repository == primary_repository and release_plan and not push_dev + else os.environ["DEVELOPMENT_PROJECT"] + ) + if not re.fullmatch(r"[a-zA-Z0-9]+", project): + raise SystemExit("A Depot project ID is required") + + images = [f"ghcr.io/astral-sh/{name}{'-dev' if push_dev else ''}"] + if os.environ["DOCKERHUB"] == "true" and not push_dev: + images.append(f"docker.io/astral/{name}") + mappings = [ + mapping + for line in os.environ["IMAGE_MAPPINGS"].splitlines() + if (mapping := line.strip()) and (login or not mapping.startswith("dhi.io/")) + ] + + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + print(f"depot-project-id={project}", file=output) + print(f"ghcr-image={images[0]}", file=output) + print("images< 0 + and ($destinations | unique | length) == ($destinations | length) + and ($destinations | index($attestation_image)) != null + and all($destinations[]; test("^[a-z0-9][a-z0-9./:_-]*[a-z0-9]$")) + and (.tags | type == "array") + and all(.tags[]; . as $tag | any($destinations[]; + . as $image | ($tag | startswith($image + ":")) + and ($tag | ltrimstr($image + ":") | test("^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$")))) + and all($destinations[]; . as $image | any($record.tags[]; startswith($image + ":"))) + and (.annotations | type == "array") + and all(.annotations[]; type == "string" and (contains("\n") | not)) +' <<< "$IMAGE_JSON" > /dev/null + +# Verify that the Depot build ID resolves to the recorded index digest. +token="$(depot pull-token --project "$DEPOT_PROJECT_ID" "$build_id")" +echo "::add-mask::$token" +printf '%s' "$token" | docker login registry.depot.dev --username x-token --password-stdin +saved_digest=$(docker buildx imagetools inspect \ + "registry.depot.dev/${DEPOT_PROJECT_ID}:$build_id" --format '{{.Manifest.Digest}}') +if [[ "$saved_digest" != "$digest" ]]; then + echo "The saved Depot image does not match the build manifest" >&2 + exit 1 +fi + +# Keep annotation values containing spaces or shell-special characters together. +annotations=() +while IFS= read -r annotation; do + annotations+=(--annotation "$annotation") +done < <(jq -r '.annotations[]' <<< "$IMAGE_JSON") + +while IFS= read -r image; do + readarray -t image_tags < <( + jq -r --arg prefix "$image:" '.tags[] | select(startswith($prefix))' <<< "$IMAGE_JSON" + ) + + # Copy the complete index without loading it through the local image store, + # which can drop platforms or change the manifest format (astral-sh/uv#14165). + bash "$ACTION_PATH/push.sh" --project "$DEPOT_PROJECT_ID" --tag "${image_tags[0]}" "$build_id" + copied_digest=$(docker buildx imagetools inspect \ + "${image_tags[0]}" --format '{{.Manifest.Digest}}') + if [[ "$copied_digest" != "$digest" ]]; then + echo "The copied image digest does not match the build manifest" >&2 + exit 1 + fi + + tags=() + for tag in "${image_tags[@]}"; do tags+=(-t "$tag"); done + docker buildx imagetools create \ + "${annotations[@]}" \ + "${tags[@]}" \ + --metadata-file "$RUNNER_TEMP/docker-publish-metadata.json" \ + "$image@$digest" + + if [[ "$image" == "$ATTESTATION_IMAGE" ]]; then + final_digest=$(jq -er '."containerimage.descriptor".digest | select(test("^sha256:[0-9a-f]{64}$"))' \ + "$RUNNER_TEMP/docker-publish-metadata.json") + echo "digest=$final_digest" >> "$GITHUB_OUTPUT" + fi +done < <(jq -r '.[]' <<< "$destinations") diff --git a/docker-publish/push.sh b/docker-publish/push.sh new file mode 100644 index 0000000..9eaad14 --- /dev/null +++ b/docker-publish/push.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +# A registry can throttle a manifest push after accepting its blobs. +# Retry the same saved build with backoff and jitter. +retry_delay=5 +for attempt in {1..5}; do + if depot push "$@"; then + exit 0 + else + push_status=$? + fi + if [[ "$attempt" -eq 5 ]]; then + exit "$push_status" + fi + sleep_seconds=$((retry_delay + RANDOM % 5)) + echo "::warning::Depot push failed (attempt $attempt/5); retrying in ${sleep_seconds}s" >&2 + sleep "$sleep_seconds" + retry_delay=$((retry_delay * 2)) +done diff --git a/docker-variant/action.yml b/docker-variant/action.yml new file mode 100644 index 0000000..9c1da17 --- /dev/null +++ b/docker-variant/action.yml @@ -0,0 +1,41 @@ +name: "Prepare Docker image variant" +description: "Generate a Dockerfile and aliases for an Astral image variant containing the selected binaries." + +inputs: + image-mapping: + description: "Base image followed by one or more comma-separated image aliases." + required: true + source-image: + description: "Image containing the binaries, preferably pinned to a digest." + required: true + binaries: + description: "Space-separated binary names at the root of the source image." + required: true + dockerfile-lines: + description: "Additional Dockerfile instructions after copying the binaries." + default: "" + +outputs: + name: + description: "First image alias, used as the manifest record name." + value: ${{ steps.prepare.outputs.name }} + file: + description: "Generated Dockerfile path." + value: ${{ steps.prepare.outputs.file }} + aliases: + description: "Newline-separated variant aliases for `docker-build`." + value: ${{ steps.prepare.outputs.aliases }} + +runs: + using: "composite" + steps: + - name: "Generate Dockerfile and aliases" + id: prepare + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + IMAGE_MAPPING: ${{ inputs.image-mapping }} + SOURCE_IMAGE: ${{ inputs.source-image }} + BINARIES: ${{ inputs.binaries }} + DOCKERFILE_LINES: ${{ inputs.dockerfile-lines }} + run: bash "$ACTION_PATH/prepare.sh" diff --git a/docker-variant/prepare.sh b/docker-variant/prepare.sh new file mode 100644 index 0000000..7a17618 --- /dev/null +++ b/docker-variant/prepare.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +IFS=',' read -r base_image aliases <<< "$IMAGE_MAPPING" +if [[ -z "$base_image" || -z "$aliases" || -z "$SOURCE_IMAGE" || "$base_image$SOURCE_IMAGE" == *[$'\n\r\t ']* ]]; then + echo "Expected a base image and at least one image alias" >&2 + exit 1 +fi +IFS=',' read -r -a image_tags <<< "$aliases" +for tag in "${image_tags[@]}"; do + if [[ ! "$tag" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]*$ ]]; then + echo "Invalid image alias: $tag" >&2 + exit 1 + fi +done + +read -r -a binary_names <<< "$BINARIES" +if [[ ${#binary_names[@]} -eq 0 ]]; then + echo "At least one binary name is required" >&2 + exit 1 +fi +for binary in "${binary_names[@]}"; do + if [[ ! "$binary" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]*$ ]]; then + echo "Invalid binary name: $binary" >&2 + exit 1 + fi +done + +dockerfile="$RUNNER_TEMP/docker-variant.Dockerfile" +{ + printf 'FROM %s\nCOPY --from=%s' "$base_image" "$SOURCE_IMAGE" + printf ' /%s' "${binary_names[@]}" + printf ' /usr/local/bin/\n' + if [[ -n "$DOCKERFILE_LINES" ]]; then printf '%s\n' "$DOCKERFILE_LINES"; fi + printf 'ENTRYPOINT []\nCMD ["/usr/local/bin/%s"]\n' "${binary_names[0]}" +} > "$dockerfile" + +{ + echo "name=${image_tags[0]}" + echo "file=$dockerfile" + echo "aliases<> "$GITHUB_OUTPUT" diff --git a/tests/docker-cli-stub.py b/tests/docker-cli-stub.py new file mode 100755 index 0000000..a3a914a --- /dev/null +++ b/tests/docker-cli-stub.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Record Docker/Depot calls for the publisher contract tests.""" + +import json +import os +import sys +from pathlib import Path + +command = [Path(sys.argv[0]).name, *sys.argv[1:]] +with open(os.environ["CALLS"], "a") as calls: + calls.write(json.dumps(command) + "\n") + +if command[:2] == ["depot", "pull-token"]: + print("test-pull-token") +elif command[:2] == ["depot", "push"]: + pushes = [ + json.loads(line) + for line in Path(os.environ["CALLS"]).read_text().splitlines() + if json.loads(line)[:2] == ["depot", "push"] + ] + if len(pushes) <= int(os.environ.get("DEPOT_PUSH_FAILURES", "0")): + raise SystemExit(1) +elif command[:2] == ["docker", "login"]: + sys.stdin.read() +elif command[0] == "sleep": + pass +elif command[:4] == ["docker", "buildx", "imagetools", "inspect"]: + variable = ( + "SAVED_DIGEST" + if command[4].startswith("registry.depot.dev/") + else "COPIED_DIGEST" + ) + print(os.environ.get(variable, os.environ["DIGEST"])) +elif command[:4] == ["docker", "buildx", "imagetools", "create"]: + metadata = Path(command[command.index("--metadata-file") + 1]) + metadata.write_text( + json.dumps( + {"containerimage.descriptor": {"digest": os.environ["FINAL_DIGEST"]}} + ) + ) +else: + raise SystemExit(f"unexpected command: {command!r}") diff --git a/tests/test_docker.py b/tests/test_docker.py new file mode 100644 index 0000000..506d70d --- /dev/null +++ b/tests/test_docker.py @@ -0,0 +1,215 @@ +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DIGEST = "sha256:" + "a" * 64 +FINAL_DIGEST = "sha256:" + "b" * 64 + + +class DockerTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.path = Path(self.directory.name) + self.output = self.path / "output" + self.output.touch() + self.calls = self.path / "calls" + self.calls.touch() + self.environment = dict( + os.environ, RUNNER_TEMP=str(self.path), GITHUB_OUTPUT=str(self.output) + ) + + def run_script(self, script, *, success=True, **environment): + result = subprocess.run( + ["bash", str(ROOT / script)], + env=dict(self.environment, **environment), + text=True, + capture_output=True, + check=False, + ) + if success: + self.assertEqual(result.returncode, 0, result.stderr) + else: + self.assertNotEqual(result.returncode, 0, result.stdout) + return result + + def image(self, name="ruff"): + return { + "name": name, + "build_id": "build123", + "digest": DIGEST, + "tags": [ + f"ghcr.io/astral-sh/{name}:1.2.3", + f"ghcr.io/astral-sh/{name}:latest", + ], + "annotations": [ + "index:org.opencontainers.image.description=spaces; $and `quotes`" + ], + } + + def test_record(self): + record = self.image() + self.run_script( + "docker-build/record.sh", + NAME=record["name"], + BUILD_ID=record["build_id"], + DIGEST=DIGEST, + TAGS="\n".join(record["tags"]), + ANNOTATIONS="\n".join(record["annotations"]), + ) + self.assertEqual( + json.loads((self.path / "docker-image-record.json").read_text()), record + ) + self.run_script( + "docker-build/record.sh", + success=False, + NAME="../ruff", + BUILD_ID="build123", + DIGEST=DIGEST, + TAGS="", + ANNOTATIONS="", + ) + + def test_variants(self): + for product, binaries, extra in [ + ("uv", "uv uvx", 'ENV UV_TOOL_BIN_DIR="/usr/local/bin"'), + ("ruff", "ruff", ""), + ("ty", "ty", ""), + ]: + with self.subTest(product=product): + self.output.write_text("") + self.run_script( + "docker-variant/prepare.sh", + IMAGE_MAPPING="alpine:3.23,alpine3.23,alpine", + SOURCE_IMAGE=f"registry.depot.dev/project@{DIGEST}", + BINARIES=binaries, + DOCKERFILE_LINES=extra, + ) + expected = ( + f"FROM alpine:3.23\nCOPY --from=registry.depot.dev/project@{DIGEST}" + + "".join(f" /{binary}" for binary in binaries.split()) + + " /usr/local/bin/\n" + + (extra + "\n" if extra else "") + + f'ENTRYPOINT []\nCMD ["/usr/local/bin/{product}"]\n' + ) + self.assertEqual( + (self.path / "docker-variant.Dockerfile").read_text(), expected + ) + self.assertEqual( + self.output.read_text(), + f"name=alpine3.23\nfile={self.path}/docker-variant.Dockerfile\n" + "aliases<