diff --git a/.github/actions/packer.build-targets/action.yaml b/.github/actions/packer.build-targets/action.yaml new file mode 100644 index 0000000..a0bd207 --- /dev/null +++ b/.github/actions/packer.build-targets/action.yaml @@ -0,0 +1,189 @@ +name: Packer - Build Targets +description: Resolve the Packer build directories to build, and the variable files they build with, for use as a job matrix. + + +inputs: + mode: + required: false + default: changed + description: "(Optional) How the build directories are chosen. `changed` resolves them from `directories`, for a push or a pull request. `all` enumerates every build directory under `builds_dir`, for a scheduled or manually dispatched full build. Defaults to `changed`." + directories: + required: false + description: "(Required in `changed` mode) A JSON array of changed directories, typically the `directories` output of the `git.changed-dirs` action. Each is resolved to the nearest directory, itself or an ancestor, that holds a Packer template." + builds_dir: + required: false + default: builds + description: "(Optional, `all` mode) The directory whose immediate subdirectories are the build directories. Defaults to `builds`." + vars_dir: + required: false + default: vars + description: "(Optional) The subdirectory of a build directory holding its variable files, one per variant. A build directory without it yields a single target that builds with no `-var-file`, leaving any `*.auto.pkrvars.hcl` beside the template to apply on its own. Defaults to `vars`." + vars_pattern: + required: false + default: "*.pkrvars.hcl" + description: "(Optional) The glob matched against `vars_dir` to find the variable files. Defaults to `*.pkrvars.hcl`." + builds: + required: false + description: "(Optional) A comma-separated list of build directory names to keep (e.g. `ubuntu-2604,amazon-linux-2023`), for a manual dispatch of part of the repository. Empty keeps every resolved build." + variants: + required: false + description: "(Optional) A comma-separated list of variant names to keep (e.g. `prod`), matched against the variable file name without its extension. Empty keeps every variant." + +outputs: + has_targets: + value: ${{ steps.resolve.outputs.has_targets }} + description: "Whether `targets` is non-empty. `true` or `false`." + targets: + value: ${{ steps.resolve.outputs.targets }} + description: "A JSON array of `{\"name\": ..., \"build\": ..., \"path\": ..., \"variant\": ..., \"var_file\": ...}` objects for `strategy.matrix.include`. `build` is the directory name, `path` the directory, `variant` the variable file's name without its extension (empty when the build has none), `var_file` that file's path (empty likewise), and `name` the ` / ` label the workflow reuses for the job name, the concurrency group, and the report row." + + +runs: + using: composite + + steps: + - name: Resolve Build Targets + id: resolve + shell: bash + env: + MODE: ${{ inputs.mode }} + DIRECTORIES: ${{ inputs.directories }} + BUILDS_DIR: ${{ inputs.builds_dir }} + VARS_DIR: ${{ inputs.vars_dir }} + VARS_PATTERN: ${{ inputs.vars_pattern }} + BUILDS: ${{ inputs.builds }} + VARIANTS: ${{ inputs.variants }} + run: | + case "$MODE" in + changed|all) ;; + *) + echo "::error::Invalid mode: $MODE. Valid values are changed or all." + exit 1 + ;; + esac + + shopt -s nullglob + + # A directory is a build directory when it holds a Packer template, the same test the + # `packer.fmt` and `packer.validate` actions use to decide whether there is anything to do. + is_build_dir() { + local dir="$1" path + for path in "$dir"/*.pkr.hcl "$dir"/*.pkr.json; do + [ -f "$path" ] && return 0 + done + return 1 + } + + # Empty keeps everything, so an unset filter is not the same as one that matched nothing. + keeps() { + local filter="$1" value="$2" wanted + [ -z "$filter" ] && return 0 + IFS=',' read -ra wanted <<< "$filter" + for name in "${wanted[@]}"; do + name="${name// /}" + [ -n "$name" ] && [ "$name" = "$value" ] && return 0 + done + return 1 + } + + # ---- the build directories to consider ---- + build_dirs=() + add_build_dir() { + local dir="${1%/}" + for existing in "${build_dirs[@]}"; do + [ "$existing" = "$dir" ] && return 0 + done + build_dirs+=("$dir") + } + + if [ "$MODE" = "all" ]; then + for path in "${BUILDS_DIR%/}"/*/; do + if is_build_dir "${path%/}"; then + add_build_dir "$path" + else + echo "Skipping ${path%/}: no Packer template files." + fi + done + else + while IFS= read -r dir; do + [ -z "$dir" ] && continue + + # Walk up from the changed directory, so a change under `/vars/` still builds ``. + current="${dir%/}" + found="" + while :; do + if is_build_dir "$current"; then + found="$current" + break + fi + [ "$current" = "." ] && break + current="$(dirname "$current")" + done + + if [ -z "$found" ]; then + echo "Skipping $dir: no Packer template files in the directory or its parents." + continue + fi + + echo "$dir -> $found" + add_build_dir "$found" + done < <(jq -r '.[]' <<< "${DIRECTORIES:-[]}") + fi + + # ---- one target per variable file, or a single one when the build has none ---- + targets="[]" + add_target() { + targets="$(jq -c \ + --arg name "$1" --arg build "$2" --arg path "$3" --arg variant "$4" --arg var_file "$5" \ + '. + [{name: $name, build: $build, path: $path, variant: $variant, var_file: $var_file}]' \ + <<< "$targets")" + } + + for dir in "${build_dirs[@]}"; do + build="$(basename "$dir")" + if ! keeps "$BUILDS" "$build"; then + echo "Skipping $dir: not in builds." + continue + fi + + # `find` takes the glob as one quoted argument, so a pattern is not word-split, a missing + # directory is not an error, and the order is stable across runs, which keeps the matrix stable. + var_files=() + while IFS= read -r var_file; do + [ -n "$var_file" ] && var_files+=("$var_file") + done < <(find "$dir/${VARS_DIR%/}" -maxdepth 1 -type f -name "$VARS_PATTERN" 2>/dev/null | sort) + + if [ "${#var_files[@]}" -eq 0 ]; then + echo "$dir (no variable files)" + add_target "$build" "$build" "$dir" "" "" + continue + fi + + for var_file in "${var_files[@]}"; do + variant="$(basename "$var_file")" + # Strip the whole known suffix, so `prod.auto.pkrvars.hcl` stays `prod.auto` rather than losing + # a second extension. Anything else loses only its last one. + if [ "$variant" != "${variant%.pkrvars.hcl}" ]; then + variant="${variant%.pkrvars.hcl}" + else + variant="${variant%.*}" + fi + if ! keeps "$VARIANTS" "$variant"; then + echo "Skipping $var_file: not in variants." + continue + fi + + echo "$dir / $variant ($var_file)" + add_target "$build / $variant" "$build" "$dir" "$variant" "$var_file" + done + done + + if [ "$(jq 'length' <<< "$targets")" -gt 0 ]; then + has_targets=true + else + has_targets=false + fi + + echo "Targets: $targets" + echo "has_targets=$has_targets" >> "$GITHUB_OUTPUT" + echo "targets=$targets" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/packer.build/action.yaml b/.github/actions/packer.build/action.yaml new file mode 100644 index 0000000..56f8c4f --- /dev/null +++ b/.github/actions/packer.build/action.yaml @@ -0,0 +1,171 @@ +name: Packer - Build +description: Install the required plugins of a Packer template directory with `packer init` and build it with `packer build`, then render a readable summary of the artifacts it produced. On failure, the output is added to the job summary. Requires the `packer` CLI on the `PATH`. + + +inputs: + target_dir: + required: false + default: ./ + description: "(Optional) The Packer template directory to build. Defaults to `./`." + var_files: + required: false + description: "(Optional) The variable files to build with, one path per line, each passed as `-var-file`. Paths are relative to the working directory, not to `target_dir`. A `*.auto.pkrvars.hcl` file beside the template is loaded by Packer on its own and does not belong here." + args: + required: false + description: "(Optional) Additional arguments to pass to `packer build` (e.g. `-only=amazon-ebs.base` or `-var 'key=value'`)." + github_token: + required: false + default: ${{ github.token }} + description: "(Optional) The GitHub token used by `packer init` to download plugins from GitHub without hitting the anonymous rate limit, exported as `PACKER_GITHUB_API_TOKEN`. Defaults to the automatically generated `github.token`." + +outputs: + skipped: + value: ${{ steps.target.outputs.skipped }} + description: "`true` when `target_dir` holds no Packer template files and neither `packer init` nor `packer build` was run. Empty otherwise." + artifact_count: + value: ${{ steps.build.outputs.artifact_count }} + description: "The number of builds that produced an artifact, counted from the artifacts section Packer prints when it finishes." + artifacts: + value: ${{ steps.build.outputs.artifacts }} + description: "The artifacts section Packer printed, one build per `--> ` line with whatever the builder reported below it, such as the AMI ids of an `amazon-ebs` build. Empty when the build produced none." + headline: + value: ${{ steps.build.outputs.headline }} + description: "A single line stating the outcome, suitable for the `headline` input of `github.matrix-report`." + summary: + value: ${{ steps.build.outputs.summary }} + description: "A Markdown summary of the build, suitable for a job summary or a pull request comment." + exitcode: + value: ${{ steps.build.outputs.exitcode }} + description: "The exit code of the call to `packer build`. Empty when skipped. The action still fails on a non-zero exit code, so use `continue-on-error: true` to inspect it." + + +runs: + using: composite + + steps: + - name: Resolve Target + id: target + shell: bash + env: + TARGET_DIR: ${{ inputs.target_dir }} + run: | + target_dir="${TARGET_DIR%/}" + target_dir="${target_dir:-.}" + echo "target_dir=$target_dir" >> "$GITHUB_OUTPUT" + + # `packer build` fails on a directory without templates, which would report a failure for an empty target. + if [ -z "$(find "$target_dir" -maxdepth 1 -type f \( -name '*.pkr.hcl' -o -name '*.pkr.json' \) -print -quit)" ]; then + echo "::notice::Skipping packer build: no Packer template files in $target_dir." + echo "skipped=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + packer version + + - name: Packer Init + id: init + if: steps.target.outputs.skipped != 'true' + uses: tedilabs/github-actions/.github/actions/shell.run@main + env: + PACKER_GITHUB_API_TOKEN: ${{ inputs.github_token }} + TARGET_DIR: ${{ steps.target.outputs.target_dir }} + with: + run: | + packer init "$TARGET_DIR" + + - name: Add Failure Details to Job Summary + id: init-summary + if: always() && steps.init.outcome == 'failure' + uses: tedilabs/github-actions/.github/actions/github.step-summary@main + with: + title: "❌ packer init · ${{ steps.target.outputs.target_dir }}" + file: ${{ steps.init.outputs.log_file }} + lang: text + + # A build runs for as long as the image takes, so the output is streamed with `tee` rather than captured + # through `shell.run`, which only prints it once the command has returned. + - name: Packer Build + id: build + if: steps.target.outputs.skipped != 'true' + shell: bash + env: + TARGET_DIR: ${{ steps.target.outputs.target_dir }} + VAR_FILES: ${{ inputs.var_files }} + ARGS: ${{ inputs.args }} + run: | + log="$RUNNER_TEMP/packer-build.log" + + args=(-color=false) + while IFS= read -r var_file; do + var_file="${var_file## }" + [ -n "$var_file" ] && args+=(-var-file="$var_file") + done <<< "$VAR_FILES" + read -ra extra_args <<< "$ARGS" + + set +e + packer build "${args[@]}" "${extra_args[@]}" "$TARGET_DIR" 2>&1 | tee "$log" + exit_code="${PIPESTATUS[0]}" + set -e + echo "exitcode=$exit_code" >> "$GITHUB_OUTPUT" + + # Packer closes with `==> Builds finished. The artifacts of successful builds are:` followed by one + # `--> : ...` line per artifact, and with `==> Some builds didn't complete successfully and had + # errors:` followed by one line per failed build. Either section may be absent, and a partly successful + # run prints both. + artifacts="$(sed -n '/^==> Builds finished\. The artifacts of successful builds are:/,$p' "$log" \ + | tail -n +2 \ + | sed '/^==> /d' \ + | sed -e :a -e '/^[[:space:]]*$/{$d;N;ba' -e '}')" + errors="$(sed -n "/^==> Some builds didn't complete successfully and had errors:/,\$p" "$log" \ + | tail -n +2 \ + | sed -n '/^==> /q;p' \ + | sed -e :a -e '/^[[:space:]]*$/{$d;N;ba' -e '}')" + + artifact_count=0 + if [ -n "$artifacts" ]; then + artifact_count="$(grep -c '^--> ' <<< "$artifacts" || true)" + fi + echo "artifact_count=$artifact_count" >> "$GITHUB_OUTPUT" + + fence() { + printf '```text\n%s\n```' "$1" + } + + if [ "$exit_code" -ne 0 ]; then + headline="Failed" + summary="> [!CAUTION]"$'\n'"> The build failed. See the job summary for the output." + if [ -n "$errors" ]; then + summary="$summary"$'\n\n'"$(fence "$errors")" + fi + if [ "$artifact_count" -gt 0 ]; then + summary="$summary"$'\n\n'"**$artifact_count** of the builds still produced an artifact."$'\n\n'"$(fence "$artifacts")" + fi + elif [ "$artifact_count" -eq 0 ]; then + headline="No artifacts" + summary="> [!NOTE]"$'\n'"> The build succeeded without producing an artifact." + else + headline="$artifact_count artifact(s)" + summary="**$artifact_count** artifact(s)."$'\n\n'"$(fence "$artifacts")" + fi + + { + echo "headline=$headline" + echo "artifacts<> "$GITHUB_OUTPUT" + + echo "$headline" + exit "$exit_code" + + - name: Add Failure Details to Job Summary + id: build-summary + if: always() && steps.build.outcome == 'failure' + uses: tedilabs/github-actions/.github/actions/github.step-summary@main + with: + title: "❌ packer build · ${{ steps.target.outputs.target_dir }}" + file: ${{ runner.temp }}/packer-build.log + lang: text diff --git a/.github/workflows/packer.templates.build.yaml b/.github/workflows/packer.templates.build.yaml new file mode 100644 index 0000000..1926f09 --- /dev/null +++ b/.github/workflows/packer.templates.build.yaml @@ -0,0 +1,228 @@ +name: Packer Templates - Build + + +on: + workflow_call: + inputs: + runs_on: + description: > + JSON-encoded runs-on value. + Examples: + - '"ubuntu-latest"' + - '["self-hosted","linux","x64"]' + required: false + type: string + default: '"ubuntu-latest"' + + scope: + type: string + required: false + default: auto + description: "(Optional) Which builds to run: `changed`, `all`, or `auto`. `auto` builds the changed directories on a `push` or a `pull_request` and everything on any other event, which is what makes one caller serve a push, a schedule, and a manual dispatch. Defaults to `auto`." + paths: + type: string + required: false + default: | + builds/** + description: "(Optional, `changed` scope) File and directory patterns used to detect changed build directories, one per line. Defaults to `builds/**`." + paths_max_depth: + type: string + required: false + default: "2" + description: "(Optional, `changed` scope) The maximum depth of the changed directories to resolve. For example, `builds/foo/source.pkr.hcl` with a max depth of `2` resolves to `builds/foo`. Defaults to `2`." + builds_dir: + type: string + required: false + default: builds + description: "(Optional, `all` scope) The directory whose immediate subdirectories are the build directories. Defaults to `builds`." + + vars_dir: + type: string + required: false + default: vars + description: "(Optional) The subdirectory of a build directory holding one variable file per variant, each of which becomes its own build. A build directory without it is built once with no `-var-file`. Defaults to `vars`." + vars_pattern: + type: string + required: false + default: "*.pkrvars.hcl" + description: "(Optional) The glob matched against `vars_dir` to find the variable files. Defaults to `*.pkrvars.hcl`." + + builds: + type: string + required: false + description: "(Optional) A comma-separated list of build directory names to keep (e.g. `ubuntu-2604,amazon-linux-2023`). Wire this to a `workflow_dispatch` input to build part of the repository by hand. Empty builds everything in scope." + variants: + type: string + required: false + description: "(Optional) A comma-separated list of variant names to keep (e.g. `prod`), matched against the variable file name without its extension. Empty builds every variant." + + build_args: + type: string + required: false + description: "(Optional) Additional arguments to pass to `packer build` (e.g. `-only=amazon-ebs.base`)." + max_parallel: + type: number + required: false + default: 4 + description: "(Optional) How many builds run at once. Images are built on real instances, so the default is deliberately below what the runner limits allow, to stay clear of cloud quotas on a full scheduled build. Defaults to `4`." + + aws_region: + type: string + required: false + description: "(Optional) The AWS region to configure before building. Only needed when the templates use an Amazon builder." + aws_github_oidc_iam_role: + type: string + required: false + description: "(Optional) The ARN of the IAM role to assume through GitHub OIDC before building. Requires the `id-token: write` permission on the caller. Leave empty when the builders of these templates need no AWS credentials." + + pr_number: + type: string + required: false + description: "(Optional) The pull request to post the report on. Needed on the events this workflow normally runs on, none of which carries a pull request context. Empty leaves the report in the job summary only." + pr_comment_enabled: + type: boolean + required: false + default: true + description: "(Optional) Whether to post the build report as a single sticky comment on `pr_number`, or on the pull request of the current event. Requires the `pull-requests: write` permission on the caller. The report is always written to the job summary. Defaults to `true`." + + +jobs: + targets: + name: Resolve Build Targets + runs-on: ${{ fromJson(inputs.runs_on) }} + + permissions: + contents: read + + env: + MODE: ${{ inputs.scope != 'auto' && inputs.scope || ((github.event_name == 'push' || github.event_name == 'pull_request') && 'changed' || 'all') }} + + outputs: + has_targets: ${{ steps.build-targets.outputs.has_targets }} + targets: ${{ steps.build-targets.outputs.targets }} + + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Get Changed Directories + id: changed-dirs + if: env.MODE == 'changed' + uses: tedilabs/github-actions/.github/actions/git.changed-dirs@main + with: + paths: ${{ inputs.paths }} + max_depth: ${{ inputs.paths_max_depth }} + + - name: Resolve Build Targets + id: build-targets + uses: tedilabs/github-actions/.github/actions/packer.build-targets@main + with: + mode: ${{ env.MODE }} + directories: ${{ steps.changed-dirs.outputs.directories }} + builds_dir: ${{ inputs.builds_dir }} + vars_dir: ${{ inputs.vars_dir }} + vars_pattern: ${{ inputs.vars_pattern }} + builds: ${{ inputs.builds }} + variants: ${{ inputs.variants }} + + + build: + name: Build (${{ matrix.name }}) + needs: + - targets + if: needs.targets.outputs.has_targets == 'true' + runs-on: ${{ fromJson(inputs.runs_on) }} + + permissions: + contents: read + id-token: write + + # Two builds of the same target must never overlap, so a scheduled full build and the build of a + # freshly merged change do not race each other into the same image name. + concurrency: + group: packer-build-${{ github.repository }}-${{ matrix.name }} + cancel-in-progress: false + + strategy: + fail-fast: false + max-parallel: ${{ inputs.max_parallel }} + matrix: + include: ${{ fromJson(needs.targets.outputs.targets) }} + + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v7 + + # The tools and their versions come from the repository's mise config (`mise.toml` or `.tool-versions`). + # The build directory is passed so a `mise.toml` placed there overrides the repository-wide one, + # and only `packer` is installed, so a repository that pins other tools does not pay to install them. + - name: Set up tools + id: setup-tools + uses: tedilabs/github-actions/.github/actions/mise.setup-tools@main + with: + working_directory: ${{ matrix.path }} + install_args: packer + + - name: Configure AWS Credentials + id: configure-aws + if: inputs.aws_github_oidc_iam_role != '' + uses: tedilabs/github-actions/.github/actions/aws.configure-credentials@main + with: + aws_region: ${{ inputs.aws_region }} + aws_github_oidc_iam_role: ${{ inputs.aws_github_oidc_iam_role }} + + - name: Build + id: build + continue-on-error: true + uses: tedilabs/github-actions/.github/actions/packer.build@main + with: + target_dir: ${{ matrix.path }} + var_files: ${{ matrix.var_file }} + args: ${{ inputs.build_args }} + + - name: Collect Results + id: results + if: always() + uses: tedilabs/github-actions/.github/actions/github.matrix-report@main + with: + mode: collect + id: ${{ matrix.name }} + id_label: Build + artifact_prefix: packer-build-report + job_status: ${{ job.status }} + headline: ${{ steps.build.outputs.headline }} + details: ${{ steps.build.outputs.summary }} + results: | + { + "build": "${{ steps.build.outcome }}" + } + + + report: + name: Report + needs: + - targets + - build + if: always() && needs.targets.outputs.has_targets == 'true' + runs-on: ${{ fromJson(inputs.runs_on) }} + + permissions: + contents: read + pull-requests: write + + steps: + - name: Publish Report + id: report + uses: tedilabs/github-actions/.github/actions/github.matrix-report@main + with: + mode: publish + id_label: Build + artifact_prefix: packer-build-report + title: Packer Build + pr_number: ${{ inputs.pr_number }} + pr_comment_enabled: ${{ inputs.pr_comment_enabled }} + pr_comment_marker: packer-build