From 659c7b94cfa023da5b5594765740099e72248e2c Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Thu, 12 Nov 2026 17:59:43 -0500 Subject: [PATCH 1/2] ci: cap build jobs at 3 hours and retry the sbom commit on a moved main Nineteen build jobs last month ran to GitHub's 6-hour limit; none of those repos builds in more than 41 minutes, they had hung. A new build_timeout input (default 180, the slowest real build is about 150) cancels them at 3 hours and applies to build-linux too. commit-sbom committed and then rebased, and failed 17 times a month when another commit landed on sbom.json first. It now keeps this run's files aside, resets to the current tip, lays them on top and pushes, retrying up to five times. The newest build's SBOM is the truth, there is nothing to merge. When only the generated timestamp changed the SBOM is left alone and just the generated files are committed. --- .github/workflows/daemonless-build.yaml | 61 ++++++++++++++++++------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/.github/workflows/daemonless-build.yaml b/.github/workflows/daemonless-build.yaml index 26d53ba..0557570 100644 --- a/.github/workflows/daemonless-build.yaml +++ b/.github/workflows/daemonless-build.yaml @@ -52,6 +52,11 @@ on: required: false type: boolean default: false + build_timeout: + description: 'Minutes before a build job is cancelled. A hung VM otherwise runs to the 6-hour cap; the slowest real build in the fleet takes about 150.' + required: false + type: number + default: 180 env: REGISTRY: ghcr.io @@ -141,6 +146,7 @@ jobs: needs: detect if: ${{ needs.detect.outputs.matrix != '' && needs.detect.outputs.compose_only != 'true' }} runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.build_timeout }} permissions: contents: read packages: write @@ -447,6 +453,7 @@ jobs: needs: detect if: ${{ inputs.linux && needs.detect.outputs.compose_only != 'true' }} runs-on: ${{ matrix.runner }} + timeout-minutes: ${{ inputs.build_timeout }} permissions: contents: read packages: write @@ -690,28 +697,48 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - # `generated` changes every run — only commit when the versions or - # package lists actually changed, so no-op rebuilds stay silent. - git show HEAD:sbom.json 2>/dev/null | jq -S 'del(.generated)' > /tmp/sbom-old.json || echo '{}' > /tmp/sbom-old.json - jq -S 'del(.generated)' sbom.json > /tmp/sbom-new.json - SBOM_CHANGED=1 - cmp -s /tmp/sbom-old.json /tmp/sbom-new.json && SBOM_CHANGED=0 - - git add sbom.json + # Keep this run's files aside: main moves while the build runs (the + # aarch64 leg and overlapping runs commit too), so each attempt + # starts from the current tip and lays our files on top. The newest + # build's SBOM is the truth; there is nothing to merge. + FILES="sbom.json" for f in ${{ steps.generate.outputs.files }}; do - [ -f "$f" ] && git add "$f" + [ -f "$f" ] && FILES="$FILES $f" + done + mkdir -p /tmp/ours + for f in $FILES; do + mkdir -p "/tmp/ours/$(dirname "$f")" && cp "$f" "/tmp/ours/$f" done - - if [ "$SBOM_CHANGED" = 0 ] && git diff --cached --quiet; then - echo "No content change to sbom.json or generated files" - exit 0 - fi git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git commit -m "Update sbom.json and generated files [skip ci]" - git pull --rebase origin main - git push + for attempt in 1 2 3 4 5; do + git fetch -q origin main + git reset -q --hard origin/main + for f in $FILES; do cp "/tmp/ours/$f" "$f"; done + + # `generated` changes every run; only commit when the versions or + # package lists actually changed, so no-op rebuilds stay silent. + git show HEAD:sbom.json 2>/dev/null | jq -S 'del(.generated)' > /tmp/sbom-old.json || echo '{}' > /tmp/sbom-old.json + jq -S 'del(.generated)' sbom.json > /tmp/sbom-new.json + if cmp -s /tmp/sbom-old.json /tmp/sbom-new.json; then + git checkout -q -- sbom.json + fi + git add $FILES + if git diff --cached --quiet; then + echo "No content change to sbom.json or generated files" + exit 0 + fi + + git commit -q -m "Update sbom.json and generated files [skip ci]" + if git push; then + exit 0 + fi + echo "Push rejected (attempt $attempt); main moved, retrying" + sleep $((attempt * 10)) + done + echo "Could not push after 5 attempts" + exit 1 # Sign the pushed images, and sign the SBOMs onto them as cosign # attestations. Runs on Linux (keyless needs the GitHub OIDC token) against From 27eb5214613830e991b41bfa34c3ed3fcf80391b Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Thu, 12 Nov 2026 18:20:14 -0500 Subject: [PATCH 2/2] ci: ignore the per-tag timestamps in the sbom no-op check Each tag in sbom.json carries its own generated timestamp, so the comparison that was meant to keep no-op rebuilds silent never matched and every run committed two timestamps. --- .github/workflows/daemonless-build.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/daemonless-build.yaml b/.github/workflows/daemonless-build.yaml index 0557570..facc4fa 100644 --- a/.github/workflows/daemonless-build.yaml +++ b/.github/workflows/daemonless-build.yaml @@ -717,10 +717,11 @@ jobs: git reset -q --hard origin/main for f in $FILES; do cp "/tmp/ours/$f" "$f"; done - # `generated` changes every run; only commit when the versions or - # package lists actually changed, so no-op rebuilds stay silent. - git show HEAD:sbom.json 2>/dev/null | jq -S 'del(.generated)' > /tmp/sbom-old.json || echo '{}' > /tmp/sbom-old.json - jq -S 'del(.generated)' sbom.json > /tmp/sbom-new.json + # The top-level and per-tag `generated` timestamps change every + # run; only commit when the versions or package lists actually + # changed, so no-op rebuilds stay silent. + git show HEAD:sbom.json 2>/dev/null | jq -S 'del(.generated, .tags[].generated)' > /tmp/sbom-old.json || echo '{}' > /tmp/sbom-old.json + jq -S 'del(.generated, .tags[].generated)' sbom.json > /tmp/sbom-new.json if cmp -s /tmp/sbom-old.json /tmp/sbom-new.json; then git checkout -q -- sbom.json fi