From 83c6e92f36431d1aabea7ce4656bdd326da727d5 Mon Sep 17 00:00:00 2001 From: bea-claude Date: Thu, 8 Oct 2026 17:05:09 +0200 Subject: [PATCH 1/2] ci(release): stage and prod catch up on every version dev released before A fast-forward promotes the commits in between too, but stage and prod published only the version the commit carries: release twice on dev, promote once, and the first never reached stage's or prod's CodeArtifact, so an estate that pinned it on dev could not install it there. scripts/release/catch-up-codeartifact.sh publishes, before the commit's own version, every earlier version dev released (in GitHub Packages, lower by SemVer) that the environment's CodeArtifact lacks: the bytes dev published, oldest first, under a temporary `catch-up` dist-tag so `latest` never moves back. A version the registry refuses is a warning the next promotion retries. prod makes their GitHub Releases full releases, never marked latest. main is unchanged. The release workflow stays this repository's own; it follows what beplus/setup-beplus's library-publish does for the libraries. Co-Authored-By: Igor Lamos --- .github/workflows/release.yml | 30 ++++++- docs/releasing.md | 12 ++- scripts/release/catch-up-codeartifact.sh | 105 +++++++++++++++++++++++ 3 files changed, 142 insertions(+), 5 deletions(-) create mode 100755 scripts/release/catch-up-codeartifact.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e185914..19b6be5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,8 @@ name: Release # pre-release, and publishes that tarball to GitHub Packages (dist-tag `dev`), then to # dev's CodeArtifact — npm.beplus.cloud, where the estates' builds install `be` from. # stage Builds nothing: fetches the tarball dev published from GitHub Packages and publishes it -# byte for byte to stage's CodeArtifact. GitHub Packages tags it `stage`. +# byte for byte to stage's CodeArtifact, with every earlier version dev released that +# stage's CodeArtifact lacks. GitHub Packages tags it `stage`. # prod The same into prod's CodeArtifact (GitHub Packages tags it `latest`), and the GitHub # Release becomes a full release, marked latest. # main The public channel, and the one thing `be` has that the libraries do not: the same @@ -190,6 +191,14 @@ jobs: with: BE_CLI_VERSION: ${{ vars.BE_CLI_VERSION }} + # A fast-forward promotes the commits in between too: every version dev released before this + # one that this CodeArtifact lacks goes out first, oldest first, without touching `latest`. + - name: Catch up on the versions before it + id: catch-up + run: scripts/release/catch-up-codeartifact.sh "${VERSION}" + env: + NODE_AUTH_TOKEN: ${{ github.token }} + - name: Publish to CodeArtifact run: scripts/release/publish-codeartifact.sh ./npm/*.tgz @@ -206,15 +215,25 @@ jobs: NODE_AUTH_TOKEN: ${{ github.token }} PROMOTED_TAG: ${{ github.ref_name == 'prod' && 'latest' || github.ref_name }} - - name: Make the GitHub release a full release + # The earlier versions prod has become full releases too, never marked latest, so a release + # an earlier promotion skipped (or left a pre-release) is fixed; this version's goes last. + - name: Make the GitHub releases full releases if: github.ref_name == 'prod' run: | - if [[ "$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isPrerelease --jq .isPrerelease)" == "true" ]]; then + prereleases="$(gh release list --repo "${GITHUB_REPOSITORY}" --exclude-drafts --limit 1000 \ + --json tagName,isPrerelease --jq '.[] | select(.isPrerelease) | .tagName')" + for version in ${EARLIER}; do + if grep -qxF "v${version}" <<< "${prereleases}"; then + gh release edit "v${version}" --repo "${GITHUB_REPOSITORY}" --prerelease=false --latest=false + fi + done + if grep -qxF "${TAG}" <<< "${prereleases}"; then if [[ "${DIST_TAG}" == "latest" ]]; then latest=(--latest); else latest=(--latest=false); fi gh release edit "${TAG}" --repo "${GITHUB_REPOSITORY}" --prerelease=false "${latest[@]}" fi env: GH_TOKEN: ${{ github.token }} + EARLIER: ${{ steps.catch-up.outputs.earlier }} - name: Summary run: | @@ -222,7 +241,12 @@ jobs: echo "### Promoted ${TAG} to ${GITHUB_REF_NAME}" echo echo "- ${GITHUB_REF_NAME}'s CodeArtifact: dist-tag \`${DIST_TAG}\`, the bytes dev published" + if [[ -n "${CAUGHT_UP}" ]]; then + echo "- caught up on the versions dev released before it: v${CAUGHT_UP// /, v}" + fi } >> "${GITHUB_STEP_SUMMARY}" + env: + CAUGHT_UP: ${{ steps.catch-up.outputs.caught_up }} # main only: the public registry. Its own job because npm's trusted publisher is bound to the # `npm` environment, whose deployment branch is main. diff --git a/docs/releasing.md b/docs/releasing.md index f34c09d..b1c64d8 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -9,8 +9,8 @@ fast-forwarded from it and build nothing. Nobody publishes by hand. The | branch | what a push does | | --- | --- | | `dev` | packs the version **once**, tags it `vX.Y.Z` with a GitHub **pre-release**, publishes the tarball to **GitHub Packages** (dist-tag `dev`), then to **dev's CodeArtifact** — `npm.beplus.cloud`, where the estates' builds install `be` from | -| `stage` | fetches that tarball from GitHub Packages and publishes it, byte for byte, to **stage's CodeArtifact**; GitHub Packages tags it `stage` | -| `prod` | the same into **prod's CodeArtifact** (GitHub Packages tags it `latest`), and the GitHub Release becomes a **full release**, marked latest | +| `stage` | fetches that tarball from GitHub Packages and publishes it, byte for byte, to **stage's CodeArtifact**, with every earlier version dev released that it lacks; GitHub Packages tags it `stage` | +| `prod` | the same into **prod's CodeArtifact** (GitHub Packages tags it `latest`), and the GitHub Release becomes a **full release**, marked latest (the earlier versions' too, never marked latest) | | `main` | the public channel, and the one thing `be` has that the libraries don't: the same bytes to the **public npm registry**, with provenance. `main` publishes only what `prod` already has | ## The flow @@ -36,6 +36,14 @@ prod ──fast-forward──▶ main ──▶ public npm public, `main` to `prod` (`git push origin origin/prod:main`). Never commit to `stage`, `prod` or `main` directly. +You don't have to promote every release. A fast-forward promotes the commits in between too, so +`stage` and `prod` also publish every earlier version dev released that their CodeArtifact lacks +([`catch-up-codeartifact.sh`](../scripts/release/catch-up-codeartifact.sh)): oldest first, under a +temporary `catch-up` dist-tag, so `latest` is only ever the version the branch is on. Each +CodeArtifact keeps every version released up to its branch, and an estate that pinned one on dev +installs it there too. A version the registry refuses (one archived there on purpose) is a +warning, and the next promotion tries again. `main` publishes only the version its commit carries. + A PR that doesn't touch `bin/be` (docs, tests, CI) can merge without a bump, and then releases nothing. A PR that changes `bin/be` must bump. `scripts/install.sh` installs from `main`, so whatever reaches `main` is a released, public version. diff --git a/scripts/release/catch-up-codeartifact.sh b/scripts/release/catch-up-codeartifact.sh new file mode 100755 index 0000000..1f3fbcf --- /dev/null +++ b/scripts/release/catch-up-codeartifact.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# +# catch-up-codeartifact.sh — publish to this environment's CodeArtifact ($BE_ENVIRONMENT's) +# every version of @beplus/be before that dev released and the CodeArtifact lacks. +# +# A fast-forward promotes every commit up to the one it lands on, and dev may have released several +# versions since the last promotion: stage and prod get each of them, not only the newest, so an +# estate that pinned one on dev installs it there too. "Released" is in GitHub Packages, and each is +# the tarball dev published there, byte for byte. "Before" is SemVer: dev releases one version at a +# time, so a lower version came from a commit before this one; dev's newer versions wait for their +# own promotion. +# +# Oldest first, under a `catch-up` dist-tag that is removed afterwards, so `latest` stays where +# publish-codeartifact.sh puts it. A version the registry refuses (one archived there on purpose) is +# a warning, and the next promotion tries it again. +# +# Writes caught_up (the versions published now) and earlier (every version before the +# CodeArtifact has now), space-separated, as key=value lines to $GITHUB_OUTPUT (stdout when unset). +# Runs after the job assumed the environment's npm-publishing role and installed the beplus CLI; +# GitHub Packages is read with $NODE_AUTH_TOKEN. + +set -euo pipefail + +readonly version="${1:?usage: catch-up-codeartifact.sh }" +readonly name="@beplus/be" +readonly github="https://npm.pkg.github.com/" + +auth="$(beplus aws codeartifact auth --env "${BE_ENVIRONMENT:?BE_ENVIRONMENT is not set}" --json)" +registry="https://$(jq -r '.data.registry // empty' <<< "${auth}")" +if [[ "${registry}" == "https://" ]]; then + echo "::error::beplus aws codeartifact auth --env ${BE_ENVIRONMENT} named no registry: ${auth}" + exit 1 +fi + +# The token stays in the environment: npm expands ${NODE_AUTH_TOKEN} itself. +temp="${RUNNER_TEMP:-$(mktemp -d)}" +npmrc="${temp}/.npmrc-github" +# shellcheck disable=SC2016 # npm expands it, not the shell +echo '//npm.pkg.github.com/:_authToken=${NODE_AUTH_TOKEN}' > "${npmrc}" + +# +# versions — every version of @beplus/be a registry has, one a line; nothing +# when it has none. Any other answer stops the run. +# + +function versions() { + local where="$1" out err + shift + err="$(mktemp)" + if out="$(npm view "${name}" versions --json "$@" 2> "${err}")"; then + rm -f "${err}" + node -e 'console.log([].concat(JSON.parse(process.argv[1] || "[]")).join("\n"))' "${out}" + elif grep -q E404 <<< "${out}$(cat "${err}")"; then + rm -f "${err}" + else + echo "::error::Cannot list the versions of ${name} in ${where}: ${out}$(cat "${err}")" >&2 + rm -f "${err}" + return 1 + fi +} + +released="$(versions "GitHub Packages" --userconfig "${npmrc}" --@beplus:registry="${github}")" +here="$(versions "${BE_ENVIRONMENT}'s CodeArtifact" --registry="${registry}" --@beplus:registry="${registry}")" + +# npm's own copy of semver, which every npm carries: nothing is installed for this. +# shellcheck disable=SC2016 # ${…} is a JavaScript template literal, not a shell expansion +earlier="$(node -e ' + const semver = require(`${process.argv[1]}/npm/node_modules/semver`); + const released = require("fs").readFileSync(0, "utf8").split("\n").filter(Boolean); + console.log(semver.sort(released.filter((v) => semver.lt(v, process.argv[2]))).join("\n")); +' "$(npm root -g)" "${version}" <<< "${released}")" + +dir="$(mktemp -d)" +caught_up=() +landed=() +refused=() +while read -r v; do + [[ -n "${v}" ]] || continue + if grep -qxF "${v}" <<< "${here}"; then + landed+=("${v}") + continue + fi + echo "Publishing ${name}@${v} to ${BE_ENVIRONMENT}'s CodeArtifact (dist-tag catch-up)" + if file="$(npm pack "${name}@${v}" --pack-destination "${dir}" --userconfig "${npmrc}" --@beplus:registry="${github}" | tail -n 1)" \ + && npm publish "${dir}/${file}" --tag catch-up --registry="${registry}" --@beplus:registry="${registry}"; then + caught_up+=("${v}") + landed+=("${v}") + else + refused+=("${v}") + echo "::warning::Could not publish ${name}@${v} to ${BE_ENVIRONMENT}'s CodeArtifact. The next promotion tries again." + fi +done <<< "${earlier}" + +if [[ "${#caught_up[@]}" -gt 0 ]]; then + npm dist-tag rm "${name}" catch-up --registry="${registry}" --@beplus:registry="${registry}" > /dev/null \ + || echo "::warning::Could not remove the catch-up dist-tag from ${name} in ${BE_ENVIRONMENT}'s CodeArtifact." + echo "Caught up on ${caught_up[*]}." +elif [[ "${#refused[@]}" -eq 0 ]]; then + echo "${BE_ENVIRONMENT}'s CodeArtifact already has every version dev released before ${version}." +fi + +{ + echo "caught_up=${caught_up[*]}" + echo "earlier=${landed[*]}" +} >> "${GITHUB_OUTPUT:-/dev/stdout}" From 94419e246c2a7ae6ca6a272a2c03ae278a7bce81 Mon Sep 17 00:00:00 2001 From: bea-claude Date: Thu, 8 Oct 2026 18:02:30 +0200 Subject: [PATCH 2/2] ci(release): a promotion removes a catch-up tag an earlier run left behind Removing a dist-tag needs codeartifact:PutPackageMetadata, which the npm-publishing roles lack (beplus/ai's first stage promotion left its tag), so a run that caught up could leave `catch-up` behind, and the next one, with nothing to catch up, did not look for it. The script now reads the tag up front and removes it whether this run set it or not; the warning names the missing permission. Co-Authored-By: Igor Lamos --- docs/releasing.md | 4 +++- scripts/release/catch-up-codeartifact.sh | 15 +++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/releasing.md b/docs/releasing.md index b1c64d8..102eb10 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -39,7 +39,9 @@ prod ──fast-forward──▶ main ──▶ public npm You don't have to promote every release. A fast-forward promotes the commits in between too, so `stage` and `prod` also publish every earlier version dev released that their CodeArtifact lacks ([`catch-up-codeartifact.sh`](../scripts/release/catch-up-codeartifact.sh)): oldest first, under a -temporary `catch-up` dist-tag, so `latest` is only ever the version the branch is on. Each +temporary `catch-up` dist-tag, so `latest` is only ever the version the branch is on. Removing +that tag needs `codeartifact:PutPackageMetadata` on the environment's `npm-publishing` role; a tag +a run could not remove goes on the next one. Each CodeArtifact keeps every version released up to its branch, and an estate that pinned one on dev installs it there too. A version the registry refuses (one archived there on purpose) is a warning, and the next promotion tries again. `main` publishes only the version its commit carries. diff --git a/scripts/release/catch-up-codeartifact.sh b/scripts/release/catch-up-codeartifact.sh index 1f3fbcf..9f11215 100755 --- a/scripts/release/catch-up-codeartifact.sh +++ b/scripts/release/catch-up-codeartifact.sh @@ -11,8 +11,9 @@ # own promotion. # # Oldest first, under a `catch-up` dist-tag that is removed afterwards, so `latest` stays where -# publish-codeartifact.sh puts it. A version the registry refuses (one archived there on purpose) is -# a warning, and the next promotion tries it again. +# publish-codeartifact.sh puts it; so is one an earlier run left behind. Removing a dist-tag needs +# codeartifact:PutPackageMetadata on the environment's npm-publishing role. A version the registry +# refuses (one archived there on purpose) is a warning, and the next promotion tries it again. # # Writes caught_up (the versions published now) and earlier (every version before the # CodeArtifact has now), space-separated, as key=value lines to $GITHUB_OUTPUT (stdout when unset). @@ -61,6 +62,8 @@ function versions() { released="$(versions "GitHub Packages" --userconfig "${npmrc}" --@beplus:registry="${github}")" here="$(versions "${BE_ENVIRONMENT}'s CodeArtifact" --registry="${registry}" --@beplus:registry="${registry}")" +# A catch-up tag an earlier run could not remove (no PutPackageMetadata, or cancelled), if any. +stale="$(npm view "${name}" dist-tags.catch-up --registry="${registry}" --@beplus:registry="${registry}" 2> /dev/null || true)" # npm's own copy of semver, which every npm carries: nothing is installed for this. # shellcheck disable=SC2016 # ${…} is a JavaScript template literal, not a shell expansion @@ -92,13 +95,17 @@ while read -r v; do done <<< "${earlier}" if [[ "${#caught_up[@]}" -gt 0 ]]; then - npm dist-tag rm "${name}" catch-up --registry="${registry}" --@beplus:registry="${registry}" > /dev/null \ - || echo "::warning::Could not remove the catch-up dist-tag from ${name} in ${BE_ENVIRONMENT}'s CodeArtifact." echo "Caught up on ${caught_up[*]}." elif [[ "${#refused[@]}" -eq 0 ]]; then echo "${BE_ENVIRONMENT}'s CodeArtifact already has every version dev released before ${version}." fi +if [[ "${#caught_up[@]}" -gt 0 || -n "${stale}" ]]; then + [[ -z "${stale}" ]] || echo "Removing the catch-up dist-tag an earlier run left on ${stale}." + npm dist-tag rm "${name}" catch-up --registry="${registry}" --@beplus:registry="${registry}" > /dev/null \ + || echo "::warning::Could not remove the catch-up dist-tag from ${name} in ${BE_ENVIRONMENT}'s CodeArtifact: the role needs codeartifact:PutPackageMetadata. The next promotion tries again." +fi + { echo "caught_up=${caught_up[*]}" echo "earlier=${landed[*]}"