ci(release): stage and prod catch up on every version dev released before

.github/workflows/release.yml

@@ -7,7 +7,8 @@ name: Release
 #          pre-release, and publishes that tarball to GitHub Packages (dist-tag `dev`), then to
 #          dev's CodeArtifact — npm.beplus.cloud, where the estates' builds install `be` from.
 #   stage  Builds nothing: fetches the tarball dev published from GitHub Packages and publishes it
-#          byte for byte to stage's CodeArtifact. GitHub Packages tags it `stage`.
+#          byte for byte to stage's CodeArtifact, with every earlier version dev released that
+#          stage's CodeArtifact lacks. GitHub Packages tags it `stage`.
 #   prod   The same into prod's CodeArtifact (GitHub Packages tags it `latest`), and the GitHub
 #          Release becomes a full release, marked latest.
 #   main   The public channel, and the one thing `be` has that the libraries do not: the same
@@ -190,6 +191,14 @@ jobs:
         with:
           BE_CLI_VERSION: ${{ vars.BE_CLI_VERSION }}
 
+      # A fast-forward promotes the commits in between too: every version dev released before this
+      # one that this CodeArtifact lacks goes out first, oldest first, without touching `latest`.
+      - name: Catch up on the versions before it
+        id: catch-up
+        run: scripts/release/catch-up-codeartifact.sh "${VERSION}"
+        env:
+          NODE_AUTH_TOKEN: ${{ github.token }}
+
       - name: Publish to CodeArtifact
         run: scripts/release/publish-codeartifact.sh ./npm/*.tgz
 
@@ -206,23 +215,38 @@ jobs:
           NODE_AUTH_TOKEN: ${{ github.token }}
           PROMOTED_TAG: ${{ github.ref_name == 'prod' && 'latest' || github.ref_name }}
 
-      - name: Make the GitHub release a full release
+      # The earlier versions prod has become full releases too, never marked latest, so a release
+      # an earlier promotion skipped (or left a pre-release) is fixed; this version's goes last.
+      - name: Make the GitHub releases full releases
         if: github.ref_name == 'prod'
         run: |
-          if [[ "$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isPrerelease --jq .isPrerelease)" == "true" ]]; then
+          prereleases="$(gh release list --repo "${GITHUB_REPOSITORY}" --exclude-drafts --limit 1000 \
+            --json tagName,isPrerelease --jq '.[] | select(.isPrerelease) | .tagName')"
+          for version in ${EARLIER}; do
+            if grep -qxF "v${version}" <<< "${prereleases}"; then
+              gh release edit "v${version}" --repo "${GITHUB_REPOSITORY}" --prerelease=false --latest=false
+            fi
+          done
+          if grep -qxF "${TAG}" <<< "${prereleases}"; then
             if [[ "${DIST_TAG}" == "latest" ]]; then latest=(--latest); else latest=(--latest=false); fi
             gh release edit "${TAG}" --repo "${GITHUB_REPOSITORY}" --prerelease=false "${latest[@]}"
           fi
         env:
           GH_TOKEN: ${{ github.token }}
+          EARLIER: ${{ steps.catch-up.outputs.earlier }}
 
       - name: Summary
         run: |
           {
             echo "### Promoted ${TAG} to ${GITHUB_REF_NAME}"
             echo
             echo "- ${GITHUB_REF_NAME}'s CodeArtifact: dist-tag \`${DIST_TAG}\`, the bytes dev published"
+            if [[ -n "${CAUGHT_UP}" ]]; then
+              echo "- caught up on the versions dev released before it: v${CAUGHT_UP// /, v}"
+            fi
           } >> "${GITHUB_STEP_SUMMARY}"
+        env:
+          CAUGHT_UP: ${{ steps.catch-up.outputs.caught_up }}
 
   # main only: the public registry. Its own job because npm's trusted publisher is bound to the
   # `npm` environment, whose deployment branch is main.

docs/releasing.md

@@ -9,8 +9,8 @@ fast-forwarded from it and build nothing. Nobody publishes by hand. The
 | branch | what a push does |
 | --- | --- |
 | `dev` | packs the version **once**, tags it `vX.Y.Z` with a GitHub **pre-release**, publishes the tarball to **GitHub Packages** (dist-tag `dev`), then to **dev's CodeArtifact** — `npm.beplus.cloud`, where the estates' builds install `be` from |
-| `stage` | fetches that tarball from GitHub Packages and publishes it, byte for byte, to **stage's CodeArtifact**; GitHub Packages tags it `stage` |
-| `prod` | the same into **prod's CodeArtifact** (GitHub Packages tags it `latest`), and the GitHub Release becomes a **full release**, marked latest |
+| `stage` | fetches that tarball from GitHub Packages and publishes it, byte for byte, to **stage's CodeArtifact**, with every earlier version dev released that it lacks; GitHub Packages tags it `stage` |
+| `prod` | the same into **prod's CodeArtifact** (GitHub Packages tags it `latest`), and the GitHub Release becomes a **full release**, marked latest (the earlier versions' too, never marked latest) |
 | `main` | the public channel, and the one thing `be` has that the libraries don't: the same bytes to the **public npm registry**, with provenance. `main` publishes only what `prod` already has |
 
 ## The flow
@@ -36,6 +36,16 @@ prod  ──fast-forward──▶  main  ──▶  public npm
    public, `main` to `prod` (`git push origin origin/prod:main`). Never commit to `stage`, `prod`
    or `main` directly.
 
+You don't have to promote every release. A fast-forward promotes the commits in between too, so
+`stage` and `prod` also publish every earlier version dev released that their CodeArtifact lacks
+([`catch-up-codeartifact.sh`](../scripts/release/catch-up-codeartifact.sh)): oldest first, under a
+temporary `catch-up` dist-tag, so `latest` is only ever the version the branch is on. Removing
+that tag needs `codeartifact:PutPackageMetadata` on the environment's `npm-publishing` role; a tag
+a run could not remove goes on the next one. Each
+CodeArtifact keeps every version released up to its branch, and an estate that pinned one on dev
+installs it there too. A version the registry refuses (one archived there on purpose) is a
+warning, and the next promotion tries again. `main` publishes only the version its commit carries.
+
 A PR that doesn't touch `bin/be` (docs, tests, CI) can merge without a bump, and then releases
 nothing. A PR that changes `bin/be` must bump. `scripts/install.sh` installs from `main`, so
 whatever reaches `main` is a released, public version.

scripts/release/catch-up-codeartifact.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+#
+# catch-up-codeartifact.sh <version> — publish to this environment's CodeArtifact ($BE_ENVIRONMENT's)
+# every version of @beplus/be before <version> that dev released and the CodeArtifact lacks.
+#
+# A fast-forward promotes every commit up to the one it lands on, and dev may have released several
+# versions since the last promotion: stage and prod get each of them, not only the newest, so an
+# estate that pinned one on dev installs it there too. "Released" is in GitHub Packages, and each is
+# the tarball dev published there, byte for byte. "Before" is SemVer: dev releases one version at a
+# time, so a lower version came from a commit before this one; dev's newer versions wait for their
+# own promotion.
+#
+# Oldest first, under a `catch-up` dist-tag that is removed afterwards, so `latest` stays where
+# publish-codeartifact.sh puts it; so is one an earlier run left behind. Removing a dist-tag needs
+# codeartifact:PutPackageMetadata on the environment's npm-publishing role. A version the registry
+# refuses (one archived there on purpose) is a warning, and the next promotion tries it again.
+#
+# Writes caught_up (the versions published now) and earlier (every version before <version> the
+# CodeArtifact has now), space-separated, as key=value lines to $GITHUB_OUTPUT (stdout when unset).
+# Runs after the job assumed the environment's npm-publishing role and installed the beplus CLI;
+# GitHub Packages is read with $NODE_AUTH_TOKEN.
+
+set -euo pipefail
+
+readonly version="${1:?usage: catch-up-codeartifact.sh <version>}"
+readonly name="@beplus/be"
+readonly github="https://npm.pkg.github.com/"
+
+auth="$(beplus aws codeartifact auth --env "${BE_ENVIRONMENT:?BE_ENVIRONMENT is not set}" --json)"
+registry="https://$(jq -r '.data.registry // empty' <<< "${auth}")"
+if [[ "${registry}" == "https://" ]]; then
+  echo "::error::beplus aws codeartifact auth --env ${BE_ENVIRONMENT} named no registry: ${auth}"
+  exit 1
+fi
+
+# The token stays in the environment: npm expands ${NODE_AUTH_TOKEN} itself.
+temp="${RUNNER_TEMP:-$(mktemp -d)}"
+npmrc="${temp}/.npmrc-github"
+# shellcheck disable=SC2016 # npm expands it, not the shell
+echo '//npm.pkg.github.com/:_authToken=${NODE_AUTH_TOKEN}' > "${npmrc}"
+
+#
+# versions <where> <npm args...> — every version of @beplus/be a registry has, one a line; nothing
+# when it has none. Any other answer stops the run.
+#
+
+function versions() {
+  local where="$1" out err
+  shift
+  err="$(mktemp)"
+  if out="$(npm view "${name}" versions --json "$@" 2> "${err}")"; then
+    rm -f "${err}"
+    node -e 'console.log([].concat(JSON.parse(process.argv[1] || "[]")).join("\n"))' "${out}"
+  elif grep -q E404 <<< "${out}$(cat "${err}")"; then
+    rm -f "${err}"
+  else
+    echo "::error::Cannot list the versions of ${name} in ${where}: ${out}$(cat "${err}")" >&2
+    rm -f "${err}"
+    return 1
+  fi
+}
+
+released="$(versions "GitHub Packages" --userconfig "${npmrc}" --@beplus:registry="${github}")"
+here="$(versions "${BE_ENVIRONMENT}'s CodeArtifact" --registry="${registry}" --@beplus:registry="${registry}")"
+# A catch-up tag an earlier run could not remove (no PutPackageMetadata, or cancelled), if any.
+stale="$(npm view "${name}" dist-tags.catch-up --registry="${registry}" --@beplus:registry="${registry}" 2> /dev/null || true)"
+
+# npm's own copy of semver, which every npm carries: nothing is installed for this.
+# shellcheck disable=SC2016 # ${…} is a JavaScript template literal, not a shell expansion
+earlier="$(node -e '
+  const semver = require(`${process.argv[1]}/npm/node_modules/semver`);
+  const released = require("fs").readFileSync(0, "utf8").split("\n").filter(Boolean);
+  console.log(semver.sort(released.filter((v) => semver.lt(v, process.argv[2]))).join("\n"));
+' "$(npm root -g)" "${version}" <<< "${released}")"
+
+dir="$(mktemp -d)"
+caught_up=()
+landed=()
+refused=()
+while read -r v; do
+  [[ -n "${v}" ]] || continue
+  if grep -qxF "${v}" <<< "${here}"; then
+    landed+=("${v}")
+    continue
+  fi
+  echo "Publishing ${name}@${v} to ${BE_ENVIRONMENT}'s CodeArtifact (dist-tag catch-up)"
+  if file="$(npm pack "${name}@${v}" --pack-destination "${dir}" --userconfig "${npmrc}" --@beplus:registry="${github}" | tail -n 1)" \
+    && npm publish "${dir}/${file}" --tag catch-up --registry="${registry}" --@beplus:registry="${registry}"; then
+    caught_up+=("${v}")
+    landed+=("${v}")
+  else
+    refused+=("${v}")
+    echo "::warning::Could not publish ${name}@${v} to ${BE_ENVIRONMENT}'s CodeArtifact. The next promotion tries again."
+  fi
+done <<< "${earlier}"
+
+if [[ "${#caught_up[@]}" -gt 0 ]]; then
+  echo "Caught up on ${caught_up[*]}."
+elif [[ "${#refused[@]}" -eq 0 ]]; then
+  echo "${BE_ENVIRONMENT}'s CodeArtifact already has every version dev released before ${version}."
+fi
+
+if [[ "${#caught_up[@]}" -gt 0 || -n "${stale}" ]]; then
+  [[ -z "${stale}" ]] || echo "Removing the catch-up dist-tag an earlier run left on ${stale}."
+  npm dist-tag rm "${name}" catch-up --registry="${registry}" --@beplus:registry="${registry}" > /dev/null \
+    || echo "::warning::Could not remove the catch-up dist-tag from ${name} in ${BE_ENVIRONMENT}'s CodeArtifact: the role needs codeartifact:PutPackageMetadata. The next promotion tries again."
+fi
+
+{
+  echo "caught_up=${caught_up[*]}"
+  echo "earlier=${landed[*]}"
+} >> "${GITHUB_OUTPUT:-/dev/stdout}"