diff --git a/.github/workflows/apps-web-check.yml b/.github/workflows/apps-web-check.yml index 5e1751b297b..2ec658ce1e8 100644 --- a/.github/workflows/apps-web-check.yml +++ b/.github/workflows/apps-web-check.yml @@ -24,9 +24,13 @@ jobs: run: working-directory: apps/web steps: + # The change detector compares two snapshots, so only the tested commit + # is checked out and the compared commit is fetched below. Fetching every + # branch and tag in full exhausted this job's budget before any check + # ran (#8559). - uses: actions/checkout@v7.0.1 with: - fetch-depth: 0 + fetch-depth: 1 filter: blob:none sparse-checkout: | apps/web @@ -39,20 +43,50 @@ jobs: EVENT_NAME: ${{ github.event_name }} BASE_REF_NAME: ${{ github.base_ref }} BEFORE_SHA: ${{ github.event.before }} + # actions/checkout retries its fetches; this one replaces part of it. + FETCH_ATTEMPTS: "3" + FETCH_RETRY_DELAY_SECONDS: "5" run: | set -euo pipefail if [ "${EVENT_NAME}" = "pull_request" ]; then - compare_ref="origin/${BASE_REF_NAME}" + compare_ref="refs/remotes/origin/${BASE_REF_NAME}" + fetch_refspec="+refs/heads/${BASE_REF_NAME}:${compare_ref}" elif [ -n "${BEFORE_SHA}" ] && [ "${BEFORE_SHA}" != "0000000000000000000000000000000000000000" ]; then compare_ref="${BEFORE_SHA}" + fetch_refspec="${BEFORE_SHA}" else echo "web=true" >> "${GITHUB_OUTPUT}" exit 0 fi - if git diff --quiet "${compare_ref}" HEAD -- apps/web .github/workflows/apps-web-check.yml; then + # A tree diff needs the compared commit, not its history. When that + # commit or the diff is unavailable, every web check runs. + fetched=false + for attempt in $(seq "${FETCH_ATTEMPTS}"); do + if git fetch --no-tags --no-recurse-submodules --depth=1 origin "${fetch_refspec}"; then + fetched=true + break + fi + if [ "${attempt}" -lt "${FETCH_ATTEMPTS}" ]; then + echo "::warning::Fetching ${compare_ref} failed (attempt ${attempt} of ${FETCH_ATTEMPTS}); retrying." + sleep "${FETCH_RETRY_DELAY_SECONDS}" + fi + done + if [ "${fetched}" != true ]; then + echo "::warning::Could not fetch ${compare_ref} after ${FETCH_ATTEMPTS} attempts; running every web check." + echo "web=true" >> "${GITHUB_OUTPUT}" + exit 0 + fi + git count-objects -v + + diff_status=0 + git diff --quiet "${compare_ref}" HEAD -- apps/web .github/workflows/apps-web-check.yml || diff_status=$? + if [ "${diff_status}" -eq 0 ]; then echo "web=false" >> "${GITHUB_OUTPUT}" else + if [ "${diff_status}" -ne 1 ]; then + echo "::warning::Could not diff against ${compare_ref}; running every web check." + fi echo "web=true" >> "${GITHUB_OUTPUT}" fi diff --git a/.github/workflows/build-check.yml b/.github/workflows/build-check.yml index 31840fc0eb1..ed9efff723c 100644 --- a/.github/workflows/build-check.yml +++ b/.github/workflows/build-check.yml @@ -27,12 +27,21 @@ jobs: # dist/protocol/ are ~1.1 GiB of the ~1.9 GiB HEAD payload and no # step in this job reads them from disk (build:protocol-tarball only # writes latest.tgz; the immutable-artifact check reads git trees). + # History stays because the immutable-artifact check diffs + # base...HEAD and treats a version without a local v tag as + # unreleased, so a shallow, tagless clone would skip its protection. filter: blob:none sparse-checkout: | /* !/dist/protocol/ sparse-checkout-cone-mode: false + - name: Report checkout footprint + # With the checkout step's duration, these counts show what it fetched (#8559). + run: | + git rev-parse --is-shallow-repository + git count-objects -vH + - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: @@ -188,8 +197,17 @@ jobs: # checked out (no sparse set), so every committed archive, doc and # schema stays on disk. Deferring historical-only blobs reduces # Git storage without omitting current release fixtures. + # Refs and history stay: the docs navigation test reads the stable + # maintenance branch and the withdrawn-release marker test checks + # real history, and both only weaken on a shallow clone. filter: blob:none + - name: Report checkout footprint + # With the checkout step's duration, these counts show what it fetched (#8559). + run: | + git rev-parse --is-shallow-repository + git count-objects -vH + - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: @@ -289,12 +307,18 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - fetch-depth: 0 - # History blobs are fetched on demand; the whole tip tree is still - # checked out (no sparse set), so every committed archive, doc and - # schema stays on disk. Deferring historical-only blobs reduces - # Git storage without omitting current release fixtures. - filter: blob:none + # The server suites read no Git history (only `git rev-parse HEAD`), + # so only the tested commit is fetched. A full-history fetch of every + # branch and tag used most of this job's budget (#8559). The whole tip + # tree is still checked out, so every committed archive, doc and + # schema stays on disk. + fetch-depth: 1 + + - name: Report checkout footprint + # With the checkout step's duration, these counts show what it fetched (#8559). + run: | + git rev-parse --is-shallow-repository + git count-objects -vH - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 @@ -362,18 +386,25 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - fetch-depth: 0 - # Same partial checkout as build-worker. Full history, refs and tags - # stay; only the committed release tarballs under dist/protocol/ + # Same sparse set as build-worker, without history: the integration + # suites read no Git history, so only the tested commit is fetched + # (#8559). The committed release tarballs under dist/protocol/ # (~1.1 GiB) are left unmaterialized. Integration setup builds # schemas/latest from source, and the one /protocol assertion # (schema-routing.test.ts) builds and mounts its own temp tarball. + fetch-depth: 1 filter: blob:none sparse-checkout: | /* !/dist/protocol/ sparse-checkout-cone-mode: false + - name: Report checkout footprint + # With the checkout step's duration, these counts show what it fetched (#8559). + run: | + git rev-parse --is-shallow-repository + git count-objects -vH + - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: diff --git a/.github/workflows/changeset-check.yml b/.github/workflows/changeset-check.yml index b64eb64c875..84376f5a08f 100644 --- a/.github/workflows/changeset-check.yml +++ b/.github/workflows/changeset-check.yml @@ -39,10 +39,16 @@ jobs: # - Release docs snapshot PRs (`auto/docs-v*`) — they freeze docs for an # already-published version and do not represent a new package release. if: ${{ !startsWith(github.head_ref, 'changeset-release/') && !startsWith(github.head_ref, 'forward-merge/') && !startsWith(github.head_ref, 'auto/docs-v') }} + env: + BASE_REF: ${{ github.base_ref }} steps: - uses: actions/checkout@v7.0.1 - with: - fetch-depth: 0 + + # Both checks below compare against the merge base with the target + # branch. Only that much history is fetched instead of every branch and + # tag (#8559); the step fails when no merge base can be found. + - name: Fetch merge base with the target branch + run: bash scripts/fetch-merge-base.sh "${BASE_REF}" - name: Setup Node.js uses: actions/setup-node@v7 @@ -50,16 +56,16 @@ jobs: node-version: '24' - name: Check changeset protocol scope - run: node scripts/check-changeset-protocol-scope.cjs origin/${{ github.base_ref }} + run: node scripts/check-changeset-protocol-scope.cjs "origin/${BASE_REF}" - name: Check for changeset run: | - if node scripts/check-changeset-protocol-scope.cjs origin/${{ github.base_ref }} --is-status-exempt-maintenance; then + if node scripts/check-changeset-protocol-scope.cjs "origin/${BASE_REF}" --is-status-exempt-maintenance; then echo "Skipping changesets status for changeset policy maintenance." exit 0 fi - if ! node scripts/check-changeset-protocol-scope.cjs origin/${{ github.base_ref }} --has-protocol-scoped-changes; then + if ! node scripts/check-changeset-protocol-scope.cjs "origin/${BASE_REF}" --has-protocol-scoped-changes; then echo "Skipping changesets status because this PR does not touch the protocol release surface." exit 0 fi - npx --yes @changesets/cli@^3.0.0 status --since=origin/${{ github.base_ref }} + npx --yes @changesets/cli@^3.0.0 status "--since=origin/${BASE_REF}" diff --git a/.github/workflows/check-image-alt-text.yml b/.github/workflows/check-image-alt-text.yml index 002646bf2ca..aa6d4704919 100644 --- a/.github/workflows/check-image-alt-text.yml +++ b/.github/workflows/check-image-alt-text.yml @@ -7,6 +7,8 @@ on: - 'docs/**/*.mdx' - 'docs/**/*.md' - 'images/**' + - 'scripts/fetch-merge-base.sh' + - '.github/workflows/check-image-alt-text.yml' jobs: check-images: @@ -17,7 +19,11 @@ jobs: - uses: actions/checkout@v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 + + # Changed images are found by diffing against the merge base with main; + # only that much history is fetched (#8559). + - name: Fetch merge base with main + run: bash scripts/fetch-merge-base.sh main - uses: actions/setup-node@v7 with: @@ -32,8 +38,9 @@ jobs: ERRORS="" WARNINGS="" - # Get files changed in this PR - CHANGED_FILES=$(git diff --name-only origin/main...HEAD 2>/dev/null || echo "") + # Get files changed in this PR. A failed diff stops the step instead + # of silently skipping the image quality checks. + CHANGED_FILES=$(git diff --name-only origin/main...HEAD) # Extract all .../dev/null || true # Files on this ref (PR's speculative merge, merge-group commit, or main). local_files=$(ls server/src/db/migrations/*.sql 2>/dev/null | xargs -n1 basename || true) diff --git a/.github/workflows/check-testable-snippets.yml b/.github/workflows/check-testable-snippets.yml index e7ebb8830d6..d4ddedf455c 100644 --- a/.github/workflows/check-testable-snippets.yml +++ b/.github/workflows/check-testable-snippets.yml @@ -9,6 +9,7 @@ on: - 'scripts/check-testable-snippets.cjs' - 'scripts/docs-example-coverage.cjs' - 'scripts/docs-json-field-audit.cjs' + - 'scripts/fetch-merge-base.sh' - '.github/workflows/check-testable-snippets.yml' jobs: @@ -16,8 +17,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7.0.1 - with: - fetch-depth: 0 # Need full history for diff + + # The snippet check diffs against the merge base with the target branch; + # only that much history is fetched (#8559). + - name: Fetch merge base with the target branch + env: + BASE_REF: ${{ github.base_ref }} + run: bash scripts/fetch-merge-base.sh "${BASE_REF}" - name: Setup Node.js uses: actions/setup-node@v7 @@ -25,11 +31,17 @@ jobs: node-version: '24' - name: Check for untested code snippets + env: + BASE_REF: ${{ github.base_ref }} run: | - git fetch origin ${{ github.base_ref }} - git diff origin/${{ github.base_ref }}...HEAD --name-only | grep -E '\.(md|mdx)$' > changed_files.txt || true + set -euo pipefail + changed=$(git diff --name-only "origin/${BASE_REF}...HEAD") + printf '%s\n' "${changed}" | grep -E '\.(md|mdx)$' > changed_files.txt || true if [ -s changed_files.txt ]; then + # The clone is blobless below HEAD and the checker diffs one file at + # a time; one diff first fetches every missing blob in a single batch. + xargs -d '\n' git diff "origin/${BASE_REF}...HEAD" -- < changed_files.txt > /dev/null echo "📋 Checking documentation changes for testable snippets..." node scripts/check-testable-snippets.cjs else diff --git a/scripts/fetch-merge-base.sh b/scripts/fetch-merge-base.sh new file mode 100755 index 00000000000..03d0b8ec85e --- /dev/null +++ b/scripts/fetch-merge-base.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Fetches just enough history for `origin/...HEAD` to resolve in a +# shallow CI checkout exactly as it would with full history, then reports how +# much it fetched. +# +# A full-history checkout (fetch-depth: 0) downloads every branch and tag and +# has consumed whole job budgets before any check ran (#8559). Path checks +# only need the merge base, so this fetches commits and trees (no blobs) for +# the base branch and HEAD with a doubling depth until an exact merge base +# appears, then falls back to their complete history. With still no merge +# base it fails: a diff without one must never pass as "no changes". +# +# Usage: scripts/fetch-merge-base.sh +set -euo pipefail + +readonly REMOTE=origin +readonly INITIAL_DEPTH="${FETCH_MERGE_BASE_INITIAL_DEPTH:-50}" +readonly MAX_DEPTH="${FETCH_MERGE_BASE_MAX_DEPTH:-3200}" +# actions/checkout retries its own fetches; these fetches replace part of its +# work, so a transient network error should not fail the check either. +readonly FETCH_ATTEMPTS=3 +readonly RETRY_DELAY_SECONDS="${FETCH_MERGE_BASE_RETRY_DELAY_SECONDS:-5}" +readonly POSITIVE_INTEGER='^[1-9][0-9]*$' +readonly NON_NEGATIVE_INTEGER='^[0-9]+$' +readonly MERGE_BASE_NOT_FOUND=1 + +fail() { + echo "::error::$1" + exit "${2:-1}" +} + +base_branch="${1:-}" +if [[ -z "$base_branch" ]] || ! git check-ref-format --branch "$base_branch" >/dev/null 2>&1; then + fail "Usage: fetch-merge-base.sh ; got \"${base_branch}\"." 2 +fi +if ! [[ "$INITIAL_DEPTH" =~ $POSITIVE_INTEGER && "$MAX_DEPTH" =~ $POSITIVE_INTEGER ]]; then + fail "Fetch depths must be positive integers (initial ${INITIAL_DEPTH}, max ${MAX_DEPTH})." 2 +fi +if ! [[ "$RETRY_DELAY_SECONDS" =~ $NON_NEGATIVE_INTEGER ]]; then + fail "The retry delay must be a whole number of seconds (got ${RETRY_DELAY_SECONDS})." 2 +fi + +readonly base_ref="refs/remotes/${REMOTE}/${base_branch}" +head_sha=$(git rev-parse --verify 'HEAD^{commit}') +readonly head_sha +shallow_file=$(git rev-parse --git-path shallow) +readonly shallow_file +readonly started_seconds=$SECONDS + +is_shallow() { + [[ "$(git rev-parse --is-shallow-repository)" == true ]] +} + +fetch_with_retry() { + local attempt + for (( attempt = 1; attempt <= FETCH_ATTEMPTS; attempt += 1 )); do + git fetch --no-tags --no-recurse-submodules --filter=blob:none "$@" && return 0 + if (( attempt < FETCH_ATTEMPTS )); then + echo "::warning::git fetch failed (attempt ${attempt} of ${FETCH_ATTEMPTS}); retrying in ${RETRY_DELAY_SECONDS}s." + sleep "$RETRY_DELAY_SECONDS" + fi + done + fail "git fetch from ${REMOTE} failed after ${FETCH_ATTEMPTS} attempts." +} + +fetch_targets=("+refs/heads/${base_branch}:${base_ref}" "$head_sha") +fetch_history() { + fetch_with_retry "$@" "$REMOTE" "${fetch_targets[@]}" + # Later fetches name the pinned base tip by SHA. Git skips an up-to-date + # ref whenever another commit is also wanted, so a refspec would leave the + # base side at its first depth while only HEAD deepened. + fetch_targets=("$(git rev-parse --verify "${base_ref}^{commit}")" "$head_sha") +} + +# A shallow graph can offer an older common ancestor (say, a release-branch +# commit merged into both sides) while the real merge base is still cut off. +# The answer is exact only when no shallow boundary lies between it and +# either tip, because then both sides are complete down to it. +merge_base_is_exact() { + local commits + [[ -s "$shallow_file" ]] || return 0 + # An unreadable range must fail, not pass as "no shallow commit above it". + commits=$(git rev-list "$base_ref" "$head_sha" --not "$1") || + fail "Could not list the commits above merge base ${1}." + ! grep -qxF -f "$shallow_file" <<<"$commits" +} + +report_merge_base() { + local merge_base status=0 + merge_base=$(git merge-base "$base_ref" "$head_sha") || status=$? + if (( status == MERGE_BASE_NOT_FOUND )); then + return 1 + elif (( status != 0 )); then + fail "git merge-base failed (exit ${status}) for ${base_ref} and HEAD." + fi + merge_base_is_exact "$merge_base" || return 1 + echo "merge_base=${merge_base} base=${base_ref} head=${head_sha} history=$1 elapsed_seconds=$((SECONDS - started_seconds))" + git count-objects -v +} + +depth=$INITIAL_DEPTH +while is_shallow && (( depth <= MAX_DEPTH )); do + fetch_history --depth="$depth" + report_merge_base "depth-${depth}" && exit 0 + depth=$((depth * 2)) +done + +if is_shallow; then + echo "::warning::No exact merge base within ${MAX_DEPTH} commits of ${base_ref} and HEAD; fetching their complete history." + fetch_history --unshallow +else + fetch_history +fi +report_merge_base complete && exit 0 +fail "No merge base between ${base_ref} and HEAD, even with their complete history." diff --git a/tests/ci-bounded-checkout.test.mjs b/tests/ci-bounded-checkout.test.mjs new file mode 100644 index 00000000000..77ef05ccbed --- /dev/null +++ b/tests/ci-bounded-checkout.test.mjs @@ -0,0 +1,395 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import { createRequire } from 'node:module'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import YAML from 'yaml'; + +// Path checks run on bounded checkouts instead of fetching every branch and +// tag (#8559). These tests drive the real fetch logic against file:// clones +// and compare it with the answer full history gives. +const require = createRequire(import.meta.url); +const { fixtureEnv } = require('./helpers/git-object-fixture.cjs'); + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const FETCH_MERGE_BASE = path.join(REPO_ROOT, 'scripts', 'fetch-merge-base.sh'); +const FIXTURE_TIMEOUT_MS = 60_000; +const ZERO_SHA = '0'.repeat(40); +const UNKNOWN_SHA = '1'.repeat(40); +// Main runs this many commits past the PR's base, so the merge base sits +// deeper than the first fetch window on the base side only. +const MAIN_COMMITS_AFTER_PR_BASE = 10; +// Windows sized to the fixture: the merge base is reachable within 16. +const BOUNDED_WINDOW = { FETCH_MERGE_BASE_INITIAL_DEPTH: '2', FETCH_MERGE_BASE_MAX_DEPTH: '16' }; +const TOO_SMALL_WINDOW = { FETCH_MERGE_BASE_INITIAL_DEPTH: '2', FETCH_MERGE_BASE_MAX_DEPTH: '8' }; +// Retries keep their real attempt counts; only the wait between them is skipped. +const NO_RETRY_DELAY = { FETCH_MERGE_BASE_RETRY_DELAY_SECONDS: '0', FETCH_RETRY_DELAY_SECONDS: '0' }; +const REAL_GIT = execFileSync('which', ['git'], { encoding: 'utf8' }).trim(); +// Fails one git subcommand outright, or the first N fetches, then defers to the +// real git. Git's own child processes use its exec path, so they bypass this. +const GIT_STUB = `#!/usr/bin/env bash +if [ -n "\${GIT_STUB_FAIL_SUBCOMMAND:-}" ] && [ "$1" = "\${GIT_STUB_FAIL_SUBCOMMAND}" ]; then + echo "stub: $1 failed" >&2 + exit 128 +fi +if [ "$1" = fetch ] && [ -n "\${GIT_STUB_FAILING_FETCHES:-}" ]; then + count=$(cat "\${GIT_STUB_COUNTER}" 2>/dev/null || echo 0) + echo $((count + 1)) > "\${GIT_STUB_COUNTER}" + if [ "$count" -lt "\${GIT_STUB_FAILING_FETCHES}" ]; then + echo "stub: transient fetch failure" >&2 + exit 128 + fi +fi +exec "${REAL_GIT}" "$@" +`; + +const loadWorkflow = (name) => + YAML.parse(fs.readFileSync(path.join(REPO_ROOT, '.github', 'workflows', name), 'utf8')); + +const temporaryDirectories = []; +function makeTemporaryDirectory(prefix) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + temporaryDirectories.push(directory); + return directory; +} + +function git(cwd, args) { + return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], env: fixtureEnv() }).trim(); +} + +function commitFiles(repo, message, files = {}) { + for (const [file, content] of Object.entries(files)) { + fs.mkdirSync(path.join(repo, path.dirname(file)), { recursive: true }); + fs.writeFileSync(path.join(repo, file), content); + } + git(repo, ['add', '-A']); + git(repo, ['commit', '-q', '--allow-empty', '-m', message]); + return git(repo, ['rev-parse', 'HEAD']); +} + +function mergeInto(repo, target, source, message) { + git(repo, ['checkout', '-q', '--detach', target]); + git(repo, ['merge', '-q', '--no-ff', '--no-edit', '-m', message, source]); + return git(repo, ['rev-parse', 'HEAD']); +} + +/** + * main: root - E(merge release R) - P - m1..m10 - J(merge release R2) - W(apps/web) + * release: root - R - R2 + * PRs from P: H (docs only) and HW (apps/web); MERGE / MERGE_WEB are their + * pull_request merge refs, built on P while main has since moved on. + * + * From the main tip, R is a few commits away through J, while P is more than + * ten away. A shallow graph therefore offers R as a common ancestor before + * the real merge base P is reachable. + */ +function createOrigin() { + const repo = makeTemporaryDirectory('adcp-bounded-origin-'); + git(repo, ['init', '-q']); + git(repo, ['checkout', '-q', '-b', 'main']); + // Fetch by SHA and partial fetches behave as they do against GitHub. + git(repo, ['config', 'uploadpack.allowFilter', 'true']); + git(repo, ['config', 'uploadpack.allowAnySHA1InWant', 'true']); + + const root = commitFiles(repo, 'root', { + 'apps/web/app.ts': 'export const app = 1;\n', + 'docs/guide.md': '# Guide\n', + 'release/notes.md': 'notes\n', + }); + git(repo, ['checkout', '-q', '-b', 'release']); + const releaseCommit = commitFiles(repo, 'release fix', { 'release/notes.md': 'notes 1\n' }); + const earlyForwardMerge = mergeInto(repo, root, releaseCommit, 'forward-merge release'); + const prBase = commitFiles(repo, 'main work'); + for (let index = 1; index <= MAIN_COMMITS_AFTER_PR_BASE; index += 1) { + commitFiles(repo, `main ${index}`); + } + const mainBeforeForwardMerge = git(repo, ['rev-parse', 'HEAD']); + + git(repo, ['checkout', '-q', releaseCommit]); + const laterReleaseCommit = commitFiles(repo, 'release fix 2', { 'release/notes.md': 'notes 2\n' }); + git(repo, ['branch', '-q', '-f', 'release', laterReleaseCommit]); + const forwardMerge = mergeInto(repo, mainBeforeForwardMerge, laterReleaseCommit, 'forward-merge release 2'); + // The detector compares against the target tip, so pull request cases + // target a branch cut before main's own apps/web change. + git(repo, ['branch', '-q', '-f', 'before-web-push', forwardMerge]); + const webPush = commitFiles(repo, 'web change on main', { 'apps/web/app.ts': 'export const app = 2;\n' }); + git(repo, ['branch', '-q', '-f', 'main', webPush]); + + git(repo, ['checkout', '-q', '--detach', prBase]); + const docsHead = commitFiles(repo, 'docs change', { 'docs/guide.md': '# Guide\n\nMore.\n' }); + git(repo, ['checkout', '-q', '--detach', prBase]); + const webHead = commitFiles(repo, 'web change', { 'apps/web/app.ts': 'export const app = 3;\n' }); + const docsMerge = mergeInto(repo, prBase, docsHead, 'Merge docs PR'); + const webMerge = mergeInto(repo, prBase, webHead, 'Merge web PR'); + git(repo, ['checkout', '-q', 'main']); + return { + repo, + url: `file://${repo}`, + shas: { + root, releaseCommit, earlyForwardMerge, prBase, forwardMerge, webPush, + docsHead, webHead, docsMerge, webMerge, + }, + }; +} + +/** Same shape as actions/checkout with fetch-depth 1. */ +function shallowCheckout(origin, sha) { + const clone = makeTemporaryDirectory('adcp-bounded-clone-'); + git(clone, ['init', '-q']); + git(clone, ['remote', 'add', 'origin', origin.url]); + git(clone, ['fetch', '-q', '--no-tags', '--depth=1', 'origin', `+${sha}:refs/remotes/pull/1/merge`]); + git(clone, ['checkout', '-q', '--detach', sha]); + expect(git(clone, ['rev-parse', '--is-shallow-repository'])).toBe('true'); + return clone; +} + +/** + * Environment that puts the git stub first on PATH. `failSubcommand` makes + * that subcommand fail; `failingFetches` makes the first N fetches fail. + */ +function gitStubEnv({ failSubcommand = '', failingFetches = '' } = {}) { + const directory = makeTemporaryDirectory('adcp-git-stub-'); + fs.writeFileSync(path.join(directory, 'git'), GIT_STUB, { mode: 0o755 }); + return { + PATH: `${directory}${path.delimiter}${process.env.PATH}`, + GIT_STUB_FAIL_SUBCOMMAND: failSubcommand, + GIT_STUB_FAILING_FETCHES: String(failingFetches), + GIT_STUB_COUNTER: path.join(directory, 'fetch-count'), + }; +} + +function runFetchMergeBase(clone, baseBranch, extraEnv) { + const result = spawnSync('bash', [FETCH_MERGE_BASE, baseBranch], { + cwd: clone, + encoding: 'utf8', + env: { ...fixtureEnv(), ...NO_RETRY_DELAY, ...extraEnv }, + }); + return { status: result.status, output: `${result.stdout}${result.stderr}` }; +} + +function runDetectWebChanges(clone, eventEnv, extraEnv = {}) { + const workflow = loadWorkflow('apps-web-check.yml'); + const step = workflow.jobs.check.steps.find((candidate) => candidate.name === 'Detect web changes'); + // The step's fixed settings (retry attempts) apply as written; the + // expression-valued ones come from each test's event. + const fixedStepEnv = Object.fromEntries( + Object.entries(step.env).filter(([, value]) => !String(value).includes('${{')), + ); + const scriptFile = path.join(makeTemporaryDirectory('adcp-web-detect-'), 'detect.sh'); + const outputFile = `${scriptFile}.output`; + fs.writeFileSync(scriptFile, step.run); + fs.writeFileSync(outputFile, ''); + // GitHub runs an unspecified `run` shell as `bash -e {0}`. + const result = spawnSync('bash', ['-e', scriptFile], { + cwd: clone, + encoding: 'utf8', + env: { + ...fixtureEnv(), + ...fixedStepEnv, + ...NO_RETRY_DELAY, + BASE_REF_NAME: '', + BEFORE_SHA: '', + ...eventEnv, + ...extraEnv, + GITHUB_OUTPUT: outputFile, + }, + }); + return { + status: result.status, + output: `${result.stdout}${result.stderr}`, + web: fs.readFileSync(outputFile, 'utf8').trim(), + }; +} + +let origin; +beforeAll(() => { + origin = createOrigin(); +}, FIXTURE_TIMEOUT_MS); + +afterAll(() => { + for (const directory of temporaryDirectories) fs.rmSync(directory, { recursive: true, force: true }); +}); + +describe('scripts/fetch-merge-base.sh', () => { + it('finds the full-history merge base when the base side needs deepening past a decoy ancestor', () => { + const { prBase, docsMerge } = origin.shas; + expect(git(origin.repo, ['merge-base', 'main', docsMerge])).toBe(prBase); + + const clone = shallowCheckout(origin, docsMerge); + const result = runFetchMergeBase(clone, 'main', BOUNDED_WINDOW); + + expect(result.status, result.output).toBe(0); + // Found inside the bounded window, so the base side deepened too. + expect(result.output).toContain(`merge_base=${prBase} `); + expect(result.output).toContain('history=depth-16'); + expect(git(clone, ['merge-base', 'origin/main', 'HEAD'])).toBe(prBase); + expect(git(clone, ['diff', '--name-only', 'origin/main...HEAD'])).toBe( + git(origin.repo, ['diff', '--name-only', `main...${docsMerge}`]), + ); + expect(git(clone, ['diff', '--name-only', 'origin/main...HEAD'])).toBe('docs/guide.md'); + }, FIXTURE_TIMEOUT_MS); + + it('falls back to complete history when the window is too small', () => { + const { prBase, docsHead } = origin.shas; + const clone = shallowCheckout(origin, docsHead); + const result = runFetchMergeBase(clone, 'main', TOO_SMALL_WINDOW); + + expect(result.status, result.output).toBe(0); + expect(result.output).toContain('::warning::No exact merge base within 8 commits'); + expect(result.output).toContain(`merge_base=${prBase} `); + expect(result.output).toContain('history=complete'); + expect(git(clone, ['diff', '--name-only', 'origin/main...HEAD'])).toBe( + git(origin.repo, ['diff', '--name-only', `main...${docsHead}`]), + ); + }, FIXTURE_TIMEOUT_MS); + + it('fails when HEAD shares no history with the base branch', () => { + const clone = makeTemporaryDirectory('adcp-bounded-unrelated-'); + git(clone, ['init', '-q']); + git(clone, ['remote', 'add', 'origin', origin.url]); + commitFiles(clone, 'unrelated'); + // The helper fetches HEAD by SHA, so the orphan commit must exist on origin. + execFileSync('git', ['push', '-q', origin.url, 'HEAD:refs/heads/unrelated'], { cwd: clone, env: fixtureEnv() }); + git(clone, ['fetch', '-q', '--depth=1', 'origin', 'refs/heads/unrelated']); + + const result = runFetchMergeBase(clone, 'main', TOO_SMALL_WINDOW); + + expect(result.status, result.output).toBe(1); + expect(result.output).toContain('::error::No merge base between refs/remotes/origin/main and HEAD'); + expect(result.output).not.toContain('merge_base='); + }, FIXTURE_TIMEOUT_MS); + + it('fails instead of accepting a merge base when the commits above it cannot be listed', () => { + const clone = shallowCheckout(origin, origin.shas.docsMerge); + const result = runFetchMergeBase(clone, 'main', { ...BOUNDED_WINDOW, ...gitStubEnv({ failSubcommand: 'rev-list' }) }); + + expect(result.status, result.output).toBe(1); + expect(result.output).toContain('::error::Could not list the commits above merge base'); + expect(result.output).not.toContain('merge_base='); + }, FIXTURE_TIMEOUT_MS); + + it('retries a failed fetch', () => { + const { prBase, docsMerge } = origin.shas; + const clone = shallowCheckout(origin, docsMerge); + const result = runFetchMergeBase(clone, 'main', { ...BOUNDED_WINDOW, ...gitStubEnv({ failingFetches: 2 }) }); + + expect(result.status, result.output).toBe(0); + expect(result.output).toContain('::warning::git fetch failed (attempt 1 of 3)'); + expect(result.output).toContain('::warning::git fetch failed (attempt 2 of 3)'); + expect(result.output).toContain(`merge_base=${prBase} `); + }, FIXTURE_TIMEOUT_MS); + + it('fails after the last fetch attempt', () => { + const clone = shallowCheckout(origin, origin.shas.docsMerge); + git(clone, ['remote', 'set-url', 'origin', `${origin.url}-missing`]); + const result = runFetchMergeBase(clone, 'main', BOUNDED_WINDOW); + + expect(result.status, result.output).toBe(1); + expect(result.output).toContain('::warning::git fetch failed (attempt 2 of 3)'); + expect(result.output).toContain('::error::git fetch from origin failed after 3 attempts.'); + expect(result.output).not.toContain('merge_base='); + }, FIXTURE_TIMEOUT_MS); + + it('rejects a missing or malformed base branch', () => { + const clone = shallowCheckout(origin, origin.shas.docsHead); + for (const baseBranch of ['', 'bad..name']) { + const result = runFetchMergeBase(clone, baseBranch, BOUNDED_WINDOW); + expect(result.status, result.output).toBe(2); + expect(result.output).toContain('::error::Usage: fetch-merge-base.sh '); + } + }, FIXTURE_TIMEOUT_MS); +}); + +describe('apps/web check change detection on a depth-1 checkout', () => { + it.each([ + ['a pull request that changes apps/web', 'webMerge', 'web=true'], + ['a pull request that does not', 'docsMerge', 'web=false'], + ])('reports %s', (_label, mergeSha, expected) => { + const clone = shallowCheckout(origin, origin.shas[mergeSha]); + const result = runDetectWebChanges(clone, { EVENT_NAME: 'pull_request', BASE_REF_NAME: 'before-web-push' }); + expect(result.status, result.output).toBe(0); + expect(result.web).toBe(expected); + }, FIXTURE_TIMEOUT_MS); + + it.each([ + ['a push that changes apps/web', 'webPush', 'forwardMerge', 'web=true'], + ['a push that does not', 'forwardMerge', 'prBase', 'web=false'], + ])('reports %s', (_label, headSha, beforeSha, expected) => { + const clone = shallowCheckout(origin, origin.shas[headSha]); + const result = runDetectWebChanges(clone, { EVENT_NAME: 'push', BEFORE_SHA: origin.shas[beforeSha] }); + expect(result.status, result.output).toBe(0); + expect(result.web).toBe(expected); + }, FIXTURE_TIMEOUT_MS); + + it('retries a failed fetch before comparing', () => { + const { forwardMerge, prBase } = origin.shas; + const clone = shallowCheckout(origin, forwardMerge); + const result = runDetectWebChanges( + clone, + { EVENT_NAME: 'push', BEFORE_SHA: prBase }, + gitStubEnv({ failingFetches: 1 }), + ); + expect(result.status, result.output).toBe(0); + expect(result.output).toContain('(attempt 1 of 3); retrying.'); + expect(result.output).not.toContain('running every web check'); + expect(result.web).toBe('web=false'); + }, FIXTURE_TIMEOUT_MS); + + it('runs every web check when the compared commit cannot be fetched', () => { + const clone = shallowCheckout(origin, origin.shas.forwardMerge); + const result = runDetectWebChanges(clone, { EVENT_NAME: 'push', BEFORE_SHA: UNKNOWN_SHA }); + expect(result.status, result.output).toBe(0); + expect(result.output).toContain('(attempt 2 of 3); retrying.'); + expect(result.output).toContain(`::warning::Could not fetch ${UNKNOWN_SHA} after 3 attempts; running every web check.`); + expect(result.web).toBe('web=true'); + }, FIXTURE_TIMEOUT_MS); + + it('runs every web check when the diff itself fails', () => { + const { forwardMerge, prBase } = origin.shas; + const clone = shallowCheckout(origin, forwardMerge); + const result = runDetectWebChanges( + clone, + { EVENT_NAME: 'push', BEFORE_SHA: prBase }, + gitStubEnv({ failSubcommand: 'diff' }), + ); + expect(result.status, result.output).toBe(0); + expect(result.output).toContain(`::warning::Could not diff against ${prBase}; running every web check.`); + expect(result.web).toBe('web=true'); + }, FIXTURE_TIMEOUT_MS); + + it('runs every web check without fetching when the push has no previous commit', () => { + const clone = shallowCheckout(origin, origin.shas.forwardMerge); + const result = runDetectWebChanges(clone, { EVENT_NAME: 'push', BEFORE_SHA: ZERO_SHA }); + expect(result.status, result.output).toBe(0); + expect(result.output).not.toContain('::warning::'); + expect(result.web).toBe('web=true'); + }, FIXTURE_TIMEOUT_MS); +}); + +describe('verification lanes keep bounded checkouts', () => { + const checkoutOf = (job) => job.steps.find((step) => String(step.uses ?? '').startsWith('actions/checkout@')); + + it.each([ + ['apps-web-check.yml', 'check'], + ['build-check.yml', 'server-unit-worker'], + ['build-check.yml', 'server-integration-worker'], + ])('%s %s fetches only the tested commit', (file, jobId) => { + expect(checkoutOf(loadWorkflow(file).jobs[jobId]).with?.['fetch-depth']).toBe(1); + }); + + it.each([ + ['changeset-check.yml', 'check'], + ['check-image-alt-text.yml', 'check-images'], + ['check-migration-numbers.yml', 'check'], + ['check-testable-snippets.yml', 'check-snippets'], + ])('%s %s fetches the merge base before diffing against it', (file, jobId) => { + const { steps } = loadWorkflow(file).jobs[jobId]; + expect(checkoutOf({ steps }).with?.['fetch-depth'] ?? 1).toBe(1); + const fetchIndex = steps.findIndex((step) => String(step.run ?? '').includes('scripts/fetch-merge-base.sh')); + const diffIndex = steps.findIndex((step) => /\.\.\.HEAD|--since=|check-changeset-protocol-scope/.test(String(step.run ?? ''))); + expect(fetchIndex).toBeGreaterThan(-1); + expect(diffIndex).toBeGreaterThan(fetchIndex); + }); +});