ci(repo): run the release gate on pull requests, scoped to the change

.github/workflows/release-gate.yml

@@ -3,14 +3,15 @@ name: Release gate
 on:
   push:
     branches: [main]
+  pull_request:
   workflow_dispatch:
 
 permissions:
   contents: read
 
 concurrency:
   group: release-gate-${{ github.ref }}
-  cancel-in-progress: false
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   plan:
@@ -20,33 +21,44 @@ jobs:
     outputs:
       matrix: ${{ steps.publish.outputs.matrix }}
       has-shards: ${{ steps.publish.outputs.has-shards }}
+      scope: ${{ steps.projects.outputs.scope }}
     steps:
       - uses: actions/checkout@v7
         with:
+          fetch-depth: 2
           persist-credentials: false
       - uses: ./.github/actions/dev-shell
+      - name: List the files the pull request changes
+        if: github.event_name == 'pull_request'
+        run: mkdir -p .cache && git diff --name-only HEAD^1 HEAD > .cache/changed-files.txt
       - name: Discover mutation projects
         uses: ./.github/actions/sandbox
         with:
           hosts: |
             jsr.io
             registry.npmjs.org
           command: >-
-            deno run --config=scripts/deno.json --allow-read --allow-write
-            scripts/stryker-plan-gate.ts projects --out .cache/mutation-projects.txt
+            deno run --config=scripts/deno.json --allow-read --allow-write --allow-env=MUTATION_SCOPE
+            scripts/stryker-plan-gate.ts projects --changed .cache/changed-files.txt
+            --out .cache/mutation-projects.txt --scope-out .cache/mutation-scope.txt
       - id: projects
-        name: Publish the project list
-        run: echo "projects=$(cat .cache/mutation-projects.txt)" >> "$GITHUB_OUTPUT"
+        name: Publish the project list and the mutation scope
+        run: |
+          echo "projects=$(cat .cache/mutation-projects.txt)" >> "$GITHUB_OUTPUT"
+          echo "scope=$(cat .cache/mutation-scope.txt)" >> "$GITHUB_OUTPUT"
       - name: Plan mutation shards
         if: steps.projects.outputs.projects != ''
         uses: ./.github/actions/sandbox
         env:
           MUTATION_PROJECTS: ${{ steps.projects.outputs.projects }}
+          MUTATION_SCOPE: ${{ steps.projects.outputs.scope }}
         with:
           hosts: |
             jsr.io
             registry.npmjs.org
-          pass-env: MUTATION_PROJECTS
+          pass-env: |
+            MUTATION_PROJECTS
+            MUTATION_SCOPE
           command: ./node_modules/.bin/stryker plan --target-seconds 900 --projects "$MUTATION_PROJECTS" --out stryker-plan.json
       - name: Gate the plan
         uses: ./.github/actions/sandbox
@@ -55,8 +67,9 @@ jobs:
             jsr.io
             registry.npmjs.org
           command: >-
-            deno run --config=scripts/deno.json --allow-read --allow-write
-            scripts/stryker-plan-gate.ts gate --plan stryker-plan.json --out .cache/mutation-plan.out
+            deno run --config=scripts/deno.json --allow-read --allow-write --allow-env=MUTATION_SCOPE
+            scripts/stryker-plan-gate.ts gate --plan stryker-plan.json --changed .cache/changed-files.txt
+            --out .cache/mutation-plan.out
       - id: publish
         name: Publish the plan
         run: cat .cache/mutation-plan.out >> "$GITHUB_OUTPUT"
@@ -78,6 +91,7 @@ jobs:
       matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
     env:
       CI: "true"
+      MUTATION_SCOPE: ${{ needs.plan.outputs.scope }}
     steps:
       - uses: actions/checkout@v7
         with:
@@ -89,6 +103,7 @@ jobs:
           name: stryker-plan
           path: .
       - name: Stryker incremental cache
+        if: github.event_name != 'pull_request'
         uses: actions/cache@v6
         with:
           path: |
@@ -104,6 +119,7 @@ jobs:
           pass-env: |
             GITHUB_ACTIONS
             MUTATION_SHARD
+            MUTATION_SCOPE
           command: ./node_modules/.bin/stryker run --plan stryker-plan.json --shard "$MUTATION_SHARD"
       - if: always()
         uses: actions/upload-artifact@v6
@@ -117,29 +133,79 @@ jobs:
           name: mutation-report-shard-${{ strategy.job-index }}
           path: packages/*/reports/mutation-report.*
           if-no-files-found: ignore
+      - name: Re-run a failed shard's project runs with their output kept
+        if: failure()
+        timeout-minutes: 20
+        uses: ./.github/actions/sandbox
+        env:
+          MUTATION_SHARD: ${{ matrix.shard }}
+        with:
+          pass-env: |
+            GITHUB_ACTIONS
+            MUTATION_SHARD
+            MUTATION_SCOPE
+          command: |
+            set -uo pipefail
+            root="$PWD"
+            jq -r --arg shard "$MUTATION_SHARD" \
+              '.shards[] | select("\(.index)/\(.count)" == $shard) | .projects[].project' \
+              stryker-plan.json > .cache/shard-projects.txt
+            while read -r project; do
+              out="$root/reports/diagnostics/$project"
+              mkdir -p "$out"
+              (cd "$project" && "$root/node_modules/.bin/stryker" run --plan "$root/stryker-plan.json" \
+                --shard "$MUTATION_SHARD" --project "$project" \
+                --progressStreamFile "$out/mutation-stream.jsonl" \
+                --incremental --incrementalFile "$out/stryker-incremental.json" \
+                > "$out/stdout.log" 2> "$out/stderr.log")
+              echo "$project exited $?" | tee "$out/exit.txt"
+            done < .cache/shard-projects.txt
+      - if: failure()
+        uses: actions/upload-artifact@v6
+        with:
+          name: mutation-diagnostics-shard-${{ strategy.job-index }}
+          path: |
+            reports/diagnostics/
+            packages/*/stryker.log
+          if-no-files-found: ignore
 
   verdict:
     name: verdict · merged report
-    needs: [mutation]
+    needs: [plan, mutation]
+    if: ${{ !cancelled() }}
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v7
+      - name: Refuse a plan or a shard that did not finish
+        env:
+          PLAN: ${{ needs.plan.result }}
+          MUTATION: ${{ needs.mutation.result }}
+          HAS_SHARDS: ${{ needs.plan.outputs.has-shards }}
+        run: |
+          if [ "$PLAN" != success ]; then echo "::error::the plan job ended $PLAN"; exit 1; fi
+          if [ "$HAS_SHARDS" != true ]; then echo "no mutated file is in this change's scope"; exit 0; fi
+          if [ "$MUTATION" != success ]; then echo "::error::a mutation shard ended $MUTATION"; exit 1; fi
+      - if: needs.plan.outputs.has-shards == 'true'
+        uses: actions/checkout@v7
         with:
           persist-credentials: false
-      - uses: ./.github/actions/dev-shell
+      - if: needs.plan.outputs.has-shards == 'true'
+        uses: ./.github/actions/dev-shell
       - name: Download the shard plan
+        if: needs.plan.outputs.has-shards == 'true'
         uses: actions/download-artifact@v6
         with:
           name: stryker-plan
           path: .
       - name: Download every shard's reports
+        if: needs.plan.outputs.has-shards == 'true'
         uses: actions/download-artifact@v6
         with:
           pattern: mutation-shard-*
           path: .cache/shard-reports
           merge-multiple: true
       - name: Merge the shard reports and gate them at zero survivors
+        if: needs.plan.outputs.has-shards == 'true'
         uses: ./.github/actions/sandbox
         with:
           command: |
@@ -149,7 +215,7 @@ jobs:
             mapfile -t shard_dirs < .cache/shard-dirs.txt
             ./node_modules/.bin/stryker merge --plan stryker-plan.json --out reports/mutation "${shard_dirs[@]}"
             ./node_modules/.bin/stryker gate --baseline .cache/no-survivors.json
-      - if: always()
+      - if: always() && needs.plan.outputs.has-shards == 'true'
         uses: actions/upload-artifact@v6
         with:
           name: mutation-report

CONTRIBUTING.md

@@ -51,7 +51,7 @@ pnpm check:ci
 nix build .#workspace-tarballs
 ```
 
-Mutation testing is not part of `pnpm check:ci`. The release gate (`.github/workflows/release-gate.yml`) runs `stryker plan` once over every workspace package that declares a `mutation` script, then `stryker run` for each planned shard at a break threshold of 100 on every push to `main`.
+Mutation testing is not part of `pnpm check:ci`. The release gate (`.github/workflows/release-gate.yml`) runs `stryker plan` once over every workspace package that declares a `mutation` script, then `stryker run` for each planned shard at a break threshold of 100 on every push to `main`. The same workflow runs on every pull request, scoped to what the pull request changes. It mutates the declared files the pull request touches. It mutates a package's whole declared set when the pull request touches any other file in that package: a test, a fixture, a config, or a source file outside the set. It mutates every package's whole set when the pull request changes the gate itself: the workflow, `scripts/stryker-plan-gate.ts`, `stryker.shared.ts`, the lockfile or the Nix toolchain. A pull request that touches no mutated package plans nothing, and its verdict job passes. Pull-request runs skip the incremental cache, so every scoped mutant runs fresh.
 
 ## Pull Requests & Commits
 

README.md

@@ -23,16 +23,16 @@ Each package is also its own flake attribute, named after the last segment of it
 
 ## Toolchain
 
-| Tool                    | Role                                                                               |
-| ----------------------- | ---------------------------------------------------------------------------------- |
-| **Nix**                 | Pins Node.js 24, pnpm 12.9.0, Deno and dprint, and builds the package tarballs     |
-| **Sandbox**             | pnpm-release-management's deny-by-default launcher; all dependency code runs in it |
-| **pnpm + Turbo**        | Workspace catalog with exact pins, cached task graph                               |
-| **TypeScript 7 (tsgo)** | Typechecking through `@effect/tsgo`                                                |
-| **oxlint**              | The `@systemfsoftware/oxlint-config-recommended` preset, at error severity         |
-| **Vitest**              | Unit and integration tests                                                         |
-| **Stryker**             | Mutation testing at a break threshold of 100, on `main` only (the release gate)    |
-| **dprint**              | Formatting for code and Markdown                                                   |
+| Tool                    | Role                                                                                                                |
+| ----------------------- | ------------------------------------------------------------------------------------------------------------------- |
+| **Nix**                 | Pins Node.js 24, pnpm 12.9.0, Deno and dprint, and builds the package tarballs                                      |
+| **Sandbox**             | pnpm-release-management's deny-by-default launcher; all dependency code runs in it                                  |
+| **pnpm + Turbo**        | Workspace catalog with exact pins, cached task graph                                                                |
+| **TypeScript 7 (tsgo)** | Typechecking through `@effect/tsgo`                                                                                 |
+| **oxlint**              | The `@systemfsoftware/oxlint-config-recommended` preset, at error severity                                          |
+| **Vitest**              | Unit and integration tests                                                                                          |
+| **Stryker**             | Mutation testing at a break threshold of 100, on `main` and scoped to each pull request's change (the release gate) |
+| **dprint**              | Formatting for code and Markdown                                                                                    |
 
 ## Contributing
 

scripts/stryker-plan-gate.test.ts

@@ -1,11 +1,12 @@
 import { assertEquals } from '@std/assert'
 import { join } from '@std/path'
-import { gatePlan, type Refusal, type ShardPlan } from './stryker-plan-gate.ts'
+import { gatePlan, type Refusal, scopeOf, type ShardPlan } from './stryker-plan-gate.ts'
 
 interface FixturePackage {
   readonly dir: string
   readonly name: string
   readonly mutates: boolean
+  readonly strykerConfig?: string
 }
 
 interface FixtureOptions {
@@ -27,6 +28,7 @@ const writeFixture = async (options: FixtureOptions): Promise<{ root: string; pl
       ...(pkg.mutates ? { scripts: { mutation: 'stryker run' } } : {}),
     }
     await Deno.writeTextFile(join(dir, 'package.json'), `${JSON.stringify(manifest, null, 2)}\n`)
+    if (pkg.strykerConfig !== undefined) await Deno.writeTextFile(join(dir, 'stryker.config.ts'), pkg.strykerConfig)
   }
   if (options.plan !== undefined) {
     await Deno.writeTextFile(join(root, 'plan.json'), `${JSON.stringify(options.plan)}\n`)
@@ -68,6 +70,7 @@ Deno.test('a valid plan yields the plan matrix and has-shards', async () => {
       matrix: { include: [{ shard: '1/2', predictedSeconds: 5 }, { shard: '2/2', predictedSeconds: 3 }] },
       hasShards: true,
       unmutated: [],
+      outOfScope: [],
     },
   })
 })
@@ -150,6 +153,7 @@ Deno.test('a mutation package with no scheduled mutants is allowed by a ledger e
     result: {
       matrix: { include: [{ shard: '1/1', predictedSeconds: 1 }] },
       hasShards: true,
+      outOfScope: [],
       unmutated: [{
         package: '@fixture/site',
         dir: 'packages/site',
@@ -176,6 +180,7 @@ Deno.test('an empty plan is allowed by a ledger exemption', async () => {
     result: {
       matrix: { include: [] },
       hasShards: false,
+      outOfScope: [],
       unmutated: [{
         package: '@fixture/core',
         dir: 'packages/core',
@@ -210,3 +215,74 @@ Deno.test('a mutation package without a plan is refused when a plan is required'
   assertEquals(outcome.ok, false)
   assertEquals(outcome.ok === false ? tagsOf(outcome.refusals) : [], ['PlanMissing'])
 })
+
+const CORE = { dir: 'packages/core', mutate: ['src/a.ts', 'src/b.ts'] }
+const SITE = { dir: 'packages/site', mutate: ['src/page.ts'] }
+
+Deno.test('a change scopes mutation to the declared files it touches and leaves untouched members out', () => {
+  assertEquals(scopeOf([CORE, SITE], ['packages/core/src/b.ts', 'README.md']), [
+    { _tag: 'ChangedFiles', project: 'packages/core', files: ['src/b.ts'] },
+  ])
+})
+
+Deno.test('a change to a member file outside its declared set mutates that member whole', () => {
+  for (const file of ['tests/a.test.ts', 'src/helper.ts', 'src/a.test.ts', 'stryker.config.ts', 'package.json']) {
+    assertEquals(scopeOf([CORE, SITE], ['packages/core/src/a.ts', `packages/core/${file}`]), [
+      { _tag: 'WholeSet', project: 'packages/core' },
+    ], file)
+  }
+})
+
+Deno.test('a change to the gate mutates every member whole, touched or not', () => {
+  for (
+    const file of ['.github/workflows/release-gate.yml', '.github/actions/sandbox/action.yml', 'stryker.shared.ts']
+  ) {
+    assertEquals(scopeOf([CORE, SITE], [file]), [
+      { _tag: 'WholeSet', project: 'packages/core' },
+      { _tag: 'WholeSet', project: 'packages/site' },
+    ], file)
+  }
+})
+
+Deno.test('a member directory that only prefixes another is not touched by it', () => {
+  assertEquals(scopeOf([CORE], ['packages/core-extra/src/a.ts']), [])
+})
+
+const CORE_CONFIG = "export default { mutate: ['src/a.ts'] }\n"
+
+Deno.test('a change that touches no mutated member passes the gate with no plan and names the members it left out', async () => {
+  const { root, planFile } = await writeFixture({
+    packages: [{ dir: 'packages/core', name: '@fixture/core', mutates: true, strykerConfig: CORE_CONFIG }],
+  })
+  assertEquals(await gatePlan({ root, planFile, changed: ['docs/notes.md'] }), {
+    ok: true,
+    result: {
+      matrix: { include: [] },
+      hasShards: false,
+      unmutated: [],
+      outOfScope: [{ package: '@fixture/core', dir: 'packages/core' }],
+    },
+  })
+})
+
+Deno.test('a change still refuses a touched mutated member the plan scheduled nothing for', async () => {
+  const { root, planFile } = await writeFixture({
+    packages: [
+      { dir: 'packages/core', name: '@fixture/core', mutates: true, strykerConfig: CORE_CONFIG },
+      { dir: 'packages/site', name: '@fixture/site', mutates: true, strykerConfig: CORE_CONFIG },
+    ],
+    plan: { ...TWO_PROJECT_PLAN, shards: [TWO_PROJECT_PLAN.shards[1]!] },
+  })
+  const outcome = await gatePlan({ root, planFile, changed: ['packages/core/src/a.ts'] })
+  assertEquals(outcome.ok ? outcome : outcome.refusals, [
+    { _tag: 'MutationPackageWithoutMutants', package: '@fixture/core' },
+  ])
+})
+
+Deno.test('a change touching a member whose stryker config declares no mutate list is refused', async () => {
+  const { root, planFile } = await writeFixture({
+    packages: [{ dir: 'packages/core', name: '@fixture/core', mutates: true, strykerConfig: 'export default {}\n' }],
+  })
+  const outcome = await gatePlan({ root, planFile, changed: ['packages/core/src/a.ts'] })
+  assertEquals(outcome.ok ? [] : tagsOf(outcome.refusals), ['MalformedStrykerConfig'])
+})