From 23cfd0082a2a69fa494b7e1a1b37821279bec503 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Fri, 9 Oct 2026 12:12:23 +0000 Subject: [PATCH 1/2] ci(repo): run the release gate on pull requests, scoped to the change Capability PRs never ran the mutation gate, so #77 merged src/graph with 45 live mutants (31 Survived, 14 NoCoverage) that main's gate then caught. The release gate now also runs on pull_request. The plan job lists the files the pull request changes (the merge commit against its first parent), and stryker-plan-gate.ts decides the scope: the declared files the change touches; a package's whole declared set when the change touches any other file in that package (a test, a fixture, a config, a source file outside the set); every package's whole set when it changes the gate (this workflow, the gate script, stryker.shared.ts, the lockfile, the Nix toolchain). The scope reaches Stryker as MUTATION_SCOPE, which stryker.shared.ts intersects with each package's declared mutate list; unset (local, main without a diff), the whole set is mutated. The gate refuses an in-scope package the plan scheduled nothing for, as before, and passes a change that touches no mutated package with no plan. The verdict job now runs whenever the workflow was not cancelled and fails unless the plan, and every shard when there are shards, finished, so a skipped mutation job can no longer read as a pass. Pull-request runs skip the incremental cache so every scoped mutant runs fresh, and a newer push cancels the older run. --- .github/workflows/release-gate.yml | 55 ++++++++--- CONTRIBUTING.md | 2 +- README.md | 20 ++-- scripts/stryker-plan-gate.test.ts | 78 +++++++++++++++- scripts/stryker-plan-gate.ts | 141 +++++++++++++++++++++++++++-- stryker.shared.ts | 13 ++- 6 files changed, 275 insertions(+), 34 deletions(-) diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 016088e..795a63f 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -3,6 +3,7 @@ name: Release gate on: push: branches: [main] + pull_request: workflow_dispatch: permissions: @@ -10,7 +11,7 @@ permissions: concurrency: group: release-gate-${{ github.ref }} - cancel-in-progress: false + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: plan: @@ -20,11 +21,16 @@ jobs: outputs: matrix: ${{ steps.publish.outputs.matrix }} has-shards: ${{ steps.publish.outputs.has-shards }} + scope: ${{ steps.projects.outputs.scope }} steps: - uses: actions/checkout@v7 with: + fetch-depth: 2 persist-credentials: false - uses: ./.github/actions/dev-shell + - name: List the files the pull request changes + if: github.event_name == 'pull_request' + run: mkdir -p .cache && git diff --name-only HEAD^1 HEAD > .cache/changed-files.txt - name: Discover mutation projects uses: ./.github/actions/sandbox with: @@ -32,21 +38,27 @@ jobs: jsr.io registry.npmjs.org command: >- - deno run --config=scripts/deno.json --allow-read --allow-write - scripts/stryker-plan-gate.ts projects --out .cache/mutation-projects.txt + deno run --config=scripts/deno.json --allow-read --allow-write --allow-env=MUTATION_SCOPE + scripts/stryker-plan-gate.ts projects --changed .cache/changed-files.txt + --out .cache/mutation-projects.txt --scope-out .cache/mutation-scope.txt - id: projects - name: Publish the project list - run: echo "projects=$(cat .cache/mutation-projects.txt)" >> "$GITHUB_OUTPUT" + name: Publish the project list and the mutation scope + run: | + echo "projects=$(cat .cache/mutation-projects.txt)" >> "$GITHUB_OUTPUT" + echo "scope=$(cat .cache/mutation-scope.txt)" >> "$GITHUB_OUTPUT" - name: Plan mutation shards if: steps.projects.outputs.projects != '' uses: ./.github/actions/sandbox env: MUTATION_PROJECTS: ${{ steps.projects.outputs.projects }} + MUTATION_SCOPE: ${{ steps.projects.outputs.scope }} with: hosts: | jsr.io registry.npmjs.org - pass-env: MUTATION_PROJECTS + pass-env: | + MUTATION_PROJECTS + MUTATION_SCOPE command: ./node_modules/.bin/stryker plan --target-seconds 900 --projects "$MUTATION_PROJECTS" --out stryker-plan.json - name: Gate the plan uses: ./.github/actions/sandbox @@ -55,8 +67,9 @@ jobs: jsr.io registry.npmjs.org command: >- - deno run --config=scripts/deno.json --allow-read --allow-write - scripts/stryker-plan-gate.ts gate --plan stryker-plan.json --out .cache/mutation-plan.out + deno run --config=scripts/deno.json --allow-read --allow-write --allow-env=MUTATION_SCOPE + scripts/stryker-plan-gate.ts gate --plan stryker-plan.json --changed .cache/changed-files.txt + --out .cache/mutation-plan.out - id: publish name: Publish the plan run: cat .cache/mutation-plan.out >> "$GITHUB_OUTPUT" @@ -78,6 +91,7 @@ jobs: matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} env: CI: "true" + MUTATION_SCOPE: ${{ needs.plan.outputs.scope }} steps: - uses: actions/checkout@v7 with: @@ -89,6 +103,7 @@ jobs: name: stryker-plan path: . - name: Stryker incremental cache + if: github.event_name != 'pull_request' uses: actions/cache@v6 with: path: | @@ -104,6 +119,7 @@ jobs: pass-env: | GITHUB_ACTIONS MUTATION_SHARD + MUTATION_SCOPE command: ./node_modules/.bin/stryker run --plan stryker-plan.json --shard "$MUTATION_SHARD" - if: always() uses: actions/upload-artifact@v6 @@ -120,26 +136,41 @@ jobs: verdict: name: verdict · merged report - needs: [mutation] + needs: [plan, mutation] + if: ${{ !cancelled() }} runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v7 + - name: Refuse a plan or a shard that did not finish + env: + PLAN: ${{ needs.plan.result }} + MUTATION: ${{ needs.mutation.result }} + HAS_SHARDS: ${{ needs.plan.outputs.has-shards }} + run: | + if [ "$PLAN" != success ]; then echo "::error::the plan job ended $PLAN"; exit 1; fi + if [ "$HAS_SHARDS" != true ]; then echo "no mutated file is in this change's scope"; exit 0; fi + if [ "$MUTATION" != success ]; then echo "::error::a mutation shard ended $MUTATION"; exit 1; fi + - if: needs.plan.outputs.has-shards == 'true' + uses: actions/checkout@v7 with: persist-credentials: false - - uses: ./.github/actions/dev-shell + - if: needs.plan.outputs.has-shards == 'true' + uses: ./.github/actions/dev-shell - name: Download the shard plan + if: needs.plan.outputs.has-shards == 'true' uses: actions/download-artifact@v6 with: name: stryker-plan path: . - name: Download every shard's reports + if: needs.plan.outputs.has-shards == 'true' uses: actions/download-artifact@v6 with: pattern: mutation-shard-* path: .cache/shard-reports merge-multiple: true - name: Merge the shard reports and gate them at zero survivors + if: needs.plan.outputs.has-shards == 'true' uses: ./.github/actions/sandbox with: command: | @@ -149,7 +180,7 @@ jobs: mapfile -t shard_dirs < .cache/shard-dirs.txt ./node_modules/.bin/stryker merge --plan stryker-plan.json --out reports/mutation "${shard_dirs[@]}" ./node_modules/.bin/stryker gate --baseline .cache/no-survivors.json - - if: always() + - if: always() && needs.plan.outputs.has-shards == 'true' uses: actions/upload-artifact@v6 with: name: mutation-report diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 07839aa..590edcf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,7 +51,7 @@ pnpm check:ci nix build .#workspace-tarballs ``` -Mutation testing is not part of `pnpm check:ci`. The release gate (`.github/workflows/release-gate.yml`) runs `stryker plan` once over every workspace package that declares a `mutation` script, then `stryker run` for each planned shard at a break threshold of 100 on every push to `main`. +Mutation testing is not part of `pnpm check:ci`. The release gate (`.github/workflows/release-gate.yml`) runs `stryker plan` once over every workspace package that declares a `mutation` script, then `stryker run` for each planned shard at a break threshold of 100 on every push to `main`. The same workflow runs on every pull request, scoped to what the pull request changes. It mutates the declared files the pull request touches. It mutates a package's whole declared set when the pull request touches any other file in that package: a test, a fixture, a config, or a source file outside the set. It mutates every package's whole set when the pull request changes the gate itself: the workflow, `scripts/stryker-plan-gate.ts`, `stryker.shared.ts`, the lockfile or the Nix toolchain. A pull request that touches no mutated package plans nothing, and its verdict job passes. Pull-request runs skip the incremental cache, so every scoped mutant runs fresh. ## Pull Requests & Commits diff --git a/README.md b/README.md index b6f344e..bcda25f 100644 --- a/README.md +++ b/README.md @@ -23,16 +23,16 @@ Each package is also its own flake attribute, named after the last segment of it ## Toolchain -| Tool | Role | -| ----------------------- | ---------------------------------------------------------------------------------- | -| **Nix** | Pins Node.js 24, pnpm 12.9.0, Deno and dprint, and builds the package tarballs | -| **Sandbox** | pnpm-release-management's deny-by-default launcher; all dependency code runs in it | -| **pnpm + Turbo** | Workspace catalog with exact pins, cached task graph | -| **TypeScript 7 (tsgo)** | Typechecking through `@effect/tsgo` | -| **oxlint** | The `@systemfsoftware/oxlint-config-recommended` preset, at error severity | -| **Vitest** | Unit and integration tests | -| **Stryker** | Mutation testing at a break threshold of 100, on `main` only (the release gate) | -| **dprint** | Formatting for code and Markdown | +| Tool | Role | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------- | +| **Nix** | Pins Node.js 24, pnpm 12.9.0, Deno and dprint, and builds the package tarballs | +| **Sandbox** | pnpm-release-management's deny-by-default launcher; all dependency code runs in it | +| **pnpm + Turbo** | Workspace catalog with exact pins, cached task graph | +| **TypeScript 7 (tsgo)** | Typechecking through `@effect/tsgo` | +| **oxlint** | The `@systemfsoftware/oxlint-config-recommended` preset, at error severity | +| **Vitest** | Unit and integration tests | +| **Stryker** | Mutation testing at a break threshold of 100, on `main` and scoped to each pull request's change (the release gate) | +| **dprint** | Formatting for code and Markdown | ## Contributing diff --git a/scripts/stryker-plan-gate.test.ts b/scripts/stryker-plan-gate.test.ts index d07756a..cc8f9e6 100644 --- a/scripts/stryker-plan-gate.test.ts +++ b/scripts/stryker-plan-gate.test.ts @@ -1,11 +1,12 @@ import { assertEquals } from '@std/assert' import { join } from '@std/path' -import { gatePlan, type Refusal, type ShardPlan } from './stryker-plan-gate.ts' +import { gatePlan, type Refusal, scopeOf, type ShardPlan } from './stryker-plan-gate.ts' interface FixturePackage { readonly dir: string readonly name: string readonly mutates: boolean + readonly strykerConfig?: string } interface FixtureOptions { @@ -27,6 +28,7 @@ const writeFixture = async (options: FixtureOptions): Promise<{ root: string; pl ...(pkg.mutates ? { scripts: { mutation: 'stryker run' } } : {}), } await Deno.writeTextFile(join(dir, 'package.json'), `${JSON.stringify(manifest, null, 2)}\n`) + if (pkg.strykerConfig !== undefined) await Deno.writeTextFile(join(dir, 'stryker.config.ts'), pkg.strykerConfig) } if (options.plan !== undefined) { await Deno.writeTextFile(join(root, 'plan.json'), `${JSON.stringify(options.plan)}\n`) @@ -68,6 +70,7 @@ Deno.test('a valid plan yields the plan matrix and has-shards', async () => { matrix: { include: [{ shard: '1/2', predictedSeconds: 5 }, { shard: '2/2', predictedSeconds: 3 }] }, hasShards: true, unmutated: [], + outOfScope: [], }, }) }) @@ -150,6 +153,7 @@ Deno.test('a mutation package with no scheduled mutants is allowed by a ledger e result: { matrix: { include: [{ shard: '1/1', predictedSeconds: 1 }] }, hasShards: true, + outOfScope: [], unmutated: [{ package: '@fixture/site', dir: 'packages/site', @@ -176,6 +180,7 @@ Deno.test('an empty plan is allowed by a ledger exemption', async () => { result: { matrix: { include: [] }, hasShards: false, + outOfScope: [], unmutated: [{ package: '@fixture/core', dir: 'packages/core', @@ -210,3 +215,74 @@ Deno.test('a mutation package without a plan is refused when a plan is required' assertEquals(outcome.ok, false) assertEquals(outcome.ok === false ? tagsOf(outcome.refusals) : [], ['PlanMissing']) }) + +const CORE = { dir: 'packages/core', mutate: ['src/a.ts', 'src/b.ts'] } +const SITE = { dir: 'packages/site', mutate: ['src/page.ts'] } + +Deno.test('a change scopes mutation to the declared files it touches and leaves untouched members out', () => { + assertEquals(scopeOf([CORE, SITE], ['packages/core/src/b.ts', 'README.md']), [ + { _tag: 'ChangedFiles', project: 'packages/core', files: ['src/b.ts'] }, + ]) +}) + +Deno.test('a change to a member file outside its declared set mutates that member whole', () => { + for (const file of ['tests/a.test.ts', 'src/helper.ts', 'src/a.test.ts', 'stryker.config.ts', 'package.json']) { + assertEquals(scopeOf([CORE, SITE], ['packages/core/src/a.ts', `packages/core/${file}`]), [ + { _tag: 'WholeSet', project: 'packages/core' }, + ], file) + } +}) + +Deno.test('a change to the gate mutates every member whole, touched or not', () => { + for ( + const file of ['.github/workflows/release-gate.yml', '.github/actions/sandbox/action.yml', 'stryker.shared.ts'] + ) { + assertEquals(scopeOf([CORE, SITE], [file]), [ + { _tag: 'WholeSet', project: 'packages/core' }, + { _tag: 'WholeSet', project: 'packages/site' }, + ], file) + } +}) + +Deno.test('a member directory that only prefixes another is not touched by it', () => { + assertEquals(scopeOf([CORE], ['packages/core-extra/src/a.ts']), []) +}) + +const CORE_CONFIG = "export default { mutate: ['src/a.ts'] }\n" + +Deno.test('a change that touches no mutated member passes the gate with no plan and names the members it left out', async () => { + const { root, planFile } = await writeFixture({ + packages: [{ dir: 'packages/core', name: '@fixture/core', mutates: true, strykerConfig: CORE_CONFIG }], + }) + assertEquals(await gatePlan({ root, planFile, changed: ['docs/notes.md'] }), { + ok: true, + result: { + matrix: { include: [] }, + hasShards: false, + unmutated: [], + outOfScope: [{ package: '@fixture/core', dir: 'packages/core' }], + }, + }) +}) + +Deno.test('a change still refuses a touched mutated member the plan scheduled nothing for', async () => { + const { root, planFile } = await writeFixture({ + packages: [ + { dir: 'packages/core', name: '@fixture/core', mutates: true, strykerConfig: CORE_CONFIG }, + { dir: 'packages/site', name: '@fixture/site', mutates: true, strykerConfig: CORE_CONFIG }, + ], + plan: { ...TWO_PROJECT_PLAN, shards: [TWO_PROJECT_PLAN.shards[1]!] }, + }) + const outcome = await gatePlan({ root, planFile, changed: ['packages/core/src/a.ts'] }) + assertEquals(outcome.ok ? outcome : outcome.refusals, [ + { _tag: 'MutationPackageWithoutMutants', package: '@fixture/core' }, + ]) +}) + +Deno.test('a change touching a member whose stryker config declares no mutate list is refused', async () => { + const { root, planFile } = await writeFixture({ + packages: [{ dir: 'packages/core', name: '@fixture/core', mutates: true, strykerConfig: 'export default {}\n' }], + }) + const outcome = await gatePlan({ root, planFile, changed: ['packages/core/src/a.ts'] }) + assertEquals(outcome.ok ? [] : tagsOf(outcome.refusals), ['MalformedStrykerConfig']) +}) diff --git a/scripts/stryker-plan-gate.ts b/scripts/stryker-plan-gate.ts index e75c0fc..de7c2d4 100755 --- a/scripts/stryker-plan-gate.ts +++ b/scripts/stryker-plan-gate.ts @@ -1,7 +1,7 @@ -#!/usr/bin/env -S deno run --config=scripts/deno.json --allow-read --allow-write +#!/usr/bin/env -S deno run --config=scripts/deno.json --allow-read --allow-write --allow-env=MUTATION_SCOPE import { parseArgs } from '@std/cli/parse-args' -import { dirname, join, relative, resolve } from '@std/path' +import { dirname, join, relative, resolve, toFileUrl } from '@std/path' import { parse as parseYaml } from '@std/yaml' import { Schema as S } from 'effect' import * as Result from 'effect/Result' @@ -55,10 +55,13 @@ const LedgerEntry = S.Struct({ const Ledger = S.Struct({ entries: S.Array(LedgerEntry) }) +const StrykerConfigModule = S.Struct({ default: S.Struct({ mutate: S.Array(S.String) }) }) + const MUTATION_EXEMPTION_RULE = 'XS1' export type Refusal = | { readonly _tag: 'MalformedWorkspace'; readonly file: string } + | { readonly _tag: 'MalformedStrykerConfig'; readonly file: string; readonly reason: string } | { readonly _tag: 'MalformedManifest'; readonly file: string } | { readonly _tag: 'MalformedLedger'; readonly file: string } | { readonly _tag: 'PlanMissing'; readonly file: string } @@ -71,6 +74,8 @@ export const renderRefusal = (refusal: Refusal): string => { switch (refusal._tag) { case 'MalformedWorkspace': return `malformed pnpm-workspace.yaml (${refusal.file})` + case 'MalformedStrykerConfig': + return `stryker config ${refusal.file} has no default export with a \`mutate\` file list: ${refusal.reason}` case 'MalformedManifest': return `malformed package manifest (${refusal.file})` case 'MalformedLedger': @@ -184,6 +189,92 @@ export const discoverMembers = async (root: string): Promise => { return { ok: true, members } } +export const GATE_FILES: readonly string[] = [ + '.github/workflows/release-gate.yml', + 'scripts/stryker-plan-gate.ts', + 'scripts/deno.json', + 'scripts/deno.lock', + 'stryker.shared.ts', + 'package.json', + 'pnpm-lock.yaml', + 'pnpm-workspace.yaml', + 'flake.nix', + 'flake.lock', +] + +const GATE_DIRECTORIES: readonly string[] = ['.github/actions/'] + +const changesGate = (file: string): boolean => + GATE_FILES.includes(file) || GATE_DIRECTORIES.some((directory) => file.startsWith(directory)) + +export interface ScopedMember { + readonly dir: string + readonly mutate: readonly string[] +} + +export type ProjectScope = + | { readonly _tag: 'WholeSet'; readonly project: string } + | { readonly _tag: 'ChangedFiles'; readonly project: string; readonly files: readonly string[] } + +export const scopeOf = (members: readonly ScopedMember[], changed: readonly string[]): readonly ProjectScope[] => + members.flatMap((member): ProjectScope[] => { + const whole: ProjectScope = { _tag: 'WholeSet', project: member.dir } + if (changed.some(changesGate)) return [whole] + const prefix = `${member.dir}/` + const touched = changed.filter((file) => file.startsWith(prefix)).map((file) => file.slice(prefix.length)) + if (touched.length === 0) return [] + return touched.every((file) => member.mutate.includes(file)) + ? [{ _tag: 'ChangedFiles', project: member.dir, files: touched }] + : [whole] + }) + +export const renderScope = (scope: readonly ProjectScope[]): string => + scope.flatMap((entry) => + entry._tag === 'WholeSet' ? [entry.project] : entry.files.map((file) => `${entry.project}/${file}`) + ).join(',') + +const declaredMutate = async ( + root: string, + member: Member, +): Promise> => { + const file = join(root, member.dir, 'stryker.config.ts') + try { + const decoded = S.decodeUnknownResult(StrykerConfigModule)(await import(toFileUrl(resolve(file)).href)) + return Result.isFailure(decoded) + ? Result.fail({ _tag: 'MalformedStrykerConfig', file, reason: decoded.failure.message }) + : Result.succeed({ dir: member.dir, mutate: decoded.success.default.mutate }) + } catch (error) { + return Result.fail({ _tag: 'MalformedStrykerConfig', file, reason: String(error) }) + } +} + +export type ScopeOutcome = + | { readonly ok: true; readonly scope: readonly ProjectScope[] } + | { readonly ok: false; readonly refusal: Refusal } + +export const resolveScope = async ( + root: string, + members: readonly Member[], + changed: readonly string[] | undefined, +): Promise => { + const mutating = members.filter((member) => member.mutates) + if (changed === undefined) { + return { ok: true, scope: mutating.map((member) => ({ _tag: 'WholeSet', project: member.dir })) } + } + const scoped: ScopedMember[] = [] + for (const member of mutating) { + const declared = await declaredMutate(root, member) + if (Result.isFailure(declared)) return { ok: false, refusal: declared.failure } + scoped.push(declared.success) + } + return { ok: true, scope: scopeOf(scoped, changed) } +} + +export const readChanged = async (file: string | undefined): Promise => { + const text = file === undefined ? undefined : await readText(file) + return text?.split('\n').map((line) => line.trim()).filter((line) => line !== '') +} + type LedgerRead = | { readonly ok: true; readonly entries: ReadonlyArray } | { readonly ok: false; readonly refusal: Refusal } @@ -226,6 +317,7 @@ const decodePlan = (file: string, text: string): PlanRead => { export interface GateInput { readonly root: string readonly planFile: string + readonly changed?: readonly string[] | undefined } export interface Unmutated { @@ -234,10 +326,16 @@ export interface Unmutated { readonly exemption: typeof LedgerEntry.Type | undefined } +export interface OutOfScope { + readonly package: string + readonly dir: string +} + export interface GateResult { readonly matrix: ShardPlan['matrix'] readonly hasShards: boolean readonly unmutated: readonly Unmutated[] + readonly outOfScope: readonly OutOfScope[] } export const renderUnmutated = (member: Unmutated): string => @@ -245,19 +343,28 @@ export const renderUnmutated = (member: Unmutated): string => ? `0 mutants for ${member.package} (${member.dir}): no mutation script and no ${MUTATION_EXEMPTION_RULE} debt-ledger entry` : `0 mutants for ${member.package} (${member.dir}): accepted by ${MUTATION_EXEMPTION_RULE} debt-ledger entry "${member.exemption.reason}", removed by ${member.exemption.removedBy}` +export const renderOutOfScope = (member: OutOfScope): string => + `0 mutants for ${member.package} (${member.dir}): the change touches none of its files` + export type GateOutcome = | { readonly ok: true; readonly result: GateResult } | { readonly ok: false; readonly refusals: readonly Refusal[] } -export const gatePlan = async ({ root, planFile }: GateInput): Promise => { +export const gatePlan = async ({ root, planFile, changed }: GateInput): Promise => { const discovery = await discoverMembers(root) if (!discovery.ok) return { ok: false, refusals: [discovery.refusal] } const ledger = await readLedger(root) if (!ledger.ok) return { ok: false, refusals: [ledger.refusal] } + const resolved = await resolveScope(root, discovery.members, changed) + if (!resolved.ok) return { ok: false, refusals: [resolved.refusal] } + const inScope = new Set(resolved.scope.map((entry) => entry.project)) const resolvedPlanFile = resolve(root, planFile) const planText = await readText(resolvedPlanFile) - const mutationMembers = discovery.members.filter((member) => member.mutates) + const mutationMembers = discovery.members.filter((member) => member.mutates && inScope.has(member.dir)) + const outOfScope = discovery.members + .filter((member) => member.mutates && !inScope.has(member.dir)) + .map((member) => ({ package: member.name, dir: member.dir })) if (planText === undefined && mutationMembers.length > 0) { return { ok: false, refusals: [{ _tag: 'PlanMissing', file: resolvedPlanFile }] } } @@ -289,22 +396,31 @@ export const gatePlan = async ({ root, planFile }: GateInput): Promise total + count, 0) - if (scheduledTotal === 0 && ledger.entries.length === 0) refusals.push({ _tag: 'VacuousPlan' }) + const scopeIsEmpty = changed !== undefined && inScope.size === 0 + if (scheduledTotal === 0 && ledger.entries.length === 0 && !scopeIsEmpty) refusals.push({ _tag: 'VacuousPlan' }) if (refusals.length > 0) return { ok: false, refusals } const unmutated = discovery.members .filter((member) => (scheduled.get(member.dir) ?? 0) === 0) + .filter((member) => !outOfScope.some((out) => out.dir === member.dir)) .map((member) => ({ package: member.name, dir: member.dir, exemption: exemption(member.name) })) - return { ok: true, result: { matrix: plan.matrix, hasShards: plan.matrix.include.length > 0, unmutated } } + return { + ok: true, + result: { matrix: plan.matrix, hasShards: plan.matrix.include.length > 0, unmutated, outOfScope }, + } } if (import.meta.main) { - const args = parseArgs(Deno.args, { string: ['root', 'plan', 'out'], default: { root: '.' } }) + const args = parseArgs(Deno.args, { + string: ['root', 'plan', 'out', 'changed', 'scope-out'], + default: { root: '.' }, + }) const mode = args._[0] const emit = async (text: string): Promise => { if (args.out === undefined) console.log(text.trimEnd()) else await Deno.writeTextFile(args.out, text) } + const changed = await readChanged(args.changed) if (mode === 'projects') { const discovery = await discoverMembers(args.root) @@ -312,8 +428,14 @@ if (import.meta.main) { console.error(`stryker-plan-gate: ${renderRefusal(discovery.refusal)}`) Deno.exit(1) } - const projects = discovery.members.filter((member) => member.mutates).map((member) => member.dir).sort() + const resolved = await resolveScope(args.root, discovery.members, changed) + if (!resolved.ok) { + console.error(`stryker-plan-gate: ${renderRefusal(resolved.refusal)}`) + Deno.exit(1) + } + const projects = resolved.scope.map((entry) => entry.project).sort() await emit(`${projects.join(',')}\n`) + if (args['scope-out'] !== undefined) await Deno.writeTextFile(args['scope-out'], `${renderScope(resolved.scope)}\n`) Deno.exit(0) } @@ -321,12 +443,13 @@ if (import.meta.main) { console.error('stryker-plan-gate: gate needs --plan ') Deno.exit(2) } - const outcome = await gatePlan({ root: args.root, planFile: args.plan }) + const outcome = await gatePlan({ root: args.root, planFile: args.plan, changed }) if (!outcome.ok) { for (const refusal of outcome.refusals) console.error(`stryker-plan-gate: ${renderRefusal(refusal)}`) Deno.exit(1) } for (const member of outcome.result.unmutated) console.error(`stryker-plan-gate: ${renderUnmutated(member)}`) + for (const member of outcome.result.outOfScope) console.error(`stryker-plan-gate: ${renderOutOfScope(member)}`) console.error(`stryker-plan-gate: ${outcome.result.matrix.include.length} shard(s)`) await emit(`matrix=${JSON.stringify(outcome.result.matrix)}\nhas-shards=${outcome.result.hasShards}\n`) Deno.exit(0) diff --git a/stryker.shared.ts b/stryker.shared.ts index f41e2c8..a4acd6b 100644 --- a/stryker.shared.ts +++ b/stryker.shared.ts @@ -1,7 +1,18 @@ import type { StrykerConfig } from '@systemfsoftware/stryker-js/config' +import { relative } from 'node:path' +import { fileURLToPath } from 'node:url' const slackForAContendedFullSuiteRunMs = 45_000 +const repoRoot = fileURLToPath(new URL('.', import.meta.url)) + +const scopedMutate = (mutate: ReadonlyArray, scope: string | undefined): string[] => { + if (scope === undefined) return [...mutate] + const entries = scope.split(',') + const project = relative(repoRoot, process.cwd()).replaceAll('\\', '/') + return entries.includes(project) ? [...mutate] : mutate.filter((file) => entries.includes(`${project}/${file}`)) +} + export const packageStrykerConfig = (mutate: ReadonlyArray): StrykerConfig => ({ checkers: [{ plugin: '@systemfsoftware/stryker-js-typescript-checker' }], @@ -13,7 +24,7 @@ export const packageStrykerConfig = (mutate: ReadonlyArray): StrykerConf incrementalFile: 'reports/stryker-incremental.json', ignorers: ['@systemfsoftware/stryker-ignorer-effect-schema-declarations'], jsonReporter: { fileName: 'reports/mutation-report.json' }, - mutate: [...mutate], + mutate: scopedMutate(mutate, process.env['MUTATION_SCOPE']), packageManager: 'pnpm', reporters: ['progress', 'html', 'json', 'progress-stream'], testRunner: { From 42a4aed58539f4aa510202d58ffe4ba6c5a44b98 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Fri, 9 Oct 2026 13:17:10 +0000 Subject: [PATCH 2/2] ci(repo): keep a failed mutation shard's own output When a shard child fails, the shard runner prints only the first 1024 characters of the child's stderr, and those are startup INFO lines. The child's error envelope goes to its stdout, which the runner discards. #79's gate runs 37930507588 and 37933520982 both failed with exit 3 and left nothing that names the cause. Every package's Stryker config now writes stryker.log at info level. When the mutation job fails, a follow-up step re-runs each of the shard's project runs directly, with the same arguments the shard runner passes, keeping stdout, stderr and the exit code under reports/diagnostics/. The job then uploads that directory and every stryker.log as mutation-diagnostics-shard-N. A green job runs neither step. --- .github/workflows/release-gate.yml | 35 ++++++++++++++++++++++++++++++ stryker.shared.ts | 1 + 2 files changed, 36 insertions(+) diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 795a63f..5fff451 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -133,6 +133,41 @@ jobs: name: mutation-report-shard-${{ strategy.job-index }} path: packages/*/reports/mutation-report.* if-no-files-found: ignore + - name: Re-run a failed shard's project runs with their output kept + if: failure() + timeout-minutes: 20 + uses: ./.github/actions/sandbox + env: + MUTATION_SHARD: ${{ matrix.shard }} + with: + pass-env: | + GITHUB_ACTIONS + MUTATION_SHARD + MUTATION_SCOPE + command: | + set -uo pipefail + root="$PWD" + jq -r --arg shard "$MUTATION_SHARD" \ + '.shards[] | select("\(.index)/\(.count)" == $shard) | .projects[].project' \ + stryker-plan.json > .cache/shard-projects.txt + while read -r project; do + out="$root/reports/diagnostics/$project" + mkdir -p "$out" + (cd "$project" && "$root/node_modules/.bin/stryker" run --plan "$root/stryker-plan.json" \ + --shard "$MUTATION_SHARD" --project "$project" \ + --progressStreamFile "$out/mutation-stream.jsonl" \ + --incremental --incrementalFile "$out/stryker-incremental.json" \ + > "$out/stdout.log" 2> "$out/stderr.log") + echo "$project exited $?" | tee "$out/exit.txt" + done < .cache/shard-projects.txt + - if: failure() + uses: actions/upload-artifact@v6 + with: + name: mutation-diagnostics-shard-${{ strategy.job-index }} + path: | + reports/diagnostics/ + packages/*/stryker.log + if-no-files-found: ignore verdict: name: verdict · merged report diff --git a/stryker.shared.ts b/stryker.shared.ts index a4acd6b..c051663 100644 --- a/stryker.shared.ts +++ b/stryker.shared.ts @@ -19,6 +19,7 @@ export const packageStrykerConfig = (mutate: ReadonlyArray): StrykerConf coverageAnalysis: 'perTest', disableBail: true, htmlReporter: { fileName: 'reports/mutation-report.html' }, + fileLogLevel: 'info', ignorePatterns: ['reports', 'coverage', 'dist'], incremental: true, incrementalFile: 'reports/stryker-incremental.json',