diff --git a/.github/actions/nextest/action.yml b/.github/actions/nextest/action.yml new file mode 100644 index 0000000..56547ca --- /dev/null +++ b/.github/actions/nextest/action.yml @@ -0,0 +1,26 @@ +name: Run the nextest action fixture +description: Repository-local entry point used by the reusable workflow tests. +inputs: + partition: + default: "1" + description: One-based partition number. + partitions: + default: "1" + description: Total number of partitions. + save-cache: + default: "false" + description: Unused by this small fixture. +runs: + using: composite + steps: + - name: Install Rust + shell: bash + run: rustup toolchain install stable --profile minimal + # Exercise the action from this test checkout, including unmerged changes. + - uses: ./nextest # zizmor: ignore[self-repository] + with: + name: fixture-${{ runner.os }}-${{ inputs.partitions }} + partition: ${{ inputs.partition }} + partitions: ${{ inputs.partitions }} + working-directory: nextest/tests/fixture + arguments: '["--profile", "ci"]' diff --git a/.github/workflows/nextest.yml b/.github/workflows/nextest.yml new file mode 100644 index 0000000..78b0549 --- /dev/null +++ b/.github/workflows/nextest.yml @@ -0,0 +1,56 @@ +name: Sharded nextest + +on: + workflow_call: + inputs: + runs-on: + description: Runner label for each shard. + type: string + default: ubuntu-latest + shards: + description: Number of independent nextest hash partitions. + type: number + default: 1 + timeout-minutes: + description: Timeout for each shard. + type: number + default: 20 + save-cache: + description: Whether the repository action may save its build cache. + type: boolean + default: true + +permissions: + contents: read + +jobs: + plan: + name: plan shards + runs-on: ubuntu-slim + outputs: + partitions: ${{ steps.plan.outputs.partitions }} + steps: + - uses: astral-sh/github-actions/nextest/plan@e6fe1b51adba9d0b2c847e503fce90e8fb765188 + id: plan + with: + shards: ${{ inputs.shards }} + + run: + needs: plan + name: shard ${{ matrix.partition }} of ${{ inputs.shards }} + runs-on: ${{ inputs.runs-on }} + timeout-minutes: ${{ inputs.timeout-minutes }} + strategy: + fail-fast: false + matrix: + partition: ${{ fromJSON(needs.plan.outputs.partitions) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # This action belongs to the checked-out caller, not this workflow repository. + - uses: ./.github/actions/nextest # zizmor: ignore[self-repository] + with: + partition: ${{ matrix.partition }} + partitions: ${{ inputs.shards }} + save-cache: ${{ inputs.save-cache }} diff --git a/.github/workflows/test-nextest.yml b/.github/workflows/test-nextest.yml new file mode 100644 index 0000000..b8b3c67 --- /dev/null +++ b/.github/workflows/test-nextest.yml @@ -0,0 +1,62 @@ +name: Test sharded nextest + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + unit: + name: unit tests on ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: node --test nextest/tests/options.test.mjs + + linux: + name: nextest on linux + uses: $/.github/workflows/nextest.yml + with: + runs-on: ubuntu-latest + shards: 2 + macos: + name: nextest on macos + uses: $/.github/workflows/nextest.yml + with: + runs-on: macos-latest + shards: 2 + windows: + name: nextest on windows + uses: $/.github/workflows/nextest.yml + with: + runs-on: windows-latest + shards: 2 + unsharded: + name: unsharded nextest on linux + uses: $/.github/workflows/nextest.yml + with: + runs-on: ubuntu-latest + shards: 1 + + coverage: + name: check test coverage + needs: [linux, macos, windows, unsharded] + runs-on: ubuntu-slim + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: junit-results-fixture-* + path: test-results + - run: python3 nextest/tests/check-coverage.py test-results diff --git a/README.md b/README.md index 4c9ab7b..4bc1fd3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ # Astral GitHub Actions Shared GitHub Actions for Astral projects. + +- [Sharded nextest](nextest/README.md) +- [Release smoke tests](release-smoke-test/README.md) diff --git a/nextest/README.md b/nextest/README.md new file mode 100644 index 0000000..495a33b --- /dev/null +++ b/nextest/README.md @@ -0,0 +1,54 @@ +# Sharded nextest + +Use `.github/workflows/nextest.yml` to fan a numeric shard count out into +independent runner jobs. Every shard checks out the caller and invokes its +`.github/actions/nextest` composite action with `partition`, `partitions`, and +`save-cache` inputs. That repository-local action owns project-specific toolchain, +cache, feature, and test-fixture configuration. + +```yaml +jobs: + test-windows: + name: cargo test on windows + uses: astral-sh/github-actions/.github/workflows/nextest.yml@ + with: + runs-on: windows-latest + shards: 3 +``` + +The workflow accepts 1–256 shards and disables matrix fail-fast so a failing +shard does not cancel the others. Its small planning job runs on `ubuntu-slim`. +Set the caller job's `name` to identify the suite; the workflow names its jobs +`plan shards` and `shard N of M`, including `shard 1 of 1` for unsharded runs. +The caller should gate the workflow as usual and grant only `contents: read`. +The checkout and repository-local action run with the caller's permissions. + +The repository-local action can use `astral-sh/github-actions/nextest@` +after preparing its environment: + +```yaml +- uses: astral-sh/github-actions/nextest@ + with: + name: windows + partition: ${{ inputs.partition }} + partitions: ${{ inputs.partitions }} + arguments: '["--workspace", "--profile", "ci"]' +``` + +The action installs nextest, passes the JSON argument array directly to Cargo, +and adds `--partition hash:N/T --no-tests=pass` when `T > 1`. Empty hash +partitions succeed; one shard runs the unpartitioned suite with nextest's normal +empty-suite behavior. It sets `INSTA_UPDATE=new` and `INSTA_PENDING_DIR`, uploads pending +snapshots after a failure, and uploads JUnit results even when tests fail. +Artifact names are `pending-snapshots-NAME[-N]` and `junit-results-NAME[-N]`; +the numeric suffix is omitted for one shard. Each independent suite in a +workflow run needs a distinct `name`. + +`working-directory` selects the Cargo workspace. `junit-path` is relative to that +directory; `snapshots-path` is relative to the original checkout. Both accept +absolute paths. Artifacts default to 14-day retention. The nextest profile must +enable JUnit output if a JUnit artifact is required. + +When moving jobs into this workflow, make any job-name-dependent build-cache +keys explicit. GitHub's aggregate reusable-workflow result includes every shard, +but individual check names acquire another reusable-workflow level. diff --git a/nextest/action.yml b/nextest/action.yml new file mode 100644 index 0000000..3852a5e --- /dev/null +++ b/nextest/action.yml @@ -0,0 +1,63 @@ +name: Run nextest shard +description: Run one nextest hash partition and collect its test artifacts. +inputs: + name: + description: Unique artifact-name component for this test suite. + required: true + partition: + description: One-based partition number. + default: "1" + partitions: + description: Total number of partitions. + default: "1" + arguments: + description: JSON array of arguments for cargo nextest run. + default: "[]" + working-directory: + description: Cargo workspace directory, relative to the checkout or absolute. + default: "." + junit-path: + description: JUnit glob relative to the Cargo workspace. + default: "target/nextest/**/junit.xml" + snapshots-path: + description: Pending insta snapshot directory relative to the checkout. + default: "pending-snapshots" + retention-days: + description: Number of days to retain test artifacts. + default: "14" +runs: + using: composite + steps: + - name: Install cargo nextest + uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2.85.10 + with: + tool: cargo-nextest + - name: Cargo test + id: run + # A separate immutable helper pin supports older self-hosted runner parsers. + uses: astral-sh/github-actions/nextest/run@e6fe1b51adba9d0b2c847e503fce90e8fb765188 + with: + name: ${{ inputs.name }} + partition: ${{ inputs.partition }} + partitions: ${{ inputs.partitions }} + arguments: ${{ inputs.arguments }} + working-directory: ${{ inputs.working-directory }} + junit-path: ${{ inputs.junit-path }} + snapshots-path: ${{ inputs.snapshots-path }} + - name: Upload pending snapshots + if: ${{ failure() && steps.run.outputs.snapshots-name != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.run.outputs.snapshots-name }} + path: ${{ steps.run.outputs.snapshots-path }} + if-no-files-found: ignore + include-hidden-files: true + retention-days: ${{ inputs.retention-days }} + - name: Upload JUnit results + if: ${{ always() && steps.run.outputs.junit-name != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.run.outputs.junit-name }} + path: ${{ steps.run.outputs.junit-path }} + if-no-files-found: ignore + retention-days: ${{ inputs.retention-days }} diff --git a/nextest/plan/action.yml b/nextest/plan/action.yml new file mode 100644 index 0000000..5bd5f17 --- /dev/null +++ b/nextest/plan/action.yml @@ -0,0 +1,12 @@ +name: Plan nextest shards +description: Convert a positive shard count into a GitHub Actions matrix. +inputs: + shards: + description: Number of shards, from 1 through 256. + required: true +outputs: + partitions: + description: JSON array of one-based partition numbers. +runs: + using: node24 + main: index.mjs diff --git a/nextest/plan/index.mjs b/nextest/plan/index.mjs new file mode 100644 index 0000000..c804978 --- /dev/null +++ b/nextest/plan/index.mjs @@ -0,0 +1,14 @@ +import { appendFileSync } from "node:fs"; +import { partitionCount } from "../run/options.mjs"; + +try { + const count = partitionCount(process.env.INPUT_SHARDS); + const partitions = Array.from({ length: count }, (_, index) => index + 1); + appendFileSync( + process.env.GITHUB_OUTPUT, + `partitions=${JSON.stringify(partitions)}\n`, + ); +} catch (error) { + console.error(error.message); + process.exitCode = 1; +} diff --git a/nextest/run/action.yml b/nextest/run/action.yml new file mode 100644 index 0000000..6fef45d --- /dev/null +++ b/nextest/run/action.yml @@ -0,0 +1,36 @@ +name: Invoke nextest +description: Validate shard inputs and invoke cargo nextest without a shell. +inputs: + name: + required: true + description: Artifact-name component. + partition: + default: "1" + description: One-based partition number. + partitions: + default: "1" + description: Total number of partitions. + arguments: + default: "[]" + description: JSON array of nextest arguments. + working-directory: + default: "." + description: Cargo workspace directory. + junit-path: + default: "target/nextest/**/junit.xml" + description: JUnit glob relative to the Cargo workspace. + snapshots-path: + default: "pending-snapshots" + description: Pending snapshot directory relative to the checkout. +outputs: + junit-name: + description: Unique JUnit artifact name. + junit-path: + description: Absolute JUnit artifact glob. + snapshots-name: + description: Unique pending-snapshot artifact name. + snapshots-path: + description: Absolute pending-snapshot directory. +runs: + using: node24 + main: index.mjs diff --git a/nextest/run/index.mjs b/nextest/run/index.mjs new file mode 100644 index 0000000..27b548f --- /dev/null +++ b/nextest/run/index.mjs @@ -0,0 +1,37 @@ +import { spawnSync } from "node:child_process"; +import { appendFileSync } from "node:fs"; +import { options } from "./options.mjs"; + +try { + const configuration = options( + { + name: process.env.INPUT_NAME, + partition: process.env.INPUT_PARTITION, + partitions: process.env.INPUT_PARTITIONS, + arguments: process.env.INPUT_ARGUMENTS, + directory: process.env["INPUT_WORKING-DIRECTORY"], + junit: process.env["INPUT_JUNIT-PATH"], + snapshots: process.env["INPUT_SNAPSHOTS-PATH"], + }, + process.env.GITHUB_WORKSPACE, + ); + for (const [name, value] of Object.entries(configuration.outputs)) { + if (/[\r\n]/.test(value)) + throw new Error("Artifact paths cannot contain newlines"); + appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`); + } + const result = spawnSync("cargo", configuration.args, { + cwd: configuration.directory, + stdio: "inherit", + env: { + ...process.env, + INSTA_UPDATE: "new", + INSTA_PENDING_DIR: configuration.outputs["snapshots-path"], + }, + }); + if (result.error) throw result.error; + process.exitCode = result.status ?? 1; +} catch (error) { + console.error(error.message); + process.exitCode = 1; +} diff --git a/nextest/run/options.mjs b/nextest/run/options.mjs new file mode 100644 index 0000000..b74f21c --- /dev/null +++ b/nextest/run/options.mjs @@ -0,0 +1,69 @@ +import path from "node:path"; + +export function partitionCount(value) { + const count = Number(value); + if ( + !Number.isSafeInteger(count) || + count < 1 || + count > 256 || + String(count) !== String(value) + ) { + throw new Error("The shard count must be an integer from 1 through 256"); + } + return count; +} + +export function options(input, workspace) { + const partitions = partitionCount(input.partitions); + const partition = partitionCount(input.partition); + if (partition > partitions) { + throw new Error("The partition number exceeds the shard count"); + } + if ( + typeof input.name !== "string" || + !/^[a-zA-Z0-9]/.test(input.name) || + /[^a-zA-Z0-9_.-]/.test(input.name) + ) { + throw new Error( + "The artifact name must contain only letters, numbers, _, ., or -", + ); + } + const args = JSON.parse(input.arguments); + if (!Array.isArray(args) || args.some((arg) => typeof arg !== "string")) { + throw new Error("The nextest arguments must be a JSON array of strings"); + } + if ( + args.some((arg) => arg === "--partition" || arg.startsWith("--partition=")) + ) { + throw new Error("Use the partition inputs instead of passing --partition"); + } + if ( + partitions > 1 && + args.some((arg) => arg === "--no-tests" || arg.startsWith("--no-tests=")) + ) { + throw new Error("Empty-shard behavior is managed by the action"); + } + const directory = path.resolve(workspace, input.directory); + const suffix = partitions === 1 ? "" : `-${partition}`; + return { + directory, + args: [ + "nextest", + "run", + ...(partitions === 1 + ? [] + : [ + "--partition", + `hash:${partition}/${partitions}`, + "--no-tests=pass", + ]), + ...args, + ], + outputs: { + "junit-name": `junit-results-${input.name}${suffix}`, + "junit-path": path.resolve(directory, input.junit), + "snapshots-name": `pending-snapshots-${input.name}${suffix}`, + "snapshots-path": path.resolve(workspace, input.snapshots), + }, + }; +} diff --git a/nextest/tests/check-coverage.py b/nextest/tests/check-coverage.py new file mode 100644 index 0000000..992e345 --- /dev/null +++ b/nextest/tests/check-coverage.py @@ -0,0 +1,30 @@ +"""Check that every real platform run covered each fixture test exactly once.""" + +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +root = Path(sys.argv[1]) +expected = {f"tests::{name}" for name in ("one", "two", "three", "four")} +for platform, partitions in (("Linux", 1), ("Linux", 2), ("macOS", 2), ("Windows", 2)): + found = set() + for partition in range(1, partitions + 1): + suffix = f"-{partition}" if partitions > 1 else "" + artifact = root / f"junit-results-fixture-{platform}-{partitions}{suffix}" + reports = list(artifact.rglob("junit.xml")) + if len(reports) != 1: + raise RuntimeError(f"Missing or duplicate JUnit report in {artifact}") + for case in ET.parse(reports[0]).iter("testcase"): + if case.find("skipped") is not None: + continue + name = case.attrib["name"] + if ( + case.find("failure") is not None + or case.find("error") is not None + or name in found + ): + raise RuntimeError(f"Failed or repeated test: {platform} {name}") + found.add(name) + if found != expected: + raise RuntimeError(f"Wrong test coverage on {platform}/{partitions}: {found}") + print(f"{platform}/{partitions}: all {len(found)} tests ran exactly once") diff --git a/nextest/tests/fixture/.config/nextest.toml b/nextest/tests/fixture/.config/nextest.toml new file mode 100644 index 0000000..76fd74b --- /dev/null +++ b/nextest/tests/fixture/.config/nextest.toml @@ -0,0 +1,2 @@ +[profile.ci.junit] +path = "junit.xml" diff --git a/nextest/tests/fixture/Cargo.lock b/nextest/tests/fixture/Cargo.lock new file mode 100644 index 0000000..910dfa7 --- /dev/null +++ b/nextest/tests/fixture/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "nextest-action-fixture" +version = "0.0.0" diff --git a/nextest/tests/fixture/Cargo.toml b/nextest/tests/fixture/Cargo.toml new file mode 100644 index 0000000..9267353 --- /dev/null +++ b/nextest/tests/fixture/Cargo.toml @@ -0,0 +1,5 @@ +[package] +name = "nextest-action-fixture" +version = "0.0.0" +edition = "2024" +publish = false diff --git a/nextest/tests/fixture/src/lib.rs b/nextest/tests/fixture/src/lib.rs new file mode 100644 index 0000000..f51db49 --- /dev/null +++ b/nextest/tests/fixture/src/lib.rs @@ -0,0 +1,19 @@ +#[cfg(test)] +mod tests { + #[test] + fn one() { + assert_eq!(1 + 1, 2); + } + #[test] + fn two() { + assert_eq!(2 + 2, 4); + } + #[test] + fn three() { + assert_eq!(3 + 3, 6); + } + #[test] + fn four() { + assert_eq!(4 + 4, 8); + } +} diff --git a/nextest/tests/options.test.mjs b/nextest/tests/options.test.mjs new file mode 100644 index 0000000..af41ce9 --- /dev/null +++ b/nextest/tests/options.test.mjs @@ -0,0 +1,110 @@ +import assert from "node:assert/strict"; +import path from "node:path"; +import test from "node:test"; +import { options, partitionCount } from "../run/options.mjs"; + +const workspace = path.resolve("checkout"); +const input = { + name: "linux", + partition: "1", + partitions: "1", + arguments: '["--workspace"]', + directory: ".", + junit: "target/nextest/**/junit.xml", + snapshots: "pending-snapshots", +}; + +test("the planner accepts every supported shard count", () => { + for (let count = 1; count <= 256; count++) + assert.equal(partitionCount(String(count)), count); + for (const value of [ + "", + "0", + "-1", + "1.5", + "01", + "1e2", + "NaN", + "257", + "2\n", + ]) { + assert.throws(() => partitionCount(value)); + } +}); + +test("one shard runs the whole suite with unsuffixed artifacts", () => { + const result = options(input, workspace); + assert.deepEqual(result.args, ["nextest", "run", "--workspace"]); + assert.equal(result.outputs["junit-name"], "junit-results-linux"); + assert.equal(result.outputs["snapshots-name"], "pending-snapshots-linux"); +}); + +test("multiple shards use one-based hash partitions and distinct artifacts", () => { + const result = options( + { ...input, partition: "2", partitions: "3" }, + workspace, + ); + assert.deepEqual(result.args, [ + "nextest", + "run", + "--partition", + "hash:2/3", + "--no-tests=pass", + "--workspace", + ]); + assert.equal(result.outputs["junit-name"], "junit-results-linux-2"); + assert.equal(result.outputs["snapshots-name"], "pending-snapshots-linux-2"); +}); + +test("Cargo and snapshot directories have independent roots", () => { + const result = options({ ...input, directory: "cargo workspace" }, workspace); + assert.equal(result.directory, path.join(workspace, "cargo workspace")); + assert.equal( + result.outputs["junit-path"], + path.join(workspace, "cargo workspace", input.junit), + ); + assert.equal( + result.outputs["snapshots-path"], + path.join(workspace, input.snapshots), + ); +}); + +test("arguments remain individual argv entries", () => { + const args = ["-E", "test(a b)", "$(touch unexpected)", "a; echo no"]; + assert.deepEqual( + options( + { ...input, arguments: JSON.stringify(args) }, + workspace, + ).args.slice(2), + args, + ); +}); + +test("invalid partitions, names, and argument overrides are rejected", () => { + for (const change of [ + { partition: "2" }, + { name: "../linux" }, + { name: "" }, + { name: "linux\n" }, + { arguments: "{}" }, + { arguments: "[1]" }, + { arguments: '["--partition", "hash:1/2"]' }, + { arguments: '["--partition=hash:1/2"]' }, + ]) + assert.throws(() => options({ ...input, ...change }, workspace)); +}); + +test("empty-shard behavior cannot be overridden in a partitioned run", () => { + for (const args of [["--no-tests", "fail"], ["--no-tests=fail"]]) { + assert.throws(() => + options( + { ...input, partitions: "2", arguments: JSON.stringify(args) }, + workspace, + ), + ); + } + assert.deepEqual( + options({ ...input, arguments: '["--no-tests=warn"]' }, workspace).args, + ["nextest", "run", "--no-tests=warn"], + ); +});