diff --git a/.github/workflows/test-rust-workspace-cache.yml b/.github/workflows/test-rust-workspace-cache.yml new file mode 100644 index 0000000..9a36192 --- /dev/null +++ b/.github/workflows/test-rust-workspace-cache.yml @@ -0,0 +1,49 @@ +name: Test Rust workspace cache + +on: + pull_request: + paths: + - rust-workspace-cache/** + - .github/workflows/test-rust-workspace-cache.yml + workflow_dispatch: + +permissions: + contents: read + +env: + RUSTC_BOOTSTRAP: "1" + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Test pruning and lifecycle + shell: bash + run: | + python -m unittest discover -s rust-workspace-cache/prune -p 'test_*.py' + node --test rust-workspace-cache/prune/lifecycle.test.cjs + + # Register this first so its post step checks the result after the cache post steps. + - name: Create workspace fixture + id: fixture + uses: $/rust-workspace-cache/tests/smoke + + - name: Cache workspace + uses: $/rust-workspace-cache + with: + workspace: ${{ steps.fixture.outputs.workspace }} + prefix-key: test-rust-workspace-cache + key: ${{ github.run_id }}-${{ github.run_attempt }} + + - name: Build workspace + working-directory: ${{ steps.fixture.outputs.workspace }} + run: cargo build --locked --offline -Z checksum-freshness diff --git a/README.md b/README.md index 3bff049..5fc7d8d 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Shared GitHub Actions for Astral projects. +- [Rust workspace cache](rust-workspace-cache/): cache workspace crates and prune + superseded artifacts before saving. - [Create release pull request](create-release-pr/): commit prepared release files and create or update a pull request. - [Release smoke test](release-smoke-test/): test release artifacts in disposable containers. diff --git a/rust-workspace-cache/README.md b/rust-workspace-cache/README.md new file mode 100644 index 0000000..da865a5 --- /dev/null +++ b/rust-workspace-cache/README.md @@ -0,0 +1,70 @@ +# Rust workspace cache + +Cache Cargo dependencies and workspace crates with `Swatinem/rust-cache`. Before +saving a new cache entry, remove superseded artifacts for workspace packages used +by the build. Pruning leaves dependencies and unused workspace packages untouched. + +```yaml +- uses: astral-sh/github-actions/rust-workspace-cache@ + with: + key: ${{ hashFiles('Cargo.toml') }} + profile: fast-build-nightly + save-if: ${{ inputs.save-rust-cache == 'true' }} + +- run: cargo test --locked --profile fast-build-nightly -Z checksum-freshness + env: + RUSTC_BOOTSTRAP: "1" +``` + +Install Rust before this action. Python 3.11 or newer must be available as +`python3` on Linux and macOS, or `python` on Windows, when the job ends. Pruning +also requires `cargo` and a current `Cargo.lock` in the workspace at that point. + +The action sets `CARGO_UNSTABLE_MTIME_ON_USE` to `true` for subsequent steps when +`save-if` is `true` and no exact cache match was restored; otherwise it sets the +variable to `false`. Usage tracking requires nightly Cargo or `RUSTC_BOOTSTRAP=1`. +Use `-Z checksum-freshness` to reuse workspace artifacts across fresh checkouts. +The action does not set `RUSTC_BOOTSTRAP` or choose a Rust toolchain. + +Pruning runs after a successful job, before `Swatinem/rust-cache` saves the cache. +Exact cache hits and `save-if: "false"` skip pruning. Pruning fails if no active +Cargo fingerprints are found, preventing a new cache from being saved without +evidence that Cargo tracked artifact usage. + +## Inputs + +| Input | Default | Description | +| ------------------ | ------------------- | -------------------------------------------------------------------------------------------------------------- | +| `workspace` | `.` | Cargo workspace directory, relative to the checkout or absolute. | +| `target-directory` | `target` | Cargo target directory, relative to `workspace`. | +| `profile` | `debug` | Profile directory within `target-directory`, such as `fast-build-nightly` or `x86_64-unknown-linux-gnu/debug`. | +| `prefix-key` | `v1-rust-workspace` | Cache key prefix. | +| `key` | Empty | Additional cache key. | +| `add-job-id-key` | `true` | Include the job ID in the cache key; set to `false` to share entries across jobs. | +| `save-if` | `true` | Whether to save a new cache entry after a successful job. | + +`workspace` and `target-directory` configure the cached paths. The build command +must use the same workspace and target directory. `profile` selects the directory +to prune; other profile directories are left unchanged. + +With `add-job-id-key: "false"`, jobs can share entries when the remaining cache key +components and GitHub cache scope also match. + +The `cache-hit` output reports whether an exact cache match was restored. + +## Windows workspaces + +An absolute `workspace` supports builds outside the checkout, including a Dev Drive: + +```yaml +- uses: astral-sh/github-actions/rust-workspace-cache@ + with: + workspace: ${{ env.UV_WORKSPACE }} + profile: fast-build-nightly + key: ${{ hashFiles('Cargo.toml') }} + add-job-id-key: "false" + save-if: ${{ inputs.save-rust-cache == 'true' }} +``` + +The action uses GitHub.com's `$/` reference to load its pruning helper from the +same commit. GitHub Enterprise Server is not supported. diff --git a/rust-workspace-cache/action.yml b/rust-workspace-cache/action.yml new file mode 100644 index 0000000..ea2f3c4 --- /dev/null +++ b/rust-workspace-cache/action.yml @@ -0,0 +1,53 @@ +name: Rust workspace cache +description: Cache Cargo dependencies and workspace crates, pruning superseded workspace artifacts before saving. + +inputs: + workspace: + description: Cargo workspace directory. + default: "." + target-directory: + description: Cargo target directory, relative to the workspace. + default: "target" + profile: + description: Build profile directory within the target directory. + default: "debug" + prefix-key: + description: Cache key prefix; change it to invalidate existing caches. + default: "v1-rust-workspace" + key: + description: Additional cache key passed to Swatinem/rust-cache. + default: "" + add-job-id-key: + description: Include the job ID in the cache key. + default: "true" + save-if: + description: Save the cache after a successful job when this is true. + default: "true" + +outputs: + cache-hit: + description: Whether an exact cache match was restored. + value: ${{ steps.cache.outputs.cache-hit }} + +runs: + using: composite + steps: + - name: Restore Rust workspace cache + id: cache + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + prefix-key: ${{ inputs.prefix-key }} + key: ${{ inputs.key }} + workspaces: ${{ inputs.workspace }} -> ${{ inputs.target-directory }} + cache-workspace-crates: "true" + add-job-id-key: ${{ inputs.add-job-id-key }} + save-if: ${{ inputs.save-if }} + + # Post steps run in reverse order, so pruning finishes before the cache is saved. + - name: Track workspace artifacts + uses: $/rust-workspace-cache/prune + with: + workspace: ${{ inputs.workspace }} + target-directory: ${{ inputs.target-directory }} + profile: ${{ inputs.profile }} + enabled: ${{ inputs.save-if == 'true' && steps.cache.outputs.cache-hit != 'true' }} diff --git a/rust-workspace-cache/prune/action.yml b/rust-workspace-cache/prune/action.yml new file mode 100644 index 0000000..cbe382b --- /dev/null +++ b/rust-workspace-cache/prune/action.yml @@ -0,0 +1,22 @@ +name: Prune Rust workspace cache +description: Track Cargo artifact usage and prune superseded artifacts after a successful job. + +inputs: + workspace: + description: Cargo workspace directory. + required: true + target-directory: + description: Cargo target directory, relative to the workspace. + required: true + profile: + description: Build profile directory within the target directory. + required: true + enabled: + description: Whether to track and prune artifacts. + required: true + +runs: + using: node24 + main: main.cjs + post: post.cjs + post-if: success() diff --git a/rust-workspace-cache/prune/lifecycle.test.cjs b/rust-workspace-cache/prune/lifecycle.test.cjs new file mode 100644 index 0000000..6ca80e1 --- /dev/null +++ b/rust-workspace-cache/prune/lifecycle.test.cjs @@ -0,0 +1,76 @@ +const assert = require("node:assert/strict"); +const { spawnSync } = require("node:child_process"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { test } = require("node:test"); + +function setup(context, enabled) { + const directory = fs.mkdtempSync( + path.join(os.tmpdir(), "workspace-cache-test-"), + ); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const workspace = path.join(directory, "workspace with spaces"); + fs.mkdirSync(workspace); + const environment = path.join(directory, "env"); + const state = path.join(directory, "state"); + fs.writeFileSync(environment, ""); + fs.writeFileSync(state, ""); + const result = spawnSync( + process.execPath, + [path.join(__dirname, "main.cjs")], + { + cwd: workspace, + env: { + ...process.env, + INPUT_ENABLED: String(enabled), + INPUT_WORKSPACE: workspace, + "INPUT_TARGET-DIRECTORY": "build-output", + INPUT_PROFILE: "fast-build-nightly", + GITHUB_ENV: environment, + GITHUB_STATE: state, + RUNNER_TEMP: directory, + }, + encoding: "utf8", + }, + ); + assert.equal(result.status, 0, result.stderr); + return { + workspace, + environment: fs.readFileSync(environment, "utf8"), + state: fs.readFileSync(state, "utf8"), + }; +} + +test("records an isolated marker and the external workspace for pruning", (context) => { + const result = setup(context, true); + assert.equal(result.environment, "CARGO_UNSTABLE_MTIME_ON_USE=true\n"); + const state = JSON.parse(result.state.slice("cache=".length)); + assert.equal(state.workspace, result.workspace); + assert.equal( + state.profile, + path.join(result.workspace, "build-output", "fast-build-nightly"), + ); + assert.ok(fs.statSync(state.marker).isFile()); + const other = JSON.parse(setup(context, true).state.slice("cache=".length)); + assert.notEqual(state.marker, other.marker); +}); + +test("disables usage tracking and pruning when the cache will not be saved", (context) => { + const result = setup(context, false); + assert.equal(result.environment, "CARGO_UNSTABLE_MTIME_ON_USE=false\n"); + assert.equal(result.state, ""); +}); + +test("the post step does nothing when no pruning state was recorded", () => { + const result = spawnSync( + process.execPath, + [path.join(__dirname, "post.cjs")], + { + env: { ...process.env, STATE_cache: "", PATH: "" }, + encoding: "utf8", + }, + ); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, ""); +}); diff --git a/rust-workspace-cache/prune/main.cjs b/rust-workspace-cache/prune/main.cjs new file mode 100644 index 0000000..02c9c00 --- /dev/null +++ b/rust-workspace-cache/prune/main.cjs @@ -0,0 +1,26 @@ +const fs = require("node:fs"); +const path = require("node:path"); + +const enabled = process.env.INPUT_ENABLED === "true"; +fs.appendFileSync( + process.env.GITHUB_ENV, + `CARGO_UNSTABLE_MTIME_ON_USE=${enabled}\n`, +); + +if (enabled) { + const workspace = path.resolve(process.env.INPUT_WORKSPACE); + const profile = path.resolve( + workspace, + process.env["INPUT_TARGET-DIRECTORY"], + process.env.INPUT_PROFILE, + ); + const directory = fs.mkdtempSync( + path.join(process.env.RUNNER_TEMP, "rust-workspace-cache-"), + ); + const marker = path.join(directory, "marker"); + fs.writeFileSync(marker, ""); + fs.appendFileSync( + process.env.GITHUB_STATE, + `cache=${JSON.stringify({ workspace, profile, marker })}\n`, + ); +} diff --git a/rust-workspace-cache/prune/post.cjs b/rust-workspace-cache/prune/post.cjs new file mode 100644 index 0000000..dc87783 --- /dev/null +++ b/rust-workspace-cache/prune/post.cjs @@ -0,0 +1,18 @@ +const fs = require("node:fs"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); + +if (process.env.STATE_cache) { + const { workspace, profile, marker } = JSON.parse(process.env.STATE_cache); + try { + const result = spawnSync( + process.platform === "win32" ? "python" : "python3", + [path.join(__dirname, "prune.py"), profile, marker], + { cwd: workspace, stdio: "inherit" }, + ); + if (result.error) throw result.error; + process.exitCode = result.status ?? 1; + } finally { + fs.rmSync(path.dirname(marker), { recursive: true, force: true }); + } +} diff --git a/rust-workspace-cache/prune/prune.py b/rust-workspace-cache/prune/prune.py new file mode 100644 index 0000000..1a2993d --- /dev/null +++ b/rust-workspace-cache/prune/prune.py @@ -0,0 +1,116 @@ +"""Remove superseded workspace artifacts without discarding reusable dependencies.""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +from pathlib import Path + +FINGERPRINT_DIRECTORY = re.compile(r"^(?P.+)-(?P[0-9a-f]{16})$") +ARTIFACT_FINGERPRINT = re.compile(r"-([0-9a-f]{16})(?:\.|$)") + + +def workspace_packages(workspace: Path) -> set[str]: + metadata = json.loads( + subprocess.check_output( + ["cargo", "metadata", "--no-deps", "--format-version", "1", "--locked"], + cwd=workspace, + text=True, + ) + ) + members = set(metadata["workspace_members"]) + return { + package["name"] for package in metadata["packages"] if package["id"] in members + } + + +def fingerprint_last_used(directory: Path) -> int | None: + timestamps = [ + fingerprint.stat().st_mtime_ns + for metadata in directory.glob("*.json") + if (fingerprint := directory / metadata.stem).is_file() + ] + return max(timestamps, default=None) + + +def prune_workspace_fingerprints( + profile: Path, marker_timestamp: int, packages: set[str] +) -> tuple[int, int, int]: + fingerprint_root = profile / ".fingerprint" + stale_fingerprints_by_package: dict[str, set[str]] = {} + active_fingerprints: set[str] = set() + active_packages: set[str] = set() + + for directory in fingerprint_root.iterdir(): + if not directory.is_dir(): + continue + + match = FINGERPRINT_DIRECTORY.fullmatch(directory.name) + if match is None: + continue + + last_used = fingerprint_last_used(directory) + if last_used is None: + continue + + package = match.group("package") + fingerprint = match.group("fingerprint") + if last_used >= marker_timestamp: + active_fingerprints.add(fingerprint) + if package in packages: + active_packages.add(package) + elif package in packages: + stale_fingerprints_by_package.setdefault(package, set()).add(fingerprint) + + if not active_fingerprints: + raise RuntimeError("No active Cargo fingerprints found; refusing to prune") + + stale_fingerprints = { + fingerprint + for package in active_packages + for fingerprint in stale_fingerprints_by_package.get(package, set()) + } + stale_fingerprints.difference_update(active_fingerprints) + removed_paths = 0 + + for directory in (fingerprint_root, profile / "deps", profile / "build"): + if not directory.is_dir(): + continue + + for artifact in directory.iterdir(): + match = ARTIFACT_FINGERPRINT.search(artifact.name) + if match is None or match.group(1) not in stale_fingerprints: + continue + + if artifact.is_symlink() or artifact.is_file(): + artifact.unlink() + elif artifact.is_dir(): + shutil.rmtree(artifact) + else: + continue + + removed_paths += 1 + + return len(stale_fingerprints), removed_paths, len(active_fingerprints) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("profile", type=Path, help="Cargo profile directory") + parser.add_argument("marker", type=Path, help="Marker created before the build") + args = parser.parse_args() + + stale, removed, active = prune_workspace_fingerprints( + args.profile, args.marker.stat().st_mtime_ns, workspace_packages(Path.cwd()) + ) + print( + f"Pruned {stale} superseded workspace fingerprints and {removed} artifact " + f"paths; retained {active} active fingerprints" + ) + + +if __name__ == "__main__": + main() diff --git a/rust-workspace-cache/prune/test_prune.py b/rust-workspace-cache/prune/test_prune.py new file mode 100644 index 0000000..6b462b5 --- /dev/null +++ b/rust-workspace-cache/prune/test_prune.py @@ -0,0 +1,104 @@ +import os +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from prune import prune_workspace_fingerprints + +OLD = 1_000_000_000 +MARKER = 2_000_000_000 +CURRENT = 3_000_000_000 + + +class PruneWorkspaceCacheTest(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.profile = Path(self.temporary.name) + (self.profile / ".fingerprint").mkdir() + (self.profile / "deps").mkdir() + (self.profile / "build").mkdir() + + def fingerprint(self, package: str, fingerprint: str, timestamp: int) -> Path: + directory = self.profile / ".fingerprint" / f"{package}-{fingerprint}" + directory.mkdir() + (directory / "lib-example.json").write_text("{}") + stamp = directory / "lib-example" + stamp.touch() + os.utime(stamp, ns=(timestamp, timestamp)) + return directory + + def test_removes_superseded_workspace_artifacts(self) -> None: + old = self.fingerprint("example", "1111111111111111", OLD) + current = self.fingerprint("example", "2222222222222222", CURRENT) + dependency = self.fingerprint("dependency", "3333333333333333", OLD) + unused = self.fingerprint("unused", "4444444444444444", OLD) + library = self.profile / "deps" / "libexample-1111111111111111.rlib" + library.touch() + build = self.profile / "build" / "example-1111111111111111" + build.mkdir() + (build / "output").touch() + active_library = self.profile / "deps" / "libexample-2222222222222222.rlib" + active_library.touch() + + self.assertEqual( + prune_workspace_fingerprints(self.profile, MARKER, {"example", "unused"}), + (1, 3, 1), + ) + self.assertFalse(old.exists()) + self.assertFalse(library.exists()) + self.assertFalse(build.exists()) + self.assertTrue(current.is_dir()) + self.assertTrue(active_library.is_file()) + self.assertTrue(dependency.is_dir()) + self.assertTrue(unused.is_dir()) + + def test_retains_hashes_used_by_another_active_package(self) -> None: + old = self.fingerprint("example", "1111111111111111", OLD) + self.fingerprint("example", "2222222222222222", CURRENT) + self.fingerprint("dependency", "1111111111111111", CURRENT) + + self.assertEqual( + prune_workspace_fingerprints(self.profile, MARKER, {"example"}), + (0, 0, 2), + ) + self.assertTrue(old.is_dir()) + + def test_refuses_to_prune_without_active_fingerprints(self) -> None: + old = self.fingerprint("example", "1111111111111111", OLD) + with self.assertRaisesRegex(RuntimeError, "No active Cargo fingerprints"): + prune_workspace_fingerprints(self.profile, MARKER, {"example"}) + self.assertTrue(old.is_dir()) + + def test_retains_fingerprints_without_usage_metadata(self) -> None: + unknown = self.fingerprint("example", "1111111111111111", OLD) + (unknown / "lib-example").unlink() + self.fingerprint("example", "2222222222222222", MARKER) + + self.assertEqual( + prune_workspace_fingerprints(self.profile, MARKER, {"example"}), + (0, 0, 1), + ) + self.assertTrue(unknown.is_dir()) + + @unittest.skipIf(sys.platform == "win32", "Creating symlinks requires privileges") + def test_removes_stale_symlinks_without_removing_their_targets(self) -> None: + self.fingerprint("example", "1111111111111111", OLD) + self.fingerprint("example", "2222222222222222", CURRENT) + target = self.profile / "keep" + target.mkdir() + (target / "output").write_text("keep") + link = self.profile / "build" / "example-1111111111111111" + link.symlink_to(target, target_is_directory=True) + + prune_workspace_fingerprints(self.profile, MARKER, {"example"}) + + self.assertFalse(link.is_symlink()) + self.assertEqual((target / "output").read_text(), "keep") + + +if __name__ == "__main__": + unittest.main() diff --git a/rust-workspace-cache/tests/smoke/action.yml b/rust-workspace-cache/tests/smoke/action.yml new file mode 100644 index 0000000..e852bd8 --- /dev/null +++ b/rust-workspace-cache/tests/smoke/action.yml @@ -0,0 +1,10 @@ +name: Workspace cache smoke fixture +description: Create a Cargo workspace and check its artifacts after the cache post steps. +outputs: + workspace: + description: Absolute path of the temporary Cargo workspace. +runs: + using: node24 + main: setup.cjs + post: check.cjs + post-if: success() diff --git a/rust-workspace-cache/tests/smoke/check.cjs b/rust-workspace-cache/tests/smoke/check.cjs new file mode 100644 index 0000000..42bdee9 --- /dev/null +++ b/rust-workspace-cache/tests/smoke/check.cjs @@ -0,0 +1,23 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); + +const fingerprints = path.join( + process.env.STATE_workspace, + "target", + "debug", + ".fingerprint", +); +assert.ok( + !fs.existsSync( + path.join(fingerprints, "workspace-cache-fixture-1111111111111111"), + ), + "The stale workspace fingerprint must be pruned", +); +assert.ok( + fs + .readdirSync(fingerprints) + .some((entry) => entry.startsWith("workspace-cache-fixture-")), + "The active workspace fingerprint must remain cached", +); +console.log("Superseded workspace artifacts were pruned."); diff --git a/rust-workspace-cache/tests/smoke/setup.cjs b/rust-workspace-cache/tests/smoke/setup.cjs new file mode 100644 index 0000000..4a162a6 --- /dev/null +++ b/rust-workspace-cache/tests/smoke/setup.cjs @@ -0,0 +1,37 @@ +const fs = require("node:fs"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); + +const workspace = fs.mkdtempSync( + path.join(process.env.RUNNER_TEMP, "cargo workspace "), +); +fs.mkdirSync(path.join(workspace, "src")); +fs.writeFileSync( + path.join(workspace, "Cargo.toml"), + '[package]\nname = "workspace-cache-fixture"\nversion = "0.1.0"\nedition = "2024"\n', +); +fs.writeFileSync( + path.join(workspace, "src", "lib.rs"), + "pub fn example() {}\n", +); +const result = spawnSync("cargo", ["generate-lockfile", "--offline"], { + cwd: workspace, + stdio: "inherit", +}); +if (result.error) throw result.error; +if (result.status !== 0) process.exit(result.status ?? 1); + +const stale = path.join( + workspace, + "target", + "debug", + ".fingerprint", + "workspace-cache-fixture-1111111111111111", +); +fs.mkdirSync(stale, { recursive: true }); +fs.writeFileSync(path.join(stale, "lib-workspace_cache_fixture.json"), "{}"); +const fingerprint = path.join(stale, "lib-workspace_cache_fixture"); +fs.writeFileSync(fingerprint, "stale"); +fs.utimesSync(fingerprint, 1, 1); +fs.appendFileSync(process.env.GITHUB_OUTPUT, `workspace=${workspace}\n`); +fs.appendFileSync(process.env.GITHUB_STATE, `workspace=${workspace}\n`);