From 1e83decdf115bac7c9a21e88136a7610061d3f62 Mon Sep 17 00:00:00 2001 From: baku-ccron Date: Sun, 20 Sep 2026 12:29:35 +0000 Subject: [PATCH 1/3] ci: witness that codegen still emits each committed generated file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rainix-copy-artifacts` currency-checks committed generated sources by re-running every consumer codegen hook and then `git diff --exit-code`. That method has one blind spot, and it is total: a generator that has STOPPED emitting a file writes nothing, so the committed copy — already correct — is left exactly as it is, nothing differs, and the job is green over a dead emitter (rainlanguage/rain.factory.deploy#35, reproduced there with a control). The blind spot is structural. The generator's output is the check's only oracle for what the committed files should contain, so a file the generator never writes has no oracle at all. Seeing it needs an INDEPENDENT statement of which committed files are generated: `script/codegen-manifest.txt`. New `rainix-static codegen-witness mark|verify --state `, wired into the workflow as a composite action at `@main`: - `mark` records every git-tracked file's mtime before the first codegen hook. - `verify` re-stats after the last hook, before `forge fmt`, and calls a file WRITTEN when it exists and its mtime moved. `vm.writeFile` rewrites unconditionally, so a live emitter moves the mtime even when the bytes are identical — exactly the case the diff cannot tell from a dead emitter. - Any declared path nothing wrote fails the job, naming the file. Listed-must-be-written, not set equality: a file written but not declared is a printed note, never a failure, so an incidental write inside the window (forge build is in there) cannot redden every consumer at once. Scope is git-tracked files, keeping out/, cache/, broadcast/ and dependencies/ out of the witness regardless of a repo's .gitignore hygiene. A composite action rather than a pinned-sha run step, so the check version always matches the action version and no RAINIX_SHA bump window leaves consumers red with `unknown subcommand` between two merges. Consumers: the 15 rainlanguage repos that carry a codegen hook must each add a `script/codegen-manifest.txt` or their copy-artifacts job fails. The failing job prints the manifest that run would justify, for review and commit. Co-Authored-By: Claude Opus 5 (1M context) --- .github/actions/codegen-witness/action.yml | 33 ++ .github/workflows/rainix-copy-artifacts.yaml | 27 ++ README.md | 16 + flake.nix | 2 + rainix-static/src/codegen_witness.rs | 393 ++++++++++++++++++ rainix-static/src/main.rs | 37 +- test/bats/action/codegen-witness.test.bats | 266 ++++++++++++ .../workflow/rainix-copy-artifacts.test.bats | 131 ++++++ 8 files changed, 903 insertions(+), 2 deletions(-) create mode 100644 .github/actions/codegen-witness/action.yml create mode 100644 rainix-static/src/codegen_witness.rs create mode 100644 test/bats/action/codegen-witness.test.bats create mode 100644 test/bats/workflow/rainix-copy-artifacts.test.bats diff --git a/.github/actions/codegen-witness/action.yml b/.github/actions/codegen-witness/action.yml new file mode 100644 index 0000000..252b03d --- /dev/null +++ b/.github/actions/codegen-witness/action.yml @@ -0,0 +1,33 @@ +name: codegen-witness +description: >- + Witnesses that the repo's codegen hooks still EMIT each committed generated file, which `rainix-copy-artifacts`' own currency check cannot. That check re-runs the generators and then `git diff --exit-code`, which proves the committed CONTENT is current but is structurally blind to a generator that has STOPPED emitting a file: the committed copy is already correct, so nothing is rewritten, nothing differs, and the job is green over a dead emitter (rainlanguage/rain.factory.deploy#35, reproduced there with a control — a marker appended to a generated file survived the generator run once the single call emitting that file was removed, with `git diff` clean throughout). The blind spot is structural rather than a bug in any repo: the generator's output is the check's only oracle for what the committed files should hold, so a file the generator never writes has no oracle at all, and seeing it needs an INDEPENDENT statement of which committed files are generated. That statement is `script/codegen-manifest.txt`. Two phases, and both must run in the same job. `mark` records every git-tracked file's mtime before the first codegen hook; `verify` re-stats them after the last one and calls a file WRITTEN when it exists and its mtime moved — `vm.writeFile` and its kin rewrite unconditionally, so a live emitter moves the mtime even when the bytes are identical, which is exactly the case a diff cannot tell from a dead one. Any path the manifest declares that nothing wrote fails the job by name. A path written but NOT declared is a printed note and never a failure: set equality would keep the manifest self-maintaining, but it would also couple a gate every Rain repo runs at `@main` to every incidental write inside the window (`forge build` is in there), so the day some tool starts touching a lock file it would redden the whole org at once. The claim worth gating on is the one the defect is about. Scope is git-tracked files, which keeps `out/`, `cache/`, `broadcast/` and `dependencies/` out of the witness whether or not a repo ignores them properly. A repo that runs no codegen hook and carries no manifest is a no-op; a repo with a hook and no manifest fails, and is printed the manifest its own run would justify. +inputs: + phase: + description: >- + `mark` before the first codegen hook runs, `verify` after the last one. `verify` must come BEFORE any formatter: `forge fmt` rewrites sources of its own, so running it first forges the witness — a file fmt happened to touch would look like one a generator wrote. + required: true +runs: + using: composite + steps: + - name: Codegen witness + shell: bash + env: + # Via env rather than interpolated into the script text: caller input + # never gets pasted into a shell command. + RAINIX_CODEGEN_WITNESS_PHASE: ${{ inputs.phase }} + # The two phases are separate steps, so the state has to outlive one of + # them — and it must not land in the working tree, where it would show + # up in the very `git diff` this check runs beside. + RAINIX_CODEGEN_WITNESS_STATE: ${{ runner.temp }}/rainix-codegen-witness.json + run: | + set -euo pipefail + # Single source of truth: the Rust rainix-static binary (its unit tests + # run in the nix build, and test/bats/action/codegen-witness.test.bats + # runs the built binary end to end). The path: flake ref runs it from + # THIS composite's own checkout, so the check version always matches the + # action version regardless of any RAINIX_SHA the caller pins — which + # also means a new subcommand ships with the action instead of waiting + # on a sha bump that would otherwise leave every consumer red in between. + nix run "path:$(cd "$GITHUB_ACTION_PATH/../../.." && pwd)#rainix-static" -- \ + codegen-witness "$RAINIX_CODEGEN_WITNESS_PHASE" \ + --state "$RAINIX_CODEGEN_WITNESS_STATE" diff --git a/.github/workflows/rainix-copy-artifacts.yaml b/.github/workflows/rainix-copy-artifacts.yaml index a15aca2..245efeb 100644 --- a/.github/workflows/rainix-copy-artifacts.yaml +++ b/.github/workflows/rainix-copy-artifacts.yaml @@ -21,6 +21,26 @@ jobs: - name: Install soldeer dependencies if: hashFiles('soldeer.lock') != '' run: nix develop github:rainlanguage/rainix/${{ env.RAINIX_SHA }}#sol-shell -c forge soldeer install + # Half of the currency check that the diff at the bottom cannot make. + # Re-running the generators and diffing proves the committed CONTENT is + # current, but it is blind to a generator that has STOPPED emitting a file: + # nothing is rewritten, the already-correct committed copy does not differ, + # and the job is green over a dead emitter + # (rainlanguage/rain.factory.deploy#35, verified there with a control). + # `mark` records every tracked file's mtime before the first codegen hook + # below; `verify`, after the last one, fails on any path + # `script/codegen-manifest.txt` declares generated that nothing wrote. + # Semantics: rainix-static/src/codegen_witness.rs. + # + # Unconditional, with no `hashFiles` guard: the binary decides whether + # there is anything to witness (a repo with no codegen hook and no manifest + # passes), so that predicate lives in one tested place rather than + # duplicated across two steps where it can drift — and a repo cannot make + # the check vanish by deleting the hooks it was checking. + - name: Mark tracked files before codegen + uses: rainlanguage/rainix/.github/actions/codegen-witness@main + with: + phase: mark # Currency-check every committed generated artifact by re-running each # consumer-provided codegen step. The final git diff fails if any # committed file has drifted from its source. The build-meta.sh hook is @@ -53,6 +73,13 @@ jobs: - name: Regenerate derived artifacts if: hashFiles('script/build.sh') != '' run: ./script/build.sh + # The other half, BEFORE `forge fmt`: fmt rewrites sources of its own, so + # running it first would forge the witness — a file fmt happened to touch + # would look like one a generator wrote. + - name: Assert every declared generated file was written + uses: rainlanguage/rainix/.github/actions/codegen-witness@main + with: + phase: verify - name: Format (so generated artifacts match committed style) run: nix develop github:rainlanguage/rainix/${{ env.RAINIX_SHA }}#sol-shell -c forge fmt - name: Assert committed artifacts match freshly built diff --git a/README.md b/README.md index dbf3358..9f9e561 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,22 @@ single job it runs whichever of these the repo has: then `forge fmt` and the `git diff` assert. +The diff is only half the check. Re-running the generators and diffing proves +the committed **content** is current, but it cannot see a generator that has +**stopped emitting a file**: the committed copy is already correct, so nothing +is rewritten, nothing differs, and the job is green over a dead emitter +(rainlanguage/rain.factory.deploy#35). So the job also witnesses which files the +hooks actually wrote, against a committed declaration: + +- `script/codegen-manifest.txt` — one repo-relative path per line (`#` comments + and blank lines ignored), naming every committed file the hooks generate. + Every path listed must be written on each run, or the job fails by name. A + file written but not listed is a printed note, never a failure. + +A repo that runs any of the hooks above **must** carry this file; a repo that +generates nothing needs neither. If it is missing, the job fails and prints the +manifest that run would justify, to be reviewed and committed. + ```yaml name: copy-artifacts on: [push] diff --git a/flake.nix b/flake.nix index d52a31f..72c2be8 100644 --- a/flake.nix +++ b/flake.nix @@ -474,6 +474,7 @@ bats test/bats/action/prompt-cap.test.bats bats test/bats/action/frozen-snapshots-append-only.test.bats bats test/bats/action/mutation-ledger.test.bats + bats test/bats/action/codegen-witness.test.bats bats test/bats/task/skip-simulation.test.bats bats test/bats/task/subgraph-build.test.bats bats test/bats/task/subgraph-deploy-version.test.bats @@ -481,6 +482,7 @@ bats test/bats/task/no-custom-natspec.test.bats bats test/bats/workflow/rainix-sol-static.test.bats bats test/bats/workflow/rainix-rs-static.test.bats + bats test/bats/workflow/rainix-copy-artifacts.test.bats bats test/bats/workflow/test-yml.test.bats ''; additionalBuildInputs = [ pkgs.bats ] ++ sol-build-inputs ++ node-build-inputs; diff --git a/rainix-static/src/codegen_witness.rs b/rainix-static/src/codegen_witness.rs new file mode 100644 index 0000000..a758226 --- /dev/null +++ b/rainix-static/src/codegen_witness.rs @@ -0,0 +1,393 @@ +//! Witness that the repo's codegen hooks still EMIT each committed generated +//! file, which re-running them and diffing cannot see. +//! +//! `rainix-copy-artifacts` currency-checks committed generated sources by +//! re-running every consumer codegen hook and then `git diff --exit-code`. That +//! method has one blind spot, and it is total: a generator that has STOPPED +//! emitting a file writes nothing, so the committed copy — already correct — +//! is left exactly as it is, nothing differs, and the job is green over a dead +//! emitter (rainlanguage/rain.factory.deploy#35, reproduced with a control: a +//! marker appended to the generated file survived the generator run once its +//! one emitting call was removed, with `git diff` clean throughout). +//! +//! The blind spot is structural. The generator's output is the check's only +//! oracle for what the committed files should contain, so a file the generator +//! never writes has no oracle at all. Seeing it requires an INDEPENDENT +//! statement of which committed files are generated, which is what +//! `script/codegen-manifest.txt` is. The check then has two halves: the diff +//! says the content is current, and this says something actually wrote it. +//! +//! ## What is witnessed +//! +//! `mark` records the mtime of every git-tracked file before the first codegen +//! hook runs; `verify` re-stats them after the last one and calls a file +//! WRITTEN when it exists now and its mtime moved. That is the property the +//! defect is about — `vm.writeFile` and friends rewrite unconditionally, so a +//! live emitter always moves the mtime even when the bytes are identical, +//! which is exactly the case the diff cannot distinguish from a dead one. +//! +//! Scope is git-tracked files: the currency check is about COMMITTED generated +//! sources, and restricting to them keeps `out/`, `cache/`, `broadcast/` and +//! `dependencies/` out of the witness whether or not a repo ignores them +//! properly. +//! +//! ## Why listed-must-be-written, and not set equality +//! +//! Set equality (every written file must be listed) would keep the manifest +//! self-maintaining, but it couples an org-wide gate to every incidental write +//! inside the window — `forge build` is in there, and the day it starts +//! rewriting a lock file it reddens every consumer at once. The claim worth +//! making is the one the defect is about: a path this repo DECLARES as +//! generated must have been written. Files written but not listed are printed +//! as a note instead, so a newly generated file is discoverable without being +//! able to break anyone. +//! +//! The residual gap is the mirror of that choice, and is named here rather than +//! papered over: a generated file nobody has listed yet is not protected. So is +//! a brand-new generated file that is never committed — `git diff --exit-code` +//! does not see untracked files either, which is a separate hole in the same +//! job. + +use std::collections::BTreeSet; +use std::path::Path; +use std::process::Command; +use std::time::UNIX_EPOCH; + +/// The committed declaration, beside the hooks it describes (`script/Build.sol`, +/// `script/build.sh`, ...). One fixed path: the workflow is consumed at `@main` +/// by every Rain repo and cannot go looking for a per-repo convention. +pub(crate) const MANIFEST_PATH: &str = "script/codegen-manifest.txt"; + +/// The consumer-supplied codegen hooks `rainix-copy-artifacts` runs. Presence of +/// any one of them is what makes a manifest mandatory: a repo with no codegen +/// has nothing to declare and must not be asked to declare it. +pub(crate) const HOOKS: [&str; 4] = [ + "script/build-meta.sh", + "script/Build.sol", + "script/CopyArtifacts.sol", + "script/build.sh", +]; + +/// Header of a rendered manifest. Present so the file explains itself to +/// whoever opens it in a diff, and parsed back out as a comment. +const HEADER: &str = "\ +# Committed files this repo's codegen hooks generate — one path per line. +# +# rainix-copy-artifacts re-runs the hooks and diffs, which proves the CONTENT is +# current but cannot see a generator that has stopped emitting a file: nothing +# is rewritten, so nothing differs and the job is green over a dead emitter. +# Every path listed here must be written on each run, so losing an emitter is a +# red job rather than silence. +# +# Add a path when the repo starts generating it; remove one only when the file +# genuinely stops being generated (and is deleted or hand-maintained from then +# on). Blank lines and # comments are ignored. +"; + +/// Paths a manifest lists. Blank lines and `#` comments are ignored, and each +/// path is trimmed, so the file can carry its own explanation. +pub(crate) fn parse_manifest(text: &str) -> BTreeSet { + text.lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(str::to_string) + .collect() +} + +/// A manifest file's content for `paths`: the header, then one path per line in +/// sorted order. What `verify` prints for a consumer to commit verbatim. +pub(crate) fn render_manifest(paths: &BTreeSet) -> String { + let mut out = String::from(HEADER); + for path in paths { + out.push_str(path); + out.push('\n'); + } + out +} + +/// Offenders for a manifest that exists: every declared path nothing wrote. +/// +/// Declared-but-absent is reported as the same offence — a path that is not +/// even on disk was certainly not written, and saying "no hook wrote it" of a +/// file that does not exist would send the reader looking for the wrong thing. +pub(crate) fn offenders( + listed: &BTreeSet, + written: &BTreeSet, + present: &BTreeSet, + manifest: &str, +) -> Vec { + let mut out = Vec::new(); + for path in listed.difference(written) { + if present.contains(path) { + out.push(format!( + "ERROR: {manifest} declares {path} generated, but no codegen hook wrote it on \ + this run. The committed file is left exactly as it was, so re-running the \ + generators and diffing passes without checking it — its emitter is dead. \ + Restore the emitter, or, if {path} is genuinely no longer generated, say so by \ + removing the line (and the file, if nothing hand-maintains it)." + )); + } else { + out.push(format!( + "ERROR: {manifest} declares {path} generated, but no such file exists after \ + running the codegen hooks. Either its emitter is dead, or the path in the \ + manifest is wrong." + )); + } + } + out +} + +/// Repo-relative paths git tracks under `root`. +fn tracked(root: &Path) -> Result, String> { + let out = Command::new("git") + .arg("-C") + .arg(root) + // -z: a path may contain anything but NUL, and git otherwise quotes the + // awkward ones, which would not match the manifest. + .args(["ls-files", "-z"]) + .output() + .map_err(|e| format!("failed to run git ls-files: {e}"))?; + if !out.status.success() { + return Err(format!( + "git ls-files in {} failed: {}", + root.display(), + String::from_utf8_lossy(&out.stderr).trim() + )); + } + Ok(String::from_utf8_lossy(&out.stdout) + .split('\0') + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect()) +} + +/// Nanoseconds-since-epoch mtime of `root/path`, or `None` when it is not there +/// (git tracks a path the worktree may not currently hold) or its mtime cannot +/// be read at all. +fn mtime_nanos(root: &Path, path: &str) -> Option { + let meta = std::fs::metadata(root.join(path)).ok()?; + let since = meta.modified().ok()?.duration_since(UNIX_EPOCH).ok()?; + u64::try_from(since.as_nanos()).ok() +} + +/// Record every tracked file's mtime into `state`, to be compared after the +/// codegen hooks have run. Returns how many files were marked. +pub(crate) fn mark(root: &Path, state: &Path) -> Result { + let files = tracked(root)?; + let mut map = serde_json::Map::new(); + for path in &files { + let value = match mtime_nanos(root, path) { + Some(nanos) => serde_json::Value::from(nanos), + None => serde_json::Value::Null, + }; + map.insert(path.clone(), value); + } + let doc = serde_json::json!({ "files": serde_json::Value::Object(map) }); + std::fs::write(state, doc.to_string()) + .map_err(|e| format!("failed to write {}: {e}", state.display()))?; + Ok(files.len()) +} + +/// The marked files that were WRITTEN since `mark`, and those that exist now. +/// +/// Written means present now with a different mtime: a rewrite with identical +/// bytes still moves it, which is the whole point, while a file the hooks +/// DELETED is not a write (and `git diff` catches a deletion on its own). +pub(crate) fn written_since( + root: &Path, + state: &Path, +) -> Result<(BTreeSet, BTreeSet), String> { + let text = std::fs::read_to_string(state).map_err(|e| { + format!( + "failed to read the mark state {}: {e} — `codegen-witness mark` must run before the \ + codegen steps, in the same job", + state.display() + ) + })?; + let doc: serde_json::Value = serde_json::from_str(&text) + .map_err(|e| format!("{} is not valid JSON: {e}", state.display()))?; + let files = doc + .get("files") + .and_then(serde_json::Value::as_object) + .ok_or_else(|| format!("{} has no `files` object", state.display()))?; + + let mut written = BTreeSet::new(); + let mut present = BTreeSet::new(); + for (path, before) in files { + let after = mtime_nanos(root, path); + if after.is_some() { + present.insert(path.clone()); + if after != before.as_u64() { + written.insert(path.clone()); + } + } + } + Ok((written, present)) +} + +/// `mark` as a subcommand: record and report, or fail loud. +pub(crate) fn run_mark(root: &Path, state: &Path) { + match mark(root, state) { + Err(e) => crate::fail(&format!("codegen-witness mark: {e}")), + Ok(n) => println!( + "codegen-witness: marked {n} tracked files in {}", + root.display() + ), + } +} + +/// `verify` as a subcommand: the manifest's declarations against what the hooks +/// actually wrote. +pub(crate) fn run_verify(root: &Path, state: &Path, manifest_rel: &str) { + let (written, present) = match written_since(root, state) { + Ok(sets) => sets, + Err(e) => crate::fail(&format!("codegen-witness verify: {e}")), + }; + + let manifest_path = root.join(manifest_rel); + let listed = match std::fs::read_to_string(&manifest_path) { + Ok(text) => Some(parse_manifest(&text)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => crate::fail(&format!( + "codegen-witness verify: failed to read {}: {e}", + manifest_path.display() + )), + }; + + // The manifest itself is written by hand and by this command's output, never + // by a codegen hook, so it is never part of its own witness. + let mut written: BTreeSet = written; + written.remove(manifest_rel); + + let Some(listed) = listed else { + let hooks: Vec<&str> = HOOKS + .iter() + .copied() + .filter(|h| root.join(h).exists()) + .collect(); + if hooks.is_empty() { + println!( + "codegen-witness: clean — no codegen hook and no {manifest_rel}; nothing declared \ + generated here" + ); + return; + } + eprintln!( + "::error::codegen-witness: this repo runs codegen ({}) but has no {manifest_rel}, so \ + the currency check cannot tell a generated file that is still emitted from one whose \ + emitter has died — the committed copy is correct either way. Commit a manifest \ + declaring the committed files the hooks generate. What they wrote on this run, as a \ + starting point:", + hooks.join(", ") + ); + eprintln!("{}", render_manifest(&written)); + std::process::exit(1); + }; + + let offenders = offenders(&listed, &written, &present, manifest_rel); + let unlisted: Vec<&String> = written.difference(&listed).collect(); + + if offenders.is_empty() { + println!( + "codegen-witness: clean — {} declared generated files, each written this run", + listed.len() + ); + // A note, never a failure: see the module doc on why an unlisted write + // must not be able to redden an org-wide job. + if !unlisted.is_empty() { + println!( + "codegen-witness: note — written but not declared in {manifest_rel}, so nothing \ + will notice if their emitters die:" + ); + for path in unlisted { + println!(" {path}"); + } + } + return; + } + + for line in &offenders { + eprintln!("::error::{line}"); + } + eprintln!("What the codegen hooks actually wrote on this run:"); + eprintln!("{}", render_manifest(&written)); + std::process::exit(1); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn set(paths: &[&str]) -> BTreeSet { + paths.iter().map(|p| p.to_string()).collect() + } + + #[test] + fn manifest_ignores_comments_and_blanks() { + let text = "# a comment\n\nsrc/a.sol\n src/b.sol \n#src/c.sol\n"; + assert_eq!(parse_manifest(text), set(&["src/a.sol", "src/b.sol"])); + } + + #[test] + fn rendered_manifest_round_trips_sorted() { + let paths = set(&["src/b.sol", "src/a.sol"]); + let rendered = render_manifest(&paths); + assert!(rendered.starts_with('#')); + // Sorted, so the committed file does not churn on set ordering. + let body: Vec<&str> = rendered + .lines() + .filter(|l| !l.starts_with('#') && !l.is_empty()) + .collect(); + assert_eq!(body, vec!["src/a.sol", "src/b.sol"]); + assert_eq!(parse_manifest(&rendered), paths); + } + + #[test] + fn empty_manifest_renders_to_header_only() { + let rendered = render_manifest(&BTreeSet::new()); + assert!(parse_manifest(&rendered).is_empty()); + } + + // THE defect: the file is on disk and byte-identical to what the generator + // would have written, so every content check is happy — and nothing wrote + // it. + #[test] + fn declared_but_unwritten_is_an_offence() { + let listed = set(&["src/lib/LibReleasedSuites.sol", "src/generated/A.sol"]); + let written = set(&["src/generated/A.sol"]); + let present = set(&["src/lib/LibReleasedSuites.sol", "src/generated/A.sol"]); + let off = offenders(&listed, &written, &present, MANIFEST_PATH); + assert_eq!(off.len(), 1); + assert!(off[0].contains("src/lib/LibReleasedSuites.sol")); + assert!(off[0].contains("emitter is dead")); + } + + #[test] + fn declared_but_missing_from_disk_says_so() { + let listed = set(&["src/generated/Gone.sol"]); + let off = offenders(&listed, &BTreeSet::new(), &BTreeSet::new(), MANIFEST_PATH); + assert_eq!(off.len(), 1); + assert!(off[0].contains("no such file exists")); + } + + #[test] + fn every_declared_path_written_is_clean() { + let all = set(&["src/a.sol", "src/b.sol"]); + assert!(offenders(&all, &all, &all, MANIFEST_PATH).is_empty()); + } + + // The deliberate asymmetry: an incidental write inside the window (forge + // touching a lock file, say) must never redden a job every Rain repo runs. + #[test] + fn a_written_but_undeclared_path_is_not_an_offence() { + let listed = set(&["src/a.sol"]); + let written = set(&["src/a.sol", "soldeer.lock"]); + assert!(offenders(&listed, &written, &written, MANIFEST_PATH).is_empty()); + } + + #[test] + fn an_empty_manifest_declares_nothing_and_is_clean() { + let written = set(&["src/a.sol"]); + assert!(offenders(&BTreeSet::new(), &written, &written, MANIFEST_PATH).is_empty()); + } +} diff --git a/rainix-static/src/main.rs b/rainix-static/src/main.rs index 8a520da..3df275e 100644 --- a/rainix-static/src/main.rs +++ b/rainix-static/src/main.rs @@ -34,6 +34,19 @@ // Nothing is stripped: a shell script reads the bytes on disk. Which // files are prompts and what they may weigh is per-repo, so both are an // input, and a glob matching nothing is an error rather than a pass. +// codegen-witness mark|verify --state [--root ] [--manifest ] +// prove the repo's codegen hooks still EMIT each committed generated +// file, which rainix-copy-artifacts' "re-run the generators, then git +// diff" cannot: a generator that has STOPPED emitting a file rewrites +// nothing, so the already-correct committed copy does not differ and the +// job is green over a dead emitter (rain.factory.deploy#35). `mark` +// records every tracked file's mtime before the first hook; `verify` +// re-stats them after the last one and fails on any path +// script/codegen-manifest.txt declares that nothing wrote. Written-but- +// undeclared paths are a printed note, never a failure, so an incidental +// write inside the window cannot redden every consumer at once. A repo +// with a codegen hook and no manifest fails (and is printed one); a repo +// with neither passes. // snapshots-append-only [--base ] [--root ] // fail if the branch modifies or deletes an existing per-tag deploy-pin // snapshot under // (default root src/generated, base @@ -96,6 +109,7 @@ mod agent_context_cap; mod ci_gate; +mod codegen_witness; mod context_bytes; mod frozen_snapshots; mod mutation_ledger; @@ -212,6 +226,25 @@ fn main() { .unwrap_or_else(|| fail("soldeer-gate: --package required")); soldeer_gate::run(&pkg, flag(&args, "--github-output").as_deref()); } + "codegen-witness" => { + let root = flag(&args, "--root").unwrap_or_else(|| ".".to_string()); + // No default: the state must outlive one step and must not land in + // the worktree, where it would show up in the very git diff this + // check runs beside. The caller names a runner temp path. + let state = flag(&args, "--state") + .unwrap_or_else(|| fail("codegen-witness: --state required")); + match args.get(2).map(String::as_str).unwrap_or("") { + "mark" => codegen_witness::run_mark(Path::new(&root), Path::new(&state)), + "verify" => { + let manifest = flag(&args, "--manifest") + .unwrap_or_else(|| codegen_witness::MANIFEST_PATH.to_string()); + codegen_witness::run_verify(Path::new(&root), Path::new(&state), &manifest); + } + other => fail(&format!( + "codegen-witness: phase must be `mark` or `verify`, got {other:?}" + )), + } + } "snapshots-append-only" => { let base = flag(&args, "--base").unwrap_or_else(|| "origin/main".to_string()); let root = flag(&args, "--root").unwrap_or_else(|| "src/generated".to_string()); @@ -272,8 +305,8 @@ fn main() { eprintln!( "rainix-static: unknown subcommand {other:?} \ (available: no-submodules, agent-context-cap, prompt-cap, \ - snapshots-append-only, mutation-ledger, ci-gate, soldeer-gate, \ - rpc-preflight, release-guard)" + codegen-witness, snapshots-append-only, mutation-ledger, ci-gate, \ + soldeer-gate, rpc-preflight, release-guard)" ); std::process::exit(2); } diff --git a/test/bats/action/codegen-witness.test.bats b/test/bats/action/codegen-witness.test.bats new file mode 100644 index 0000000..37b9467 --- /dev/null +++ b/test/bats/action/codegen-witness.test.bats @@ -0,0 +1,266 @@ +# The half of rainix-copy-artifacts' currency check that a diff cannot make. +# +# That job re-runs the consumer's codegen hooks and then `git diff --exit-code`, +# which is blind to a generator that has STOPPED emitting a file: the committed +# copy is already correct, so nothing is rewritten, nothing differs, and the job +# is green over a dead emitter (rainlanguage/rain.factory.deploy#35). The +# control/mutant pair from that issue is reproduced below against the real +# binary — including the part that makes it invisible, that `git diff` stays +# clean in BOTH halves. +# +# Two layers, as elsewhere in test/bats/action: the composite's wiring with +# `nix` stubbed (what reaches the binary), then the binary itself, which is on +# PATH in every shell. + +setup() { + repo_root="$BATS_TEST_DIRNAME/../../.." + action="$repo_root/.github/actions/codegen-witness/action.yml" + action_script="$(yq -r '.runs.steps[0].run' "$action")" + work="$(mktemp -d)" + state="$work/witness.json" +} + +teardown() { + rm -rf "$work" +} + +# The action script with `nix` stubbed to echo its argv, so what reaches the +# binary is asserted without building it. +run_witness_action() { + RAINIX_CODEGEN_WITNESS_PHASE="$1" \ + RAINIX_CODEGEN_WITNESS_STATE="$2" \ + GITHUB_ACTION_PATH="$repo_root/.github/actions/codegen-witness" \ + ACTION_SCRIPT="$action_script" \ + bash -c ' + nix() { + printf "nix" + printf " <%s>" "$@" + printf "\n" + } + export -f nix + bash -c "$ACTION_SCRIPT" + ' +} + +@test "the phase reaches the binary as the subcommand's phase argument" { + run run_witness_action mark /tmp/w.json + + [ "$status" -eq 0 ] + [[ "$output" == *" <--state> " ]] +} + +@test "both phases are wired through the same action, not two spellings" { + run run_witness_action verify /tmp/w.json + + [ "$status" -eq 0 ] + [[ "$output" == *" <--state> " ]] +} + +@test "the state path is a runner temp file, never a path in the working tree" { + # It must not land in the tree: the job runs `git diff --exit-code` right + # after this check, and a stray state file there would fail it. + local state_expr + state_expr="$(yq -r '.runs.steps[0].env.RAINIX_CODEGEN_WITNESS_STATE' "$action")" + # The single quotes are the point: `${{ runner.temp }}` is a GitHub Actions + # expression that must reach the YAML verbatim, so it is matched as a literal + # prefix and must not be expanded by the shell running this test. + # shellcheck disable=SC2016 + [[ "$state_expr" == '${{ runner.temp }}/'* ]] +} + +# The binary itself, as CI invokes it. + +# A consumer repo as copy-artifacts finds it: a codegen hook, a committed +# generated file, and a manifest declaring it. Files are staged rather than +# committed because `git ls-files` reads the index, and because `git diff` then +# compares the worktree against exactly the "committed" bytes. +mk_consumer() { + git -C "$work" init -q + mkdir -p "$work/script" "$work/src/generated" "$work/src/lib" + printf 'contract Build {}\n' >"$work/script/Build.sol" + printf 'GENERATED SNAPSHOT\n' >"$work/src/generated/Thing.sol" + printf 'GENERATED AGGREGATE\n' >"$work/src/lib/LibReleasedSuites.sol" + cat >"$work/script/codegen-manifest.txt" <<'EOF' +# declared generated files +src/generated/Thing.sol +src/lib/LibReleasedSuites.sol +EOF + git -C "$work" add -A +} + +# Age every file so that a real write during the window is unambiguously a +# later mtime, whatever the filesystem's timestamp resolution. In CI the same +# gap comes for free: checkout runs minutes before the codegen hooks do. +age_tree() { + find "$work" -path "$work/.git" -prune -o -type f -exec touch -m -t 202001010000 {} + +} + +# A live generator: rewrites each file it is given with the SAME bytes it +# already holds, which is what regeneration does on a current tree. +regenerate() { + local f + for f in "$@"; do + local content + content="$(cat "$work/$f")" + printf '%s\n' "$content" >"$work/$f" + done +} + +# THE CONTROL, from the issue: a live generator rewrites the file. Byte +# identical, so `git diff` sees nothing — and the witness sees the write. +@test "a live generator's identical rewrite is witnessed, with git diff clean" { + mk_consumer + age_tree + + run rainix-static codegen-witness mark --root "$work" --state "$state" + [ "$status" -eq 0 ] + + regenerate src/generated/Thing.sol src/lib/LibReleasedSuites.sol + + run rainix-static codegen-witness verify --root "$work" --state "$state" + [ "$status" -eq 0 ] + [[ "$output" == *"clean — 2 declared generated files"* ]] + + # The old check's whole signal, for contrast with the next test. + run git -C "$work" diff --exit-code + [ "$status" -eq 0 ] +} + +# THE MUTANT, from the issue: the one call emitting the aggregate is removed. +# The generator still exits 0, the committed file is untouched and correct, +# `git diff` is still clean — and this is the only thing that notices. +@test "a generator that stopped emitting a file fails, though git diff is clean" { + mk_consumer + age_tree + + run rainix-static codegen-witness mark --root "$work" --state "$state" + [ "$status" -eq 0 ] + + # Every emitter but the aggregate's. + regenerate src/generated/Thing.sol + + run rainix-static codegen-witness verify --root "$work" --state "$state" + [ "$status" -eq 1 ] + [[ "$output" == *"src/lib/LibReleasedSuites.sol"* ]] + [[ "$output" == *"emitter is dead"* ]] + # The file that IS still emitted must not be blamed. + [[ "$output" != *"declares src/generated/Thing.sol generated, but no codegen hook"* ]] + + # The reason the defect was invisible: the check it sits beside is happy. + run git -C "$work" diff --exit-code + [ "$status" -eq 0 ] +} + +@test "a declared file that no longer exists at all is named as such" { + mk_consumer + rm "$work/src/lib/LibReleasedSuites.sol" + age_tree + + rainix-static codegen-witness mark --root "$work" --state "$state" + regenerate src/generated/Thing.sol + + run rainix-static codegen-witness verify --root "$work" --state "$state" + [ "$status" -eq 1 ] + [[ "$output" == *"src/lib/LibReleasedSuites.sol"* ]] + [[ "$output" == *"no such file exists"* ]] +} + +# The workflow is consumed at @main by every Rain repo, most of which generate +# nothing: they must pass without being asked for anything. +@test "a repo with no codegen hook and no manifest passes" { + git -C "$work" init -q + mkdir -p "$work/src" + printf 'contract A {}\n' >"$work/src/A.sol" + git -C "$work" add -A + age_tree + + rainix-static codegen-witness mark --root "$work" --state "$state" + + run rainix-static codegen-witness verify --root "$work" --state "$state" + [ "$status" -eq 0 ] + [[ "$output" == *"no codegen hook and no"* ]] +} + +@test "a repo that runs codegen but declares nothing fails, and is printed one" { + mk_consumer + rm "$work/script/codegen-manifest.txt" + git -C "$work" rm -q --cached script/codegen-manifest.txt + age_tree + + rainix-static codegen-witness mark --root "$work" --state "$state" + regenerate src/generated/Thing.sol src/lib/LibReleasedSuites.sol + + run rainix-static codegen-witness verify --root "$work" --state "$state" + [ "$status" -eq 1 ] + [[ "$output" == *"script/Build.sol"* ]] + # The starting point it prints is what this run actually wrote. + [[ "$output" == *"src/generated/Thing.sol"* ]] + [[ "$output" == *"src/lib/LibReleasedSuites.sol"* ]] +} + +# Deliberate asymmetry: an incidental write inside the window must never redden +# a job every Rain repo runs. It is reported, not enforced. +@test "a written but undeclared file is a note, not a failure" { + mk_consumer + printf 'lock\n' >"$work/soldeer.lock" + git -C "$work" add soldeer.lock + age_tree + + rainix-static codegen-witness mark --root "$work" --state "$state" + regenerate src/generated/Thing.sol src/lib/LibReleasedSuites.sol soldeer.lock + + run rainix-static codegen-witness verify --root "$work" --state "$state" + [ "$status" -eq 0 ] + [[ "$output" == *"note — written but not declared"* ]] + [[ "$output" == *"soldeer.lock"* ]] +} + +# A deletion is not a write. The diff catches a deleted generated file on its +# own; blaming the emitter for it would send the reader to the wrong place. +@test "a file the hooks deleted is not counted as written" { + mk_consumer + age_tree + + rainix-static codegen-witness mark --root "$work" --state "$state" + regenerate src/generated/Thing.sol + rm "$work/src/lib/LibReleasedSuites.sol" + + run rainix-static codegen-witness verify --root "$work" --state "$state" + [ "$status" -eq 1 ] + [[ "$output" == *"no such file exists"* ]] +} + +@test "verify without a prior mark fails naming mark, rather than passing" { + mk_consumer + + run rainix-static codegen-witness verify --root "$work" --state "$state" + [ "$status" -eq 1 ] + [[ "$output" == *"codegen-witness mark"* ]] +} + +@test "an unknown phase fails rather than defaulting to one" { + run rainix-static codegen-witness --root "$work" --state "$state" + [ "$status" -eq 1 ] + [[ "$output" == *"mark"* ]] + [[ "$output" == *"verify"* ]] +} + +@test "a missing --state fails rather than inventing a path" { + run rainix-static codegen-witness mark --root "$work" + [ "$status" -eq 1 ] + [[ "$output" == *"--state"* ]] +} + +# The default the workflow relies on: neither the action nor the workflow passes +# --manifest, so the path the binary defaults to is the one consumers commit. +@test "the manifest path defaults to script/codegen-manifest.txt" { + mk_consumer + age_tree + + rainix-static codegen-witness mark --root "$work" --state "$state" + regenerate src/generated/Thing.sol + + run rainix-static codegen-witness verify --root "$work" --state "$state" + [ "$status" -eq 1 ] + [[ "$output" == *"script/codegen-manifest.txt declares"* ]] +} diff --git a/test/bats/workflow/rainix-copy-artifacts.test.bats b/test/bats/workflow/rainix-copy-artifacts.test.bats new file mode 100644 index 0000000..6984af3 --- /dev/null +++ b/test/bats/workflow/rainix-copy-artifacts.test.bats @@ -0,0 +1,131 @@ +# Nothing in this repo executes rainix-copy-artifacts.yaml — it is +# `workflow_call` only, so its only runners are the consumer repos, which means +# a step silently dropped from it goes unnoticed here and ungated everywhere. +# +# What the codegen witness DOES is covered in test/bats/action/codegen-witness +# and in rainix-static/src/codegen_witness.rs. What is asserted here is the part +# that only the workflow can get wrong: that both phases are still invoked, and +# that they still bracket every codegen hook — a witness taken on the wrong side +# of a step measures nothing and reports green, which is the exact failure mode +# it was added to remove (rainlanguage/rain.factory.deploy#35). + +setup() { + repo_root="$BATS_TEST_DIRNAME/../../.." + workflow="$repo_root/.github/workflows/rainix-copy-artifacts.yaml" + witness="$repo_root/rainix-static/src/codegen_witness.rs" + names="$(yq -r '.jobs["copy-artifacts"].steps[] | .name // "«unnamed»"' "$workflow")" + runs="$(yq -r '.jobs["copy-artifacts"].steps[] | select(.run) | .run' "$workflow")" + mark_step="Mark tracked files before codegen" + verify_step="Assert every declared generated file was written" +} + +# 1-based position of a step in the job, by name. +step_at() { + echo "$names" | grep -nxF "$1" | cut -d: -f1 +} + +# The `uses`/`with.phase` of a step, by name. +step_field() { + yq -r ".jobs[\"copy-artifacts\"].steps[] | select(.name == \"$1\") | $2" "$workflow" +} + +@test "the job invokes both codegen-witness phases" { + [ "$(step_field "$mark_step" .uses)" = "rainlanguage/rainix/.github/actions/codegen-witness@main" ] + [ "$(step_field "$mark_step" .with.phase)" = "mark" ] + [ "$(step_field "$verify_step" .uses)" = "rainlanguage/rainix/.github/actions/codegen-witness@main" ] + [ "$(step_field "$verify_step" .with.phase)" = "verify" ] +} + +# THE ordering invariant. A hook that runs before `mark` or after `verify` is +# outside the window, so whatever it writes looks unwritten — or, worse, a hook +# added later beside `forge fmt` would make a dead emitter look alive. +@test "both witness phases bracket every codegen hook step" { + local mark verify + mark="$(step_at "$mark_step")" + verify="$(step_at "$verify_step")" + [ -n "$mark" ] + [ -n "$verify" ] + [ "$mark" -lt "$verify" ] + + # Every step that INVOKES a consumer codegen hook (`./script/`), by + # position. The invocation form is what matters: the final assertion step + # names the same paths in its error text without running any of them. + local hooked + hooked="$(yq -r '.jobs["copy-artifacts"].steps | to_entries[] + | select((.value.run // "") | test("\./script/(build-meta\.sh|Build\.sol|CopyArtifacts\.sol|build\.sh)")) + | (.key + 1 | tostring) + " " + (.value.name // "«unnamed»")' "$workflow")" + [ -n "$hooked" ] + + local pos name + while read -r pos name; do + if [ "$pos" -lt "$mark" ] || [ "$pos" -gt "$verify" ]; then + echo "FAIL: codegen step '$name' (position $pos) is outside the witness window ($mark..$verify)" >&2 + return 1 + fi + done <<<"$hooked" +} + +# `forge fmt` rewrites sources of its own, so a witness taken after it cannot +# tell a file a generator wrote from one the formatter touched. +@test "the witness closes before forge fmt runs" { + local verify fmt + verify="$(step_at "$verify_step")" + fmt="$(echo "$names" | grep -n 'Format' | cut -d: -f1)" + [ -n "$fmt" ] + [ "$verify" -lt "$fmt" ] +} + +# The diff is the other half of the check and must not be traded away for this +# one: content currency and emitter liveness are different claims. +@test "the committed-artifacts diff is still asserted after the witness" { + local verify diff + verify="$(step_at "$verify_step")" + diff="$(step_at "Assert committed artifacts match freshly built")" + [ -n "$diff" ] + [ "$verify" -lt "$diff" ] + echo "$runs" | grep -q 'git diff --exit-code' +} + +# A `hashFiles` guard would let a repo delete its codegen hooks and take the +# check that was watching them along with it. Whether there is anything to +# witness is the binary's decision, in one tested place. +@test "neither witness step is conditional" { + local step guard + for step in "$mark_step" "$verify_step"; do + guard="$(step_field "$step" '.["if"] // "none"')" + if [ "$guard" != "none" ]; then + echo "FAIL: '$step' is guarded by: $guard" >&2 + return 1 + fi + done +} + +# The binary decides "does this repo run codegen?" from its own HOOKS list. If +# the workflow gains or renames a hook and that list does not follow, a repo +# with codegen is told it has none and is never asked for a manifest. +@test "the binary's hook list is exactly the hooks the workflow runs" { + local declared invoked + declared="$(sed -n '/pub(crate) const HOOKS/,/^];/p' "$witness" | + grep -o '"script/[^"]*"' | tr -d '"' | sort)" + invoked="$(echo "$runs" | grep -oE '\./script/[A-Za-z0-9_-]+\.(sol|sh)' | + sed 's|^\./||' | sort -u)" + [ -n "$declared" ] + if [ "$declared" != "$invoked" ]; then + echo "FAIL: codegen_witness.rs HOOKS and the hooks the workflow invokes disagree" >&2 + diff <(echo "$declared") <(echo "$invoked") >&2 || true + return 1 + fi +} + +@test "every rainix-copy-artifacts run step resolves rainix through the pinned sha" { + local sha unpinned + sha="$(yq -r '.env.RAINIX_SHA' "$workflow")" + [ -n "$sha" ] + [ "$sha" != "null" ] + unpinned="$(echo "$runs" | grep 'github:rainlanguage/rainix' | grep -v 'env.RAINIX_SHA' || true)" + if [ -n "$unpinned" ]; then + echo "FAIL: unpinned rainix refs in rainix-copy-artifacts.yaml:" >&2 + echo "$unpinned" >&2 + return 1 + fi +} From ec68201b7c324e7f1bf2eddba3aa38bbf8130d85 Mon Sep 17 00:00:00 2001 From: baku-ccron Date: Sun, 20 Sep 2026 13:26:15 +0000 Subject: [PATCH 2/3] docs: cut the codegen-witness design essays from comments Co-Authored-By: Claude Opus 5 (1M context) --- .github/actions/codegen-witness/action.yml | 14 +-- .github/workflows/rainix-copy-artifacts.yaml | 20 +--- rainix-static/src/codegen_witness.rs | 94 +------------------ rainix-static/src/main.rs | 19 +--- test/bats/action/codegen-witness.test.bats | 53 +---------- .../workflow/rainix-copy-artifacts.test.bats | 29 ------ 6 files changed, 11 insertions(+), 218 deletions(-) diff --git a/.github/actions/codegen-witness/action.yml b/.github/actions/codegen-witness/action.yml index 252b03d..9c0fb89 100644 --- a/.github/actions/codegen-witness/action.yml +++ b/.github/actions/codegen-witness/action.yml @@ -12,22 +12,12 @@ runs: - name: Codegen witness shell: bash env: - # Via env rather than interpolated into the script text: caller input - # never gets pasted into a shell command. + # Via env, never interpolated into the script: caller input must not reach a shell command. RAINIX_CODEGEN_WITNESS_PHASE: ${{ inputs.phase }} - # The two phases are separate steps, so the state has to outlive one of - # them — and it must not land in the working tree, where it would show - # up in the very `git diff` this check runs beside. + # Must not land in the working tree: it would show up in the git diff this check runs beside. RAINIX_CODEGEN_WITNESS_STATE: ${{ runner.temp }}/rainix-codegen-witness.json run: | set -euo pipefail - # Single source of truth: the Rust rainix-static binary (its unit tests - # run in the nix build, and test/bats/action/codegen-witness.test.bats - # runs the built binary end to end). The path: flake ref runs it from - # THIS composite's own checkout, so the check version always matches the - # action version regardless of any RAINIX_SHA the caller pins — which - # also means a new subcommand ships with the action instead of waiting - # on a sha bump that would otherwise leave every consumer red in between. nix run "path:$(cd "$GITHUB_ACTION_PATH/../../.." && pwd)#rainix-static" -- \ codegen-witness "$RAINIX_CODEGEN_WITNESS_PHASE" \ --state "$RAINIX_CODEGEN_WITNESS_STATE" diff --git a/.github/workflows/rainix-copy-artifacts.yaml b/.github/workflows/rainix-copy-artifacts.yaml index 245efeb..3476843 100644 --- a/.github/workflows/rainix-copy-artifacts.yaml +++ b/.github/workflows/rainix-copy-artifacts.yaml @@ -21,22 +21,6 @@ jobs: - name: Install soldeer dependencies if: hashFiles('soldeer.lock') != '' run: nix develop github:rainlanguage/rainix/${{ env.RAINIX_SHA }}#sol-shell -c forge soldeer install - # Half of the currency check that the diff at the bottom cannot make. - # Re-running the generators and diffing proves the committed CONTENT is - # current, but it is blind to a generator that has STOPPED emitting a file: - # nothing is rewritten, the already-correct committed copy does not differ, - # and the job is green over a dead emitter - # (rainlanguage/rain.factory.deploy#35, verified there with a control). - # `mark` records every tracked file's mtime before the first codegen hook - # below; `verify`, after the last one, fails on any path - # `script/codegen-manifest.txt` declares generated that nothing wrote. - # Semantics: rainix-static/src/codegen_witness.rs. - # - # Unconditional, with no `hashFiles` guard: the binary decides whether - # there is anything to witness (a repo with no codegen hook and no manifest - # passes), so that predicate lives in one tested place rather than - # duplicated across two steps where it can drift — and a repo cannot make - # the check vanish by deleting the hooks it was checking. - name: Mark tracked files before codegen uses: rainlanguage/rainix/.github/actions/codegen-witness@main with: @@ -73,9 +57,7 @@ jobs: - name: Regenerate derived artifacts if: hashFiles('script/build.sh') != '' run: ./script/build.sh - # The other half, BEFORE `forge fmt`: fmt rewrites sources of its own, so - # running it first would forge the witness — a file fmt happened to touch - # would look like one a generator wrote. + # Before `forge fmt`: fmt rewrites sources of its own, which would forge the witness. - name: Assert every declared generated file was written uses: rainlanguage/rainix/.github/actions/codegen-witness@main with: diff --git a/rainix-static/src/codegen_witness.rs b/rainix-static/src/codegen_witness.rs index a758226..05efe1e 100644 --- a/rainix-static/src/codegen_witness.rs +++ b/rainix-static/src/codegen_witness.rs @@ -1,66 +1,10 @@ -//! Witness that the repo's codegen hooks still EMIT each committed generated -//! file, which re-running them and diffing cannot see. -//! -//! `rainix-copy-artifacts` currency-checks committed generated sources by -//! re-running every consumer codegen hook and then `git diff --exit-code`. That -//! method has one blind spot, and it is total: a generator that has STOPPED -//! emitting a file writes nothing, so the committed copy — already correct — -//! is left exactly as it is, nothing differs, and the job is green over a dead -//! emitter (rainlanguage/rain.factory.deploy#35, reproduced with a control: a -//! marker appended to the generated file survived the generator run once its -//! one emitting call was removed, with `git diff` clean throughout). -//! -//! The blind spot is structural. The generator's output is the check's only -//! oracle for what the committed files should contain, so a file the generator -//! never writes has no oracle at all. Seeing it requires an INDEPENDENT -//! statement of which committed files are generated, which is what -//! `script/codegen-manifest.txt` is. The check then has two halves: the diff -//! says the content is current, and this says something actually wrote it. -//! -//! ## What is witnessed -//! -//! `mark` records the mtime of every git-tracked file before the first codegen -//! hook runs; `verify` re-stats them after the last one and calls a file -//! WRITTEN when it exists now and its mtime moved. That is the property the -//! defect is about — `vm.writeFile` and friends rewrite unconditionally, so a -//! live emitter always moves the mtime even when the bytes are identical, -//! which is exactly the case the diff cannot distinguish from a dead one. -//! -//! Scope is git-tracked files: the currency check is about COMMITTED generated -//! sources, and restricting to them keeps `out/`, `cache/`, `broadcast/` and -//! `dependencies/` out of the witness whether or not a repo ignores them -//! properly. -//! -//! ## Why listed-must-be-written, and not set equality -//! -//! Set equality (every written file must be listed) would keep the manifest -//! self-maintaining, but it couples an org-wide gate to every incidental write -//! inside the window — `forge build` is in there, and the day it starts -//! rewriting a lock file it reddens every consumer at once. The claim worth -//! making is the one the defect is about: a path this repo DECLARES as -//! generated must have been written. Files written but not listed are printed -//! as a note instead, so a newly generated file is discoverable without being -//! able to break anyone. -//! -//! The residual gap is the mirror of that choice, and is named here rather than -//! papered over: a generated file nobody has listed yet is not protected. So is -//! a brand-new generated file that is never committed — `git diff --exit-code` -//! does not see untracked files either, which is a separate hole in the same -//! job. - use std::collections::BTreeSet; use std::path::Path; use std::process::Command; use std::time::UNIX_EPOCH; -/// The committed declaration, beside the hooks it describes (`script/Build.sol`, -/// `script/build.sh`, ...). One fixed path: the workflow is consumed at `@main` -/// by every Rain repo and cannot go looking for a per-repo convention. pub(crate) const MANIFEST_PATH: &str = "script/codegen-manifest.txt"; -/// The consumer-supplied codegen hooks `rainix-copy-artifacts` runs. Presence of -/// any one of them is what makes a manifest mandatory: a repo with no codegen -/// has nothing to declare and must not be asked to declare it. pub(crate) const HOOKS: [&str; 4] = [ "script/build-meta.sh", "script/Build.sol", @@ -68,8 +12,6 @@ pub(crate) const HOOKS: [&str; 4] = [ "script/build.sh", ]; -/// Header of a rendered manifest. Present so the file explains itself to -/// whoever opens it in a diff, and parsed back out as a comment. const HEADER: &str = "\ # Committed files this repo's codegen hooks generate — one path per line. # @@ -84,8 +26,6 @@ const HEADER: &str = "\ # on). Blank lines and # comments are ignored. "; -/// Paths a manifest lists. Blank lines and `#` comments are ignored, and each -/// path is trimmed, so the file can carry its own explanation. pub(crate) fn parse_manifest(text: &str) -> BTreeSet { text.lines() .map(str::trim) @@ -94,8 +34,6 @@ pub(crate) fn parse_manifest(text: &str) -> BTreeSet { .collect() } -/// A manifest file's content for `paths`: the header, then one path per line in -/// sorted order. What `verify` prints for a consumer to commit verbatim. pub(crate) fn render_manifest(paths: &BTreeSet) -> String { let mut out = String::from(HEADER); for path in paths { @@ -105,11 +43,6 @@ pub(crate) fn render_manifest(paths: &BTreeSet) -> String { out } -/// Offenders for a manifest that exists: every declared path nothing wrote. -/// -/// Declared-but-absent is reported as the same offence — a path that is not -/// even on disk was certainly not written, and saying "no hook wrote it" of a -/// file that does not exist would send the reader looking for the wrong thing. pub(crate) fn offenders( listed: &BTreeSet, written: &BTreeSet, @@ -137,13 +70,11 @@ pub(crate) fn offenders( out } -/// Repo-relative paths git tracks under `root`. fn tracked(root: &Path) -> Result, String> { let out = Command::new("git") .arg("-C") .arg(root) - // -z: a path may contain anything but NUL, and git otherwise quotes the - // awkward ones, which would not match the manifest. + // -z: git otherwise quotes awkward paths, which would not match the manifest. .args(["ls-files", "-z"]) .output() .map_err(|e| format!("failed to run git ls-files: {e}"))?; @@ -161,17 +92,12 @@ fn tracked(root: &Path) -> Result, String> { .collect()) } -/// Nanoseconds-since-epoch mtime of `root/path`, or `None` when it is not there -/// (git tracks a path the worktree may not currently hold) or its mtime cannot -/// be read at all. fn mtime_nanos(root: &Path, path: &str) -> Option { let meta = std::fs::metadata(root.join(path)).ok()?; let since = meta.modified().ok()?.duration_since(UNIX_EPOCH).ok()?; u64::try_from(since.as_nanos()).ok() } -/// Record every tracked file's mtime into `state`, to be compared after the -/// codegen hooks have run. Returns how many files were marked. pub(crate) fn mark(root: &Path, state: &Path) -> Result { let files = tracked(root)?; let mut map = serde_json::Map::new(); @@ -188,11 +114,6 @@ pub(crate) fn mark(root: &Path, state: &Path) -> Result { Ok(files.len()) } -/// The marked files that were WRITTEN since `mark`, and those that exist now. -/// -/// Written means present now with a different mtime: a rewrite with identical -/// bytes still moves it, which is the whole point, while a file the hooks -/// DELETED is not a write (and `git diff` catches a deletion on its own). pub(crate) fn written_since( root: &Path, state: &Path, @@ -225,7 +146,6 @@ pub(crate) fn written_since( Ok((written, present)) } -/// `mark` as a subcommand: record and report, or fail loud. pub(crate) fn run_mark(root: &Path, state: &Path) { match mark(root, state) { Err(e) => crate::fail(&format!("codegen-witness mark: {e}")), @@ -236,8 +156,6 @@ pub(crate) fn run_mark(root: &Path, state: &Path) { } } -/// `verify` as a subcommand: the manifest's declarations against what the hooks -/// actually wrote. pub(crate) fn run_verify(root: &Path, state: &Path, manifest_rel: &str) { let (written, present) = match written_since(root, state) { Ok(sets) => sets, @@ -254,8 +172,6 @@ pub(crate) fn run_verify(root: &Path, state: &Path, manifest_rel: &str) { )), }; - // The manifest itself is written by hand and by this command's output, never - // by a codegen hook, so it is never part of its own witness. let mut written: BTreeSet = written; written.remove(manifest_rel); @@ -292,8 +208,6 @@ pub(crate) fn run_verify(root: &Path, state: &Path, manifest_rel: &str) { "codegen-witness: clean — {} declared generated files, each written this run", listed.len() ); - // A note, never a failure: see the module doc on why an unlisted write - // must not be able to redden an org-wide job. if !unlisted.is_empty() { println!( "codegen-witness: note — written but not declared in {manifest_rel}, so nothing \ @@ -333,7 +247,6 @@ mod tests { let paths = set(&["src/b.sol", "src/a.sol"]); let rendered = render_manifest(&paths); assert!(rendered.starts_with('#')); - // Sorted, so the committed file does not churn on set ordering. let body: Vec<&str> = rendered .lines() .filter(|l| !l.starts_with('#') && !l.is_empty()) @@ -348,9 +261,6 @@ mod tests { assert!(parse_manifest(&rendered).is_empty()); } - // THE defect: the file is on disk and byte-identical to what the generator - // would have written, so every content check is happy — and nothing wrote - // it. #[test] fn declared_but_unwritten_is_an_offence() { let listed = set(&["src/lib/LibReleasedSuites.sol", "src/generated/A.sol"]); @@ -376,8 +286,6 @@ mod tests { assert!(offenders(&all, &all, &all, MANIFEST_PATH).is_empty()); } - // The deliberate asymmetry: an incidental write inside the window (forge - // touching a lock file, say) must never redden a job every Rain repo runs. #[test] fn a_written_but_undeclared_path_is_not_an_offence() { let listed = set(&["src/a.sol"]); diff --git a/rainix-static/src/main.rs b/rainix-static/src/main.rs index 3df275e..3f540ea 100644 --- a/rainix-static/src/main.rs +++ b/rainix-static/src/main.rs @@ -35,18 +35,10 @@ // files are prompts and what they may weigh is per-repo, so both are an // input, and a glob matching nothing is an error rather than a pass. // codegen-witness mark|verify --state [--root ] [--manifest ] -// prove the repo's codegen hooks still EMIT each committed generated -// file, which rainix-copy-artifacts' "re-run the generators, then git -// diff" cannot: a generator that has STOPPED emitting a file rewrites -// nothing, so the already-correct committed copy does not differ and the -// job is green over a dead emitter (rain.factory.deploy#35). `mark` -// records every tracked file's mtime before the first hook; `verify` -// re-stats them after the last one and fails on any path -// script/codegen-manifest.txt declares that nothing wrote. Written-but- -// undeclared paths are a printed note, never a failure, so an incidental -// write inside the window cannot redden every consumer at once. A repo -// with a codegen hook and no manifest fails (and is printed one); a repo -// with neither passes. +// fail if any path script/codegen-manifest.txt declares generated was not +// written between `mark` (before the first codegen hook) and `verify` +// (after the last), by mtime. A repo with a codegen hook and no manifest +// fails (and is printed one); a repo with neither passes. // snapshots-append-only [--base ] [--root ] // fail if the branch modifies or deletes an existing per-tag deploy-pin // snapshot under // (default root src/generated, base @@ -228,9 +220,6 @@ fn main() { } "codegen-witness" => { let root = flag(&args, "--root").unwrap_or_else(|| ".".to_string()); - // No default: the state must outlive one step and must not land in - // the worktree, where it would show up in the very git diff this - // check runs beside. The caller names a runner temp path. let state = flag(&args, "--state") .unwrap_or_else(|| fail("codegen-witness: --state required")); match args.get(2).map(String::as_str).unwrap_or("") { diff --git a/test/bats/action/codegen-witness.test.bats b/test/bats/action/codegen-witness.test.bats index 37b9467..302ddad 100644 --- a/test/bats/action/codegen-witness.test.bats +++ b/test/bats/action/codegen-witness.test.bats @@ -1,17 +1,3 @@ -# The half of rainix-copy-artifacts' currency check that a diff cannot make. -# -# That job re-runs the consumer's codegen hooks and then `git diff --exit-code`, -# which is blind to a generator that has STOPPED emitting a file: the committed -# copy is already correct, so nothing is rewritten, nothing differs, and the job -# is green over a dead emitter (rainlanguage/rain.factory.deploy#35). The -# control/mutant pair from that issue is reproduced below against the real -# binary — including the part that makes it invisible, that `git diff` stays -# clean in BOTH halves. -# -# Two layers, as elsewhere in test/bats/action: the composite's wiring with -# `nix` stubbed (what reaches the binary), then the binary itself, which is on -# PATH in every shell. - setup() { repo_root="$BATS_TEST_DIRNAME/../../.." action="$repo_root/.github/actions/codegen-witness/action.yml" @@ -24,8 +10,6 @@ teardown() { rm -rf "$work" } -# The action script with `nix` stubbed to echo its argv, so what reaches the -# binary is asserted without building it. run_witness_action() { RAINIX_CODEGEN_WITNESS_PHASE="$1" \ RAINIX_CODEGEN_WITNESS_STATE="$2" \ @@ -57,23 +41,14 @@ run_witness_action() { } @test "the state path is a runner temp file, never a path in the working tree" { - # It must not land in the tree: the job runs `git diff --exit-code` right - # after this check, and a stray state file there would fail it. local state_expr state_expr="$(yq -r '.runs.steps[0].env.RAINIX_CODEGEN_WITNESS_STATE' "$action")" - # The single quotes are the point: `${{ runner.temp }}` is a GitHub Actions - # expression that must reach the YAML verbatim, so it is matched as a literal - # prefix and must not be expanded by the shell running this test. + # `${{ runner.temp }}` is a GitHub Actions expression: single-quoted so the shell cannot expand it. # shellcheck disable=SC2016 [[ "$state_expr" == '${{ runner.temp }}/'* ]] } -# The binary itself, as CI invokes it. - -# A consumer repo as copy-artifacts finds it: a codegen hook, a committed -# generated file, and a manifest declaring it. Files are staged rather than -# committed because `git ls-files` reads the index, and because `git diff` then -# compares the worktree against exactly the "committed" bytes. +# Staged, not committed: git ls-files reads the index. mk_consumer() { git -C "$work" init -q mkdir -p "$work/script" "$work/src/generated" "$work/src/lib" @@ -88,15 +63,11 @@ EOF git -C "$work" add -A } -# Age every file so that a real write during the window is unambiguously a -# later mtime, whatever the filesystem's timestamp resolution. In CI the same -# gap comes for free: checkout runs minutes before the codegen hooks do. +# Filesystem mtime resolution: a write inside the window must be unambiguously later. age_tree() { find "$work" -path "$work/.git" -prune -o -type f -exec touch -m -t 202001010000 {} + } -# A live generator: rewrites each file it is given with the SAME bytes it -# already holds, which is what regeneration does on a current tree. regenerate() { local f for f in "$@"; do @@ -106,8 +77,6 @@ regenerate() { done } -# THE CONTROL, from the issue: a live generator rewrites the file. Byte -# identical, so `git diff` sees nothing — and the witness sees the write. @test "a live generator's identical rewrite is witnessed, with git diff clean" { mk_consumer age_tree @@ -121,14 +90,10 @@ regenerate() { [ "$status" -eq 0 ] [[ "$output" == *"clean — 2 declared generated files"* ]] - # The old check's whole signal, for contrast with the next test. run git -C "$work" diff --exit-code [ "$status" -eq 0 ] } -# THE MUTANT, from the issue: the one call emitting the aggregate is removed. -# The generator still exits 0, the committed file is untouched and correct, -# `git diff` is still clean — and this is the only thing that notices. @test "a generator that stopped emitting a file fails, though git diff is clean" { mk_consumer age_tree @@ -136,17 +101,14 @@ regenerate() { run rainix-static codegen-witness mark --root "$work" --state "$state" [ "$status" -eq 0 ] - # Every emitter but the aggregate's. regenerate src/generated/Thing.sol run rainix-static codegen-witness verify --root "$work" --state "$state" [ "$status" -eq 1 ] [[ "$output" == *"src/lib/LibReleasedSuites.sol"* ]] [[ "$output" == *"emitter is dead"* ]] - # The file that IS still emitted must not be blamed. [[ "$output" != *"declares src/generated/Thing.sol generated, but no codegen hook"* ]] - # The reason the defect was invisible: the check it sits beside is happy. run git -C "$work" diff --exit-code [ "$status" -eq 0 ] } @@ -165,8 +127,6 @@ regenerate() { [[ "$output" == *"no such file exists"* ]] } -# The workflow is consumed at @main by every Rain repo, most of which generate -# nothing: they must pass without being asked for anything. @test "a repo with no codegen hook and no manifest passes" { git -C "$work" init -q mkdir -p "$work/src" @@ -193,13 +153,10 @@ regenerate() { run rainix-static codegen-witness verify --root "$work" --state "$state" [ "$status" -eq 1 ] [[ "$output" == *"script/Build.sol"* ]] - # The starting point it prints is what this run actually wrote. [[ "$output" == *"src/generated/Thing.sol"* ]] [[ "$output" == *"src/lib/LibReleasedSuites.sol"* ]] } -# Deliberate asymmetry: an incidental write inside the window must never redden -# a job every Rain repo runs. It is reported, not enforced. @test "a written but undeclared file is a note, not a failure" { mk_consumer printf 'lock\n' >"$work/soldeer.lock" @@ -215,8 +172,6 @@ regenerate() { [[ "$output" == *"soldeer.lock"* ]] } -# A deletion is not a write. The diff catches a deleted generated file on its -# own; blaming the emitter for it would send the reader to the wrong place. @test "a file the hooks deleted is not counted as written" { mk_consumer age_tree @@ -251,8 +206,6 @@ regenerate() { [[ "$output" == *"--state"* ]] } -# The default the workflow relies on: neither the action nor the workflow passes -# --manifest, so the path the binary defaults to is the one consumers commit. @test "the manifest path defaults to script/codegen-manifest.txt" { mk_consumer age_tree diff --git a/test/bats/workflow/rainix-copy-artifacts.test.bats b/test/bats/workflow/rainix-copy-artifacts.test.bats index 6984af3..37fc5be 100644 --- a/test/bats/workflow/rainix-copy-artifacts.test.bats +++ b/test/bats/workflow/rainix-copy-artifacts.test.bats @@ -1,14 +1,3 @@ -# Nothing in this repo executes rainix-copy-artifacts.yaml — it is -# `workflow_call` only, so its only runners are the consumer repos, which means -# a step silently dropped from it goes unnoticed here and ungated everywhere. -# -# What the codegen witness DOES is covered in test/bats/action/codegen-witness -# and in rainix-static/src/codegen_witness.rs. What is asserted here is the part -# that only the workflow can get wrong: that both phases are still invoked, and -# that they still bracket every codegen hook — a witness taken on the wrong side -# of a step measures nothing and reports green, which is the exact failure mode -# it was added to remove (rainlanguage/rain.factory.deploy#35). - setup() { repo_root="$BATS_TEST_DIRNAME/../../.." workflow="$repo_root/.github/workflows/rainix-copy-artifacts.yaml" @@ -19,12 +8,10 @@ setup() { verify_step="Assert every declared generated file was written" } -# 1-based position of a step in the job, by name. step_at() { echo "$names" | grep -nxF "$1" | cut -d: -f1 } -# The `uses`/`with.phase` of a step, by name. step_field() { yq -r ".jobs[\"copy-artifacts\"].steps[] | select(.name == \"$1\") | $2" "$workflow" } @@ -36,9 +23,6 @@ step_field() { [ "$(step_field "$verify_step" .with.phase)" = "verify" ] } -# THE ordering invariant. A hook that runs before `mark` or after `verify` is -# outside the window, so whatever it writes looks unwritten — or, worse, a hook -# added later beside `forge fmt` would make a dead emitter look alive. @test "both witness phases bracket every codegen hook step" { local mark verify mark="$(step_at "$mark_step")" @@ -47,9 +31,6 @@ step_field() { [ -n "$verify" ] [ "$mark" -lt "$verify" ] - # Every step that INVOKES a consumer codegen hook (`./script/`), by - # position. The invocation form is what matters: the final assertion step - # names the same paths in its error text without running any of them. local hooked hooked="$(yq -r '.jobs["copy-artifacts"].steps | to_entries[] | select((.value.run // "") | test("\./script/(build-meta\.sh|Build\.sol|CopyArtifacts\.sol|build\.sh)")) @@ -65,8 +46,6 @@ step_field() { done <<<"$hooked" } -# `forge fmt` rewrites sources of its own, so a witness taken after it cannot -# tell a file a generator wrote from one the formatter touched. @test "the witness closes before forge fmt runs" { local verify fmt verify="$(step_at "$verify_step")" @@ -75,8 +54,6 @@ step_field() { [ "$verify" -lt "$fmt" ] } -# The diff is the other half of the check and must not be traded away for this -# one: content currency and emitter liveness are different claims. @test "the committed-artifacts diff is still asserted after the witness" { local verify diff verify="$(step_at "$verify_step")" @@ -86,9 +63,6 @@ step_field() { echo "$runs" | grep -q 'git diff --exit-code' } -# A `hashFiles` guard would let a repo delete its codegen hooks and take the -# check that was watching them along with it. Whether there is anything to -# witness is the binary's decision, in one tested place. @test "neither witness step is conditional" { local step guard for step in "$mark_step" "$verify_step"; do @@ -100,9 +74,6 @@ step_field() { done } -# The binary decides "does this repo run codegen?" from its own HOOKS list. If -# the workflow gains or renames a hook and that list does not follow, a repo -# with codegen is told it has none and is never asked for a manifest. @test "the binary's hook list is exactly the hooks the workflow runs" { local declared invoked declared="$(sed -n '/pub(crate) const HOOKS/,/^];/p' "$witness" | From 70b8d944fd6f3daa0b1b9867883cd19146a09006 Mon Sep 17 00:00:00 2001 From: baku-ccron Date: Mon, 21 Sep 2026 10:31:33 +0000 Subject: [PATCH 3/3] ci: have the generator declare which paths it owns, and check it wrote them Replaces the hand-written script/codegen-manifest.txt witness with a declaration the generator computes: `rainix-codegen owns|wrote ` on stdout, teed into one log per job, compared by `rainix-static codegen-declaration`. No repo declares anything, so none goes red on merge. Co-Authored-By: Claude Opus 5 (1M context) --- .../actions/codegen-declaration/action.yml | 15 + .github/actions/codegen-witness/action.yml | 23 -- .github/workflows/rainix-copy-artifacts.yaml | 32 +- README.md | 32 +- flake.nix | 2 +- rainix-static/src/codegen_declaration.rs | 200 ++++++++++++ rainix-static/src/codegen_witness.rs | 301 ------------------ rainix-static/src/main.rs | 37 +-- .../bats/action/codegen-declaration.test.bats | 142 +++++++++ test/bats/action/codegen-witness.test.bats | 219 ------------- .../workflow/rainix-copy-artifacts.test.bats | 124 +++----- 11 files changed, 458 insertions(+), 669 deletions(-) create mode 100644 .github/actions/codegen-declaration/action.yml delete mode 100644 .github/actions/codegen-witness/action.yml create mode 100644 rainix-static/src/codegen_declaration.rs delete mode 100644 rainix-static/src/codegen_witness.rs create mode 100644 test/bats/action/codegen-declaration.test.bats delete mode 100644 test/bats/action/codegen-witness.test.bats diff --git a/.github/actions/codegen-declaration/action.yml b/.github/actions/codegen-declaration/action.yml new file mode 100644 index 0000000..0437ac8 --- /dev/null +++ b/.github/actions/codegen-declaration/action.yml @@ -0,0 +1,15 @@ +name: codegen-declaration +description: >- + Fails `rainix-copy-artifacts` when a codegen hook declared a generated path it then did not write. Re-running the generators and diffing proves the committed CONTENT is current, but a generator that has STOPPED emitting a file writes nothing, so the already-correct committed copy is left alone, nothing differs and the job is green over a dead emitter (rainlanguage/rain.factory.deploy#35). Only the generator knows which paths it owns rather than deliberately leaves frozen, so it declares them on stdout — `rainix-codegen owns ` and `rainix-codegen wrote ` — and the workflow tees that into the log this reads. A repo whose hooks declare nothing is a no-op. +runs: + using: composite + steps: + - name: Check the codegen declaration + shell: bash + run: | + set -euo pipefail + # path: ref runs the check from THIS composite's own checkout, so the + # check version always matches the action version (same pattern as + # mutation-ledger) regardless of any RAINIX_SHA the caller pins. + nix run "path:$(cd "$GITHUB_ACTION_PATH/../../.." && pwd)#rainix-static" -- \ + codegen-declaration --log "$RUNNER_TEMP/rainix-codegen.log" diff --git a/.github/actions/codegen-witness/action.yml b/.github/actions/codegen-witness/action.yml deleted file mode 100644 index 9c0fb89..0000000 --- a/.github/actions/codegen-witness/action.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: codegen-witness -description: >- - Witnesses that the repo's codegen hooks still EMIT each committed generated file, which `rainix-copy-artifacts`' own currency check cannot. That check re-runs the generators and then `git diff --exit-code`, which proves the committed CONTENT is current but is structurally blind to a generator that has STOPPED emitting a file: the committed copy is already correct, so nothing is rewritten, nothing differs, and the job is green over a dead emitter (rainlanguage/rain.factory.deploy#35, reproduced there with a control — a marker appended to a generated file survived the generator run once the single call emitting that file was removed, with `git diff` clean throughout). The blind spot is structural rather than a bug in any repo: the generator's output is the check's only oracle for what the committed files should hold, so a file the generator never writes has no oracle at all, and seeing it needs an INDEPENDENT statement of which committed files are generated. That statement is `script/codegen-manifest.txt`. Two phases, and both must run in the same job. `mark` records every git-tracked file's mtime before the first codegen hook; `verify` re-stats them after the last one and calls a file WRITTEN when it exists and its mtime moved — `vm.writeFile` and its kin rewrite unconditionally, so a live emitter moves the mtime even when the bytes are identical, which is exactly the case a diff cannot tell from a dead one. Any path the manifest declares that nothing wrote fails the job by name. A path written but NOT declared is a printed note and never a failure: set equality would keep the manifest self-maintaining, but it would also couple a gate every Rain repo runs at `@main` to every incidental write inside the window (`forge build` is in there), so the day some tool starts touching a lock file it would redden the whole org at once. The claim worth gating on is the one the defect is about. Scope is git-tracked files, which keeps `out/`, `cache/`, `broadcast/` and `dependencies/` out of the witness whether or not a repo ignores them properly. A repo that runs no codegen hook and carries no manifest is a no-op; a repo with a hook and no manifest fails, and is printed the manifest its own run would justify. -inputs: - phase: - description: >- - `mark` before the first codegen hook runs, `verify` after the last one. `verify` must come BEFORE any formatter: `forge fmt` rewrites sources of its own, so running it first forges the witness — a file fmt happened to touch would look like one a generator wrote. - required: true -runs: - using: composite - steps: - - name: Codegen witness - shell: bash - env: - # Via env, never interpolated into the script: caller input must not reach a shell command. - RAINIX_CODEGEN_WITNESS_PHASE: ${{ inputs.phase }} - # Must not land in the working tree: it would show up in the git diff this check runs beside. - RAINIX_CODEGEN_WITNESS_STATE: ${{ runner.temp }}/rainix-codegen-witness.json - run: | - set -euo pipefail - nix run "path:$(cd "$GITHUB_ACTION_PATH/../../.." && pwd)#rainix-static" -- \ - codegen-witness "$RAINIX_CODEGEN_WITNESS_PHASE" \ - --state "$RAINIX_CODEGEN_WITNESS_STATE" diff --git a/.github/workflows/rainix-copy-artifacts.yaml b/.github/workflows/rainix-copy-artifacts.yaml index 3476843..195b387 100644 --- a/.github/workflows/rainix-copy-artifacts.yaml +++ b/.github/workflows/rainix-copy-artifacts.yaml @@ -21,47 +21,53 @@ jobs: - name: Install soldeer dependencies if: hashFiles('soldeer.lock') != '' run: nix develop github:rainlanguage/rainix/${{ env.RAINIX_SHA }}#sol-shell -c forge soldeer install - - name: Mark tracked files before codegen - uses: rainlanguage/rainix/.github/actions/codegen-witness@main - with: - phase: mark # Currency-check every committed generated artifact by re-running each # consumer-provided codegen step. The final git diff fails if any # committed file has drifted from its source. The build-meta.sh hook is # consumer-supplied because rain meta build's invocation (input/output # filenames, meta type) varies per repo. + # + # Each hook's stdout is teed into one log: a hook may declare there which + # generated paths it owns and which it wrote, which is the half of the + # currency check the diff cannot do (codegen-declaration, below). + # `pipefail` is set in every teed step — bash otherwise reports `tee`'s + # exit status, so a failing generator would pass. - name: Regenerate meta artifacts if: hashFiles('script/build-meta.sh') != '' - run: nix develop github:rainlanguage/rainix/${{ env.RAINIX_SHA }}#sol-shell -c ./script/build-meta.sh + run: | + set -euo pipefail + nix develop github:rainlanguage/rainix/${{ env.RAINIX_SHA }}#sol-shell -c ./script/build-meta.sh | tee -a "$RUNNER_TEMP/rainix-codegen.log" # Committed generated sources must be regenerable here, or the currency # check below passes without checking anything. The codegen script is # `script/Build.sol`, matched exactly: a repo that renames or drops it # goes red rather than skipping regeneration and reporting green. - name: Regenerate generated sources run: | + set -euo pipefail if [ -d src/generated ] && [ ! -f script/Build.sol ]; then echo "::error::src/generated/ is committed but script/Build.sol was not found, so the committed sources cannot be currency checked here. The codegen script must be script/Build.sol." exit 1 fi if [ -f script/Build.sol ]; then - nix develop github:rainlanguage/rainix/${{ env.RAINIX_SHA }}#sol-shell -c forge script ./script/Build.sol + nix develop github:rainlanguage/rainix/${{ env.RAINIX_SHA }}#sol-shell -c forge script ./script/Build.sol | tee -a "$RUNNER_TEMP/rainix-codegen.log" fi - name: Build Solidity run: nix develop github:rainlanguage/rainix/${{ env.RAINIX_SHA }}#sol-shell -c forge build - name: Copy forge artifacts into committed location if: hashFiles('script/CopyArtifacts.sol') != '' - run: nix develop github:rainlanguage/rainix/${{ env.RAINIX_SHA }}#sol-shell -c forge script ./script/CopyArtifacts.sol --ffi + run: | + set -euo pipefail + nix develop github:rainlanguage/rainix/${{ env.RAINIX_SHA }}#sol-shell -c forge script ./script/CopyArtifacts.sol --ffi | tee -a "$RUNNER_TEMP/rainix-codegen.log" # Catch-all post-forge regen hook: consumer-supplied. Runs outside any # nix devshell so the script picks shells per command (subgraph-shell, # sol-shell, etc.) for whatever derived artifacts it emits. - name: Regenerate derived artifacts if: hashFiles('script/build.sh') != '' - run: ./script/build.sh - # Before `forge fmt`: fmt rewrites sources of its own, which would forge the witness. - - name: Assert every declared generated file was written - uses: rainlanguage/rainix/.github/actions/codegen-witness@main - with: - phase: verify + run: | + set -euo pipefail + ./script/build.sh | tee -a "$RUNNER_TEMP/rainix-codegen.log" + - name: Assert every declared generated path was written + uses: rainlanguage/rainix/.github/actions/codegen-declaration@main - name: Format (so generated artifacts match committed style) run: nix develop github:rainlanguage/rainix/${{ env.RAINIX_SHA }}#sol-shell -c forge fmt - name: Assert committed artifacts match freshly built diff --git a/README.md b/README.md index 9f9e561..f2aca97 100644 --- a/README.md +++ b/README.md @@ -163,21 +163,23 @@ single job it runs whichever of these the repo has: then `forge fmt` and the `git diff` assert. -The diff is only half the check. Re-running the generators and diffing proves -the committed **content** is current, but it cannot see a generator that has -**stopped emitting a file**: the committed copy is already correct, so nothing -is rewritten, nothing differs, and the job is green over a dead emitter -(rainlanguage/rain.factory.deploy#35). So the job also witnesses which files the -hooks actually wrote, against a committed declaration: - -- `script/codegen-manifest.txt` — one repo-relative path per line (`#` comments - and blank lines ignored), naming every committed file the hooks generate. - Every path listed must be written on each run, or the job fails by name. A - file written but not listed is a printed note, never a failure. - -A repo that runs any of the hooks above **must** carry this file; a repo that -generates nothing needs neither. If it is missing, the job fails and prints the -manifest that run would justify, to be reviewed and committed. +The diff answers "is the committed **content** current". It cannot answer "is +anything still generating it": a generator that has stopped emitting a file +writes nothing, the already-correct committed copy is left alone, and the job is +green over a dead emitter (rainlanguage/rain.factory.deploy#35). Only the +generator knows which paths it owns, as against a `src/generated//` +snapshot deliberately frozen forever — so the generator says so, on stdout: + +``` +rainix-codegen owns src/lib/LibReleasedSuites.sol +rainix-codegen wrote src/lib/LibReleasedSuites.sol +``` + +The job tees every hook's stdout into one log and fails, naming the path, on +anything declared `owns` that no hook then `wrote`. Nothing is declared by hand +and no repo maintains a list: the same code that computes where to write emits +these lines. A repo whose hooks print neither keeps exactly today's behaviour — +green, with a note that a dead emitter there is still invisible. ```yaml name: copy-artifacts diff --git a/flake.nix b/flake.nix index 72c2be8..a7cf508 100644 --- a/flake.nix +++ b/flake.nix @@ -474,7 +474,7 @@ bats test/bats/action/prompt-cap.test.bats bats test/bats/action/frozen-snapshots-append-only.test.bats bats test/bats/action/mutation-ledger.test.bats - bats test/bats/action/codegen-witness.test.bats + bats test/bats/action/codegen-declaration.test.bats bats test/bats/task/skip-simulation.test.bats bats test/bats/task/subgraph-build.test.bats bats test/bats/task/subgraph-deploy-version.test.bats diff --git a/rainix-static/src/codegen_declaration.rs b/rainix-static/src/codegen_declaration.rs new file mode 100644 index 0000000..5500efa --- /dev/null +++ b/rainix-static/src/codegen_declaration.rs @@ -0,0 +1,200 @@ +use std::collections::BTreeSet; +use std::path::Path; + +/// Line prefix a codegen hook prints to declare a path. `forge script` indents +/// `console.log` output under `== Logs ==`, so lines are matched trimmed. +pub(crate) const SENTINEL: &str = "rainix-codegen "; + +#[derive(Default, PartialEq, Eq, Debug)] +pub(crate) struct Declaration { + pub(crate) owns: BTreeSet, + pub(crate) wrote: BTreeSet, + pub(crate) malformed: Vec, +} + +pub(crate) fn parse(log: &str) -> Declaration { + let mut out = Declaration::default(); + for line in log.lines() { + let Some(rest) = line.trim().strip_prefix(SENTINEL) else { + continue; + }; + match rest.split_once(char::is_whitespace) { + Some(("owns", path)) if !path.trim().is_empty() => { + out.owns.insert(path.trim().to_string()); + } + Some(("wrote", path)) if !path.trim().is_empty() => { + out.wrote.insert(path.trim().to_string()); + } + // Fail-closed: a verb this check does not know is a declaration it + // is silently not making, which is the defect one level up. + _ => out.malformed.push(line.trim().to_string()), + } + } + out +} + +pub(crate) fn offences( + owns: &BTreeSet, + wrote: &BTreeSet, + present: &BTreeSet, +) -> Vec { + let mut out = Vec::new(); + for path in owns { + match (wrote.contains(path), present.contains(path)) { + (false, true) => out.push(format!( + "the codegen hooks declare {path} generated, but nothing wrote it on this run. \ + The committed copy is left exactly as it was, so regenerating and diffing \ + passes without ever checking it — its emitter is dead. Restore the emitter, or \ + stop declaring the path if it is genuinely no longer generated." + )), + (false, false) => out.push(format!( + "the codegen hooks declare {path} generated, but nothing wrote it and no such \ + file exists after the run." + )), + (true, false) => out.push(format!( + "a codegen hook reported writing {path}, but no such file exists after the run." + )), + (true, true) => {} + } + } + out +} + +pub(crate) fn run(root: &Path, log: &Path) { + let text = match std::fs::read_to_string(log) { + Ok(text) => text, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(e) => crate::fail(&format!( + "codegen-declaration: failed to read {}: {e}", + log.display() + )), + }; + let declaration = parse(&text); + + if !declaration.malformed.is_empty() { + for line in &declaration.malformed { + eprintln!("::error::codegen-declaration: unreadable declaration: {line}"); + } + std::process::exit(1); + } + + if declaration.owns.is_empty() { + println!( + "codegen-declaration: clean — this repo's codegen declares no generated paths, so \ + nothing here can tell a live emitter from one that has died" + ); + return; + } + + let present: BTreeSet = declaration + .owns + .union(&declaration.wrote) + .filter(|path| root.join(path).exists()) + .cloned() + .collect(); + let offences = offences(&declaration.owns, &declaration.wrote, &present); + + if !offences.is_empty() { + for line in &offences { + eprintln!("::error::codegen-declaration: {line}"); + } + std::process::exit(1); + } + + println!( + "codegen-declaration: clean — {} declared generated paths, each written this run", + declaration.owns.len() + ); + for path in declaration.wrote.difference(&declaration.owns) { + println!("codegen-declaration: note — {path} was written but not declared, so nothing will notice if its emitter dies"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn set(paths: &[&str]) -> BTreeSet { + paths.iter().map(|p| p.to_string()).collect() + } + + #[test] + fn forge_indented_lines_parse_into_both_sets() { + let log = "== Logs ==\n rainix-codegen owns src/a.sol\n rainix-codegen wrote src/a.sol\n"; + let d = parse(log); + assert_eq!(d.owns, set(&["src/a.sol"])); + assert_eq!(d.wrote, set(&["src/a.sol"])); + assert!(d.malformed.is_empty()); + } + + #[test] + fn a_line_that_merely_mentions_the_sentinel_is_not_a_declaration() { + let d = parse("error: expected `rainix-codegen owns src/a.sol`\n"); + assert_eq!(d, Declaration::default()); + } + + #[test] + fn repeated_declarations_of_one_path_are_one_path() { + let d = parse("rainix-codegen wrote src/a.sol\nrainix-codegen wrote src/a.sol\n"); + assert_eq!(d.wrote, set(&["src/a.sol"])); + } + + #[test] + fn an_unknown_verb_is_malformed_rather_than_ignored() { + let d = parse("rainix-codegen skipped src/a.sol\n"); + assert_eq!(d.malformed, vec!["rainix-codegen skipped src/a.sol"]); + } + + #[test] + fn a_verb_with_no_path_is_malformed() { + let d = parse("rainix-codegen owns\nrainix-codegen wrote \n"); + assert_eq!(d.malformed.len(), 2); + assert!(d.owns.is_empty()); + assert!(d.wrote.is_empty()); + } + + #[test] + fn a_declared_path_nothing_wrote_is_an_offence_though_it_is_on_disk() { + let owns = set(&["src/lib/LibReleasedSuites.sol", "src/generated/A.sol"]); + let wrote = set(&["src/generated/A.sol"]); + let off = offences(&owns, &wrote, &owns); + assert_eq!(off.len(), 1); + assert!(off[0].contains("src/lib/LibReleasedSuites.sol")); + assert!(off[0].contains("emitter is dead")); + } + + #[test] + fn a_declared_path_that_is_not_on_disk_says_so_instead() { + let owns = set(&["src/generated/Gone.sol"]); + let off = offences(&owns, &BTreeSet::new(), &BTreeSet::new()); + assert_eq!(off.len(), 1); + assert!(off[0].contains("no such file exists")); + } + + #[test] + fn a_path_reported_written_that_is_not_on_disk_is_an_offence() { + let all = set(&["src/a.sol"]); + let off = offences(&all, &all, &BTreeSet::new()); + assert_eq!(off.len(), 1); + assert!(off[0].contains("reported writing")); + } + + #[test] + fn every_declared_path_written_is_clean() { + let all = set(&["src/a.sol", "src/b.sol"]); + assert!(offences(&all, &all, &all).is_empty()); + } + + #[test] + fn a_written_but_undeclared_path_is_not_an_offence() { + let owns = set(&["src/a.sol"]); + let wrote = set(&["src/a.sol", "soldeer.lock"]); + assert!(offences(&owns, &wrote, &wrote).is_empty()); + } + + #[test] + fn declaring_nothing_is_not_an_offence() { + let wrote = set(&["src/a.sol"]); + assert!(offences(&BTreeSet::new(), &wrote, &wrote).is_empty()); + } +} diff --git a/rainix-static/src/codegen_witness.rs b/rainix-static/src/codegen_witness.rs deleted file mode 100644 index 05efe1e..0000000 --- a/rainix-static/src/codegen_witness.rs +++ /dev/null @@ -1,301 +0,0 @@ -use std::collections::BTreeSet; -use std::path::Path; -use std::process::Command; -use std::time::UNIX_EPOCH; - -pub(crate) const MANIFEST_PATH: &str = "script/codegen-manifest.txt"; - -pub(crate) const HOOKS: [&str; 4] = [ - "script/build-meta.sh", - "script/Build.sol", - "script/CopyArtifacts.sol", - "script/build.sh", -]; - -const HEADER: &str = "\ -# Committed files this repo's codegen hooks generate — one path per line. -# -# rainix-copy-artifacts re-runs the hooks and diffs, which proves the CONTENT is -# current but cannot see a generator that has stopped emitting a file: nothing -# is rewritten, so nothing differs and the job is green over a dead emitter. -# Every path listed here must be written on each run, so losing an emitter is a -# red job rather than silence. -# -# Add a path when the repo starts generating it; remove one only when the file -# genuinely stops being generated (and is deleted or hand-maintained from then -# on). Blank lines and # comments are ignored. -"; - -pub(crate) fn parse_manifest(text: &str) -> BTreeSet { - text.lines() - .map(str::trim) - .filter(|line| !line.is_empty() && !line.starts_with('#')) - .map(str::to_string) - .collect() -} - -pub(crate) fn render_manifest(paths: &BTreeSet) -> String { - let mut out = String::from(HEADER); - for path in paths { - out.push_str(path); - out.push('\n'); - } - out -} - -pub(crate) fn offenders( - listed: &BTreeSet, - written: &BTreeSet, - present: &BTreeSet, - manifest: &str, -) -> Vec { - let mut out = Vec::new(); - for path in listed.difference(written) { - if present.contains(path) { - out.push(format!( - "ERROR: {manifest} declares {path} generated, but no codegen hook wrote it on \ - this run. The committed file is left exactly as it was, so re-running the \ - generators and diffing passes without checking it — its emitter is dead. \ - Restore the emitter, or, if {path} is genuinely no longer generated, say so by \ - removing the line (and the file, if nothing hand-maintains it)." - )); - } else { - out.push(format!( - "ERROR: {manifest} declares {path} generated, but no such file exists after \ - running the codegen hooks. Either its emitter is dead, or the path in the \ - manifest is wrong." - )); - } - } - out -} - -fn tracked(root: &Path) -> Result, String> { - let out = Command::new("git") - .arg("-C") - .arg(root) - // -z: git otherwise quotes awkward paths, which would not match the manifest. - .args(["ls-files", "-z"]) - .output() - .map_err(|e| format!("failed to run git ls-files: {e}"))?; - if !out.status.success() { - return Err(format!( - "git ls-files in {} failed: {}", - root.display(), - String::from_utf8_lossy(&out.stderr).trim() - )); - } - Ok(String::from_utf8_lossy(&out.stdout) - .split('\0') - .filter(|s| !s.is_empty()) - .map(str::to_string) - .collect()) -} - -fn mtime_nanos(root: &Path, path: &str) -> Option { - let meta = std::fs::metadata(root.join(path)).ok()?; - let since = meta.modified().ok()?.duration_since(UNIX_EPOCH).ok()?; - u64::try_from(since.as_nanos()).ok() -} - -pub(crate) fn mark(root: &Path, state: &Path) -> Result { - let files = tracked(root)?; - let mut map = serde_json::Map::new(); - for path in &files { - let value = match mtime_nanos(root, path) { - Some(nanos) => serde_json::Value::from(nanos), - None => serde_json::Value::Null, - }; - map.insert(path.clone(), value); - } - let doc = serde_json::json!({ "files": serde_json::Value::Object(map) }); - std::fs::write(state, doc.to_string()) - .map_err(|e| format!("failed to write {}: {e}", state.display()))?; - Ok(files.len()) -} - -pub(crate) fn written_since( - root: &Path, - state: &Path, -) -> Result<(BTreeSet, BTreeSet), String> { - let text = std::fs::read_to_string(state).map_err(|e| { - format!( - "failed to read the mark state {}: {e} — `codegen-witness mark` must run before the \ - codegen steps, in the same job", - state.display() - ) - })?; - let doc: serde_json::Value = serde_json::from_str(&text) - .map_err(|e| format!("{} is not valid JSON: {e}", state.display()))?; - let files = doc - .get("files") - .and_then(serde_json::Value::as_object) - .ok_or_else(|| format!("{} has no `files` object", state.display()))?; - - let mut written = BTreeSet::new(); - let mut present = BTreeSet::new(); - for (path, before) in files { - let after = mtime_nanos(root, path); - if after.is_some() { - present.insert(path.clone()); - if after != before.as_u64() { - written.insert(path.clone()); - } - } - } - Ok((written, present)) -} - -pub(crate) fn run_mark(root: &Path, state: &Path) { - match mark(root, state) { - Err(e) => crate::fail(&format!("codegen-witness mark: {e}")), - Ok(n) => println!( - "codegen-witness: marked {n} tracked files in {}", - root.display() - ), - } -} - -pub(crate) fn run_verify(root: &Path, state: &Path, manifest_rel: &str) { - let (written, present) = match written_since(root, state) { - Ok(sets) => sets, - Err(e) => crate::fail(&format!("codegen-witness verify: {e}")), - }; - - let manifest_path = root.join(manifest_rel); - let listed = match std::fs::read_to_string(&manifest_path) { - Ok(text) => Some(parse_manifest(&text)), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, - Err(e) => crate::fail(&format!( - "codegen-witness verify: failed to read {}: {e}", - manifest_path.display() - )), - }; - - let mut written: BTreeSet = written; - written.remove(manifest_rel); - - let Some(listed) = listed else { - let hooks: Vec<&str> = HOOKS - .iter() - .copied() - .filter(|h| root.join(h).exists()) - .collect(); - if hooks.is_empty() { - println!( - "codegen-witness: clean — no codegen hook and no {manifest_rel}; nothing declared \ - generated here" - ); - return; - } - eprintln!( - "::error::codegen-witness: this repo runs codegen ({}) but has no {manifest_rel}, so \ - the currency check cannot tell a generated file that is still emitted from one whose \ - emitter has died — the committed copy is correct either way. Commit a manifest \ - declaring the committed files the hooks generate. What they wrote on this run, as a \ - starting point:", - hooks.join(", ") - ); - eprintln!("{}", render_manifest(&written)); - std::process::exit(1); - }; - - let offenders = offenders(&listed, &written, &present, manifest_rel); - let unlisted: Vec<&String> = written.difference(&listed).collect(); - - if offenders.is_empty() { - println!( - "codegen-witness: clean — {} declared generated files, each written this run", - listed.len() - ); - if !unlisted.is_empty() { - println!( - "codegen-witness: note — written but not declared in {manifest_rel}, so nothing \ - will notice if their emitters die:" - ); - for path in unlisted { - println!(" {path}"); - } - } - return; - } - - for line in &offenders { - eprintln!("::error::{line}"); - } - eprintln!("What the codegen hooks actually wrote on this run:"); - eprintln!("{}", render_manifest(&written)); - std::process::exit(1); -} - -#[cfg(test)] -mod tests { - use super::*; - - fn set(paths: &[&str]) -> BTreeSet { - paths.iter().map(|p| p.to_string()).collect() - } - - #[test] - fn manifest_ignores_comments_and_blanks() { - let text = "# a comment\n\nsrc/a.sol\n src/b.sol \n#src/c.sol\n"; - assert_eq!(parse_manifest(text), set(&["src/a.sol", "src/b.sol"])); - } - - #[test] - fn rendered_manifest_round_trips_sorted() { - let paths = set(&["src/b.sol", "src/a.sol"]); - let rendered = render_manifest(&paths); - assert!(rendered.starts_with('#')); - let body: Vec<&str> = rendered - .lines() - .filter(|l| !l.starts_with('#') && !l.is_empty()) - .collect(); - assert_eq!(body, vec!["src/a.sol", "src/b.sol"]); - assert_eq!(parse_manifest(&rendered), paths); - } - - #[test] - fn empty_manifest_renders_to_header_only() { - let rendered = render_manifest(&BTreeSet::new()); - assert!(parse_manifest(&rendered).is_empty()); - } - - #[test] - fn declared_but_unwritten_is_an_offence() { - let listed = set(&["src/lib/LibReleasedSuites.sol", "src/generated/A.sol"]); - let written = set(&["src/generated/A.sol"]); - let present = set(&["src/lib/LibReleasedSuites.sol", "src/generated/A.sol"]); - let off = offenders(&listed, &written, &present, MANIFEST_PATH); - assert_eq!(off.len(), 1); - assert!(off[0].contains("src/lib/LibReleasedSuites.sol")); - assert!(off[0].contains("emitter is dead")); - } - - #[test] - fn declared_but_missing_from_disk_says_so() { - let listed = set(&["src/generated/Gone.sol"]); - let off = offenders(&listed, &BTreeSet::new(), &BTreeSet::new(), MANIFEST_PATH); - assert_eq!(off.len(), 1); - assert!(off[0].contains("no such file exists")); - } - - #[test] - fn every_declared_path_written_is_clean() { - let all = set(&["src/a.sol", "src/b.sol"]); - assert!(offenders(&all, &all, &all, MANIFEST_PATH).is_empty()); - } - - #[test] - fn a_written_but_undeclared_path_is_not_an_offence() { - let listed = set(&["src/a.sol"]); - let written = set(&["src/a.sol", "soldeer.lock"]); - assert!(offenders(&listed, &written, &written, MANIFEST_PATH).is_empty()); - } - - #[test] - fn an_empty_manifest_declares_nothing_and_is_clean() { - let written = set(&["src/a.sol"]); - assert!(offenders(&BTreeSet::new(), &written, &written, MANIFEST_PATH).is_empty()); - } -} diff --git a/rainix-static/src/main.rs b/rainix-static/src/main.rs index 3f540ea..6413118 100644 --- a/rainix-static/src/main.rs +++ b/rainix-static/src/main.rs @@ -34,11 +34,14 @@ // Nothing is stripped: a shell script reads the bytes on disk. Which // files are prompts and what they may weigh is per-repo, so both are an // input, and a glob matching nothing is an error rather than a pass. -// codegen-witness mark|verify --state [--root ] [--manifest ] -// fail if any path script/codegen-manifest.txt declares generated was not -// written between `mark` (before the first codegen hook) and `verify` -// (after the last), by mtime. A repo with a codegen hook and no manifest -// fails (and is printed one); a repo with neither passes. +// codegen-declaration --log [--root ] +// fail if a codegen hook declared a generated path it then did not +// write. Re-running the generators and diffing proves the committed +// CONTENT is current but is blind to a generator that has STOPPED +// emitting a file, and only the generator knows which paths it owns +// rather than deliberately leaves frozen. Hooks declare on stdout as +// `rainix-codegen owns ` and `rainix-codegen wrote `, which +// the workflow tees into . A repo declaring nothing passes. // snapshots-append-only [--base ] [--root ] // fail if the branch modifies or deletes an existing per-tag deploy-pin // snapshot under // (default root src/generated, base @@ -101,7 +104,7 @@ mod agent_context_cap; mod ci_gate; -mod codegen_witness; +mod codegen_declaration; mod context_bytes; mod frozen_snapshots; mod mutation_ledger; @@ -218,21 +221,11 @@ fn main() { .unwrap_or_else(|| fail("soldeer-gate: --package required")); soldeer_gate::run(&pkg, flag(&args, "--github-output").as_deref()); } - "codegen-witness" => { + "codegen-declaration" => { let root = flag(&args, "--root").unwrap_or_else(|| ".".to_string()); - let state = flag(&args, "--state") - .unwrap_or_else(|| fail("codegen-witness: --state required")); - match args.get(2).map(String::as_str).unwrap_or("") { - "mark" => codegen_witness::run_mark(Path::new(&root), Path::new(&state)), - "verify" => { - let manifest = flag(&args, "--manifest") - .unwrap_or_else(|| codegen_witness::MANIFEST_PATH.to_string()); - codegen_witness::run_verify(Path::new(&root), Path::new(&state), &manifest); - } - other => fail(&format!( - "codegen-witness: phase must be `mark` or `verify`, got {other:?}" - )), - } + let log = flag(&args, "--log") + .unwrap_or_else(|| fail("codegen-declaration: --log required")); + codegen_declaration::run(Path::new(&root), Path::new(&log)); } "snapshots-append-only" => { let base = flag(&args, "--base").unwrap_or_else(|| "origin/main".to_string()); @@ -294,8 +287,8 @@ fn main() { eprintln!( "rainix-static: unknown subcommand {other:?} \ (available: no-submodules, agent-context-cap, prompt-cap, \ - codegen-witness, snapshots-append-only, mutation-ledger, ci-gate, \ - soldeer-gate, rpc-preflight, release-guard)" + codegen-declaration, snapshots-append-only, mutation-ledger, \ + ci-gate, soldeer-gate, rpc-preflight, release-guard)" ); std::process::exit(2); } diff --git a/test/bats/action/codegen-declaration.test.bats b/test/bats/action/codegen-declaration.test.bats new file mode 100644 index 0000000..a1fb7be --- /dev/null +++ b/test/bats/action/codegen-declaration.test.bats @@ -0,0 +1,142 @@ +setup() { + repo_root="$BATS_TEST_DIRNAME/../../.." + action="$repo_root/.github/actions/codegen-declaration/action.yml" + action_script="$(yq -r '.runs.steps[0].run' "$action")" + work="$(mktemp -d)" + log="$work/codegen.log" +} + +teardown() { + rm -rf "$work" +} + +run_action() { + RUNNER_TEMP="$1" \ + GITHUB_ACTION_PATH="$repo_root/.github/actions/codegen-declaration" \ + ACTION_SCRIPT="$action_script" \ + bash -c ' + nix() { + printf "nix" + printf " <%s>" "$@" + printf "\n" + } + export -f nix + bash -c "$ACTION_SCRIPT" + ' +} + +@test "the action checks the log the workflow tees the hooks into" { + run run_action /tmp/runner-temp + + [ "$status" -eq 0 ] + [[ "$output" == *" <--log> " ]] +} + +# Staged, not committed: git ls-files and git diff read the index. +mk_consumer() { + git -C "$work" init -q + mkdir -p "$work/src/generated" "$work/src/lib" + printf 'GENERATED SNAPSHOT\n' >"$work/src/generated/Thing.sol" + printf 'GENERATED AGGREGATE\n' >"$work/src/lib/LibReleasedSuites.sol" + git -C "$work" add -A +} + +# The issue's control/mutant pair as a generator: ownership is computed from the +# repo's contract list, so removing the CALL that emits a path leaves the `owns` +# line and drops the write and the `wrote` line. Output is indented the way +# `forge script` indents console.log under `== Logs ==`. +# +# $1, when given, is the path whose emitter was removed. +generate() { + local dead="${1:-}" path + echo "== Logs ==" + for path in src/generated/Thing.sol src/lib/LibReleasedSuites.sol; do + printf ' rainix-codegen owns %s\n' "$path" + if [ "$path" != "$dead" ]; then + printf '%s\n' "$(cat "$work/$path")" >"$work/$path" + printf ' rainix-codegen wrote %s\n' "$path" + fi + done + echo "Script ran successfully." +} + +@test "a live generator rewriting identical bytes is clean, and so is git diff" { + mk_consumer + generate >"$log" + + run rainix-static codegen-declaration --root "$work" --log "$log" + [ "$status" -eq 0 ] + [[ "$output" == *"clean — 2 declared generated paths"* ]] + + run git -C "$work" diff --exit-code + [ "$status" -eq 0 ] +} + +@test "a generator that stopped emitting a file fails, though git diff is clean" { + mk_consumer + generate src/lib/LibReleasedSuites.sol >"$log" + + run rainix-static codegen-declaration --root "$work" --log "$log" + [ "$status" -eq 1 ] + [[ "$output" == *"src/lib/LibReleasedSuites.sol"* ]] + [[ "$output" == *"emitter is dead"* ]] + [[ "$output" != *"declare src/generated/Thing.sol generated, but nothing wrote"* ]] + + run git -C "$work" diff --exit-code + [ "$status" -eq 0 ] +} + +@test "a declared path that is not on disk at all is named as such" { + mk_consumer + rm "$work/src/lib/LibReleasedSuites.sol" + generate src/lib/LibReleasedSuites.sol >"$log" + + run rainix-static codegen-declaration --root "$work" --log "$log" + [ "$status" -eq 1 ] + [[ "$output" == *"no such file exists"* ]] +} + +@test "a hook that reports writing a path that never appeared fails" { + mk_consumer + printf 'rainix-codegen owns src/lib/Absent.sol\nrainix-codegen wrote src/lib/Absent.sol\n' >"$log" + + run rainix-static codegen-declaration --root "$work" --log "$log" + [ "$status" -eq 1 ] + [[ "$output" == *"reported writing src/lib/Absent.sol"* ]] +} + +@test "a repo whose hooks declare nothing passes, and is told it is unprotected" { + mk_consumer + printf 'Script ran successfully.\n' >"$log" + + run rainix-static codegen-declaration --root "$work" --log "$log" + [ "$status" -eq 0 ] + [[ "$output" == *"declares no generated paths"* ]] +} + +@test "a repo that ran no codegen hook at all passes" { + mk_consumer + + run rainix-static codegen-declaration --root "$work" --log "$work/never-written.log" + [ "$status" -eq 0 ] + [[ "$output" == *"declares no generated paths"* ]] +} + +@test "a path written but not declared is a note, not a failure" { + mk_consumer + printf 'rainix-codegen owns src/generated/Thing.sol\nrainix-codegen wrote src/generated/Thing.sol\nrainix-codegen wrote soldeer.lock\n' >"$log" + printf 'x\n' >"$work/soldeer.lock" + + run rainix-static codegen-declaration --root "$work" --log "$log" + [ "$status" -eq 0 ] + [[ "$output" == *"note — soldeer.lock was written but not declared"* ]] +} + +@test "a verb this check does not know fails rather than being ignored" { + mk_consumer + printf 'rainix-codegen skipped src/generated/Thing.sol\n' >"$log" + + run rainix-static codegen-declaration --root "$work" --log "$log" + [ "$status" -eq 1 ] + [[ "$output" == *"unreadable declaration"* ]] +} diff --git a/test/bats/action/codegen-witness.test.bats b/test/bats/action/codegen-witness.test.bats deleted file mode 100644 index 302ddad..0000000 --- a/test/bats/action/codegen-witness.test.bats +++ /dev/null @@ -1,219 +0,0 @@ -setup() { - repo_root="$BATS_TEST_DIRNAME/../../.." - action="$repo_root/.github/actions/codegen-witness/action.yml" - action_script="$(yq -r '.runs.steps[0].run' "$action")" - work="$(mktemp -d)" - state="$work/witness.json" -} - -teardown() { - rm -rf "$work" -} - -run_witness_action() { - RAINIX_CODEGEN_WITNESS_PHASE="$1" \ - RAINIX_CODEGEN_WITNESS_STATE="$2" \ - GITHUB_ACTION_PATH="$repo_root/.github/actions/codegen-witness" \ - ACTION_SCRIPT="$action_script" \ - bash -c ' - nix() { - printf "nix" - printf " <%s>" "$@" - printf "\n" - } - export -f nix - bash -c "$ACTION_SCRIPT" - ' -} - -@test "the phase reaches the binary as the subcommand's phase argument" { - run run_witness_action mark /tmp/w.json - - [ "$status" -eq 0 ] - [[ "$output" == *" <--state> " ]] -} - -@test "both phases are wired through the same action, not two spellings" { - run run_witness_action verify /tmp/w.json - - [ "$status" -eq 0 ] - [[ "$output" == *" <--state> " ]] -} - -@test "the state path is a runner temp file, never a path in the working tree" { - local state_expr - state_expr="$(yq -r '.runs.steps[0].env.RAINIX_CODEGEN_WITNESS_STATE' "$action")" - # `${{ runner.temp }}` is a GitHub Actions expression: single-quoted so the shell cannot expand it. - # shellcheck disable=SC2016 - [[ "$state_expr" == '${{ runner.temp }}/'* ]] -} - -# Staged, not committed: git ls-files reads the index. -mk_consumer() { - git -C "$work" init -q - mkdir -p "$work/script" "$work/src/generated" "$work/src/lib" - printf 'contract Build {}\n' >"$work/script/Build.sol" - printf 'GENERATED SNAPSHOT\n' >"$work/src/generated/Thing.sol" - printf 'GENERATED AGGREGATE\n' >"$work/src/lib/LibReleasedSuites.sol" - cat >"$work/script/codegen-manifest.txt" <<'EOF' -# declared generated files -src/generated/Thing.sol -src/lib/LibReleasedSuites.sol -EOF - git -C "$work" add -A -} - -# Filesystem mtime resolution: a write inside the window must be unambiguously later. -age_tree() { - find "$work" -path "$work/.git" -prune -o -type f -exec touch -m -t 202001010000 {} + -} - -regenerate() { - local f - for f in "$@"; do - local content - content="$(cat "$work/$f")" - printf '%s\n' "$content" >"$work/$f" - done -} - -@test "a live generator's identical rewrite is witnessed, with git diff clean" { - mk_consumer - age_tree - - run rainix-static codegen-witness mark --root "$work" --state "$state" - [ "$status" -eq 0 ] - - regenerate src/generated/Thing.sol src/lib/LibReleasedSuites.sol - - run rainix-static codegen-witness verify --root "$work" --state "$state" - [ "$status" -eq 0 ] - [[ "$output" == *"clean — 2 declared generated files"* ]] - - run git -C "$work" diff --exit-code - [ "$status" -eq 0 ] -} - -@test "a generator that stopped emitting a file fails, though git diff is clean" { - mk_consumer - age_tree - - run rainix-static codegen-witness mark --root "$work" --state "$state" - [ "$status" -eq 0 ] - - regenerate src/generated/Thing.sol - - run rainix-static codegen-witness verify --root "$work" --state "$state" - [ "$status" -eq 1 ] - [[ "$output" == *"src/lib/LibReleasedSuites.sol"* ]] - [[ "$output" == *"emitter is dead"* ]] - [[ "$output" != *"declares src/generated/Thing.sol generated, but no codegen hook"* ]] - - run git -C "$work" diff --exit-code - [ "$status" -eq 0 ] -} - -@test "a declared file that no longer exists at all is named as such" { - mk_consumer - rm "$work/src/lib/LibReleasedSuites.sol" - age_tree - - rainix-static codegen-witness mark --root "$work" --state "$state" - regenerate src/generated/Thing.sol - - run rainix-static codegen-witness verify --root "$work" --state "$state" - [ "$status" -eq 1 ] - [[ "$output" == *"src/lib/LibReleasedSuites.sol"* ]] - [[ "$output" == *"no such file exists"* ]] -} - -@test "a repo with no codegen hook and no manifest passes" { - git -C "$work" init -q - mkdir -p "$work/src" - printf 'contract A {}\n' >"$work/src/A.sol" - git -C "$work" add -A - age_tree - - rainix-static codegen-witness mark --root "$work" --state "$state" - - run rainix-static codegen-witness verify --root "$work" --state "$state" - [ "$status" -eq 0 ] - [[ "$output" == *"no codegen hook and no"* ]] -} - -@test "a repo that runs codegen but declares nothing fails, and is printed one" { - mk_consumer - rm "$work/script/codegen-manifest.txt" - git -C "$work" rm -q --cached script/codegen-manifest.txt - age_tree - - rainix-static codegen-witness mark --root "$work" --state "$state" - regenerate src/generated/Thing.sol src/lib/LibReleasedSuites.sol - - run rainix-static codegen-witness verify --root "$work" --state "$state" - [ "$status" -eq 1 ] - [[ "$output" == *"script/Build.sol"* ]] - [[ "$output" == *"src/generated/Thing.sol"* ]] - [[ "$output" == *"src/lib/LibReleasedSuites.sol"* ]] -} - -@test "a written but undeclared file is a note, not a failure" { - mk_consumer - printf 'lock\n' >"$work/soldeer.lock" - git -C "$work" add soldeer.lock - age_tree - - rainix-static codegen-witness mark --root "$work" --state "$state" - regenerate src/generated/Thing.sol src/lib/LibReleasedSuites.sol soldeer.lock - - run rainix-static codegen-witness verify --root "$work" --state "$state" - [ "$status" -eq 0 ] - [[ "$output" == *"note — written but not declared"* ]] - [[ "$output" == *"soldeer.lock"* ]] -} - -@test "a file the hooks deleted is not counted as written" { - mk_consumer - age_tree - - rainix-static codegen-witness mark --root "$work" --state "$state" - regenerate src/generated/Thing.sol - rm "$work/src/lib/LibReleasedSuites.sol" - - run rainix-static codegen-witness verify --root "$work" --state "$state" - [ "$status" -eq 1 ] - [[ "$output" == *"no such file exists"* ]] -} - -@test "verify without a prior mark fails naming mark, rather than passing" { - mk_consumer - - run rainix-static codegen-witness verify --root "$work" --state "$state" - [ "$status" -eq 1 ] - [[ "$output" == *"codegen-witness mark"* ]] -} - -@test "an unknown phase fails rather than defaulting to one" { - run rainix-static codegen-witness --root "$work" --state "$state" - [ "$status" -eq 1 ] - [[ "$output" == *"mark"* ]] - [[ "$output" == *"verify"* ]] -} - -@test "a missing --state fails rather than inventing a path" { - run rainix-static codegen-witness mark --root "$work" - [ "$status" -eq 1 ] - [[ "$output" == *"--state"* ]] -} - -@test "the manifest path defaults to script/codegen-manifest.txt" { - mk_consumer - age_tree - - rainix-static codegen-witness mark --root "$work" --state "$state" - regenerate src/generated/Thing.sol - - run rainix-static codegen-witness verify --root "$work" --state "$state" - [ "$status" -eq 1 ] - [[ "$output" == *"script/codegen-manifest.txt declares"* ]] -} diff --git a/test/bats/workflow/rainix-copy-artifacts.test.bats b/test/bats/workflow/rainix-copy-artifacts.test.bats index 37fc5be..1e8ef64 100644 --- a/test/bats/workflow/rainix-copy-artifacts.test.bats +++ b/test/bats/workflow/rainix-copy-artifacts.test.bats @@ -1,98 +1,72 @@ +# Nothing in this repo executes rainix-copy-artifacts.yaml — it is +# `workflow_call` only, so its only runners are the consumer repos. +# +# The declaration check's verdict is covered by the Rust unit tests and +# test/bats/action/codegen-declaration.test.bats; what is asserted here is the +# wiring it cannot see: that the hooks' stdout actually reaches the log the +# action reads, and that teeing did not swallow a hook's exit status. + setup() { repo_root="$BATS_TEST_DIRNAME/../../.." workflow="$repo_root/.github/workflows/rainix-copy-artifacts.yaml" - witness="$repo_root/rainix-static/src/codegen_witness.rs" - names="$(yq -r '.jobs["copy-artifacts"].steps[] | .name // "«unnamed»"' "$workflow")" - runs="$(yq -r '.jobs["copy-artifacts"].steps[] | select(.run) | .run' "$workflow")" - mark_step="Mark tracked files before codegen" - verify_step="Assert every declared generated file was written" + action="$repo_root/.github/actions/codegen-declaration/action.yml" + runs="$(yq -r '.jobs.copy-artifacts.steps[] | select(.run) | .run' "$workflow")" + uses="$(yq -r '.jobs.copy-artifacts.steps[] | select(.uses) | .uses' "$workflow")" + sha="$(yq -r '.env.RAINIX_SHA' "$workflow")" + # The one path the two files have to agree on. + log_path="$(yq -r '.runs.steps[0].run' "$action" | grep -o '\--log "[^"]*"' | sed 's/--log "//; s/"$//')" + teed="$(yq -r '.jobs.copy-artifacts.steps[] | select(.run) | select(.run | contains("tee")) | .run' "$workflow")" } -step_at() { - echo "$names" | grep -nxF "$1" | cut -d: -f1 +@test "the codegen declaration is checked, at the ref the check ships from" { + echo "$uses" | grep -q '^rainlanguage/rainix/.github/actions/codegen-declaration@main$' } -step_field() { - yq -r ".jobs[\"copy-artifacts\"].steps[] | select(.name == \"$1\") | $2" "$workflow" +@test "the action reads a log under the runner temp dir, never the working tree" { + [ -n "$log_path" ] + # The literal the action script carries, not an expansion of it. + # shellcheck disable=SC2016 + [[ "$log_path" == '$RUNNER_TEMP/'* ]] } -@test "the job invokes both codegen-witness phases" { - [ "$(step_field "$mark_step" .uses)" = "rainlanguage/rainix/.github/actions/codegen-witness@main" ] - [ "$(step_field "$mark_step" .with.phase)" = "mark" ] - [ "$(step_field "$verify_step" .uses)" = "rainlanguage/rainix/.github/actions/codegen-witness@main" ] - [ "$(step_field "$verify_step" .with.phase)" = "verify" ] +@test "every codegen hook tees into the log the action reads" { + local teed_steps occurrences + teed_steps="$(yq -r '[.jobs.copy-artifacts.steps[] | select(.run) | select(.run | contains("tee -a"))] | length' "$workflow")" + [ "$teed_steps" -eq 4 ] + occurrences="$(grep -cF "tee -a \"$log_path\"" "$workflow")" + [ "$occurrences" -eq 4 ] } -@test "both witness phases bracket every codegen hook step" { - local mark verify - mark="$(step_at "$mark_step")" - verify="$(step_at "$verify_step")" - [ -n "$mark" ] - [ -n "$verify" ] - [ "$mark" -lt "$verify" ] - - local hooked - hooked="$(yq -r '.jobs["copy-artifacts"].steps | to_entries[] - | select((.value.run // "") | test("\./script/(build-meta\.sh|Build\.sol|CopyArtifacts\.sol|build\.sh)")) - | (.key + 1 | tostring) + " " + (.value.name // "«unnamed»")' "$workflow")" - [ -n "$hooked" ] - - local pos name - while read -r pos name; do - if [ "$pos" -lt "$mark" ] || [ "$pos" -gt "$verify" ]; then - echo "FAIL: codegen step '$name' (position $pos) is outside the witness window ($mark..$verify)" >&2 - return 1 - fi - done <<<"$hooked" -} - -@test "the witness closes before forge fmt runs" { - local verify fmt - verify="$(step_at "$verify_step")" - fmt="$(echo "$names" | grep -n 'Format' | cut -d: -f1)" - [ -n "$fmt" ] - [ "$verify" -lt "$fmt" ] -} - -@test "the committed-artifacts diff is still asserted after the witness" { - local verify diff - verify="$(step_at "$verify_step")" - diff="$(step_at "Assert committed artifacts match freshly built")" - [ -n "$diff" ] - [ "$verify" -lt "$diff" ] - echo "$runs" | grep -q 'git diff --exit-code' +@test "every codegen hook reaches the log — none is left undeclarable" { + local hook + for hook in script/build-meta.sh script/Build.sol script/CopyArtifacts.sol script/build.sh; do + echo "$teed" | grep -qF "$hook" + done } -@test "neither witness step is conditional" { - local step guard - for step in "$mark_step" "$verify_step"; do - guard="$(step_field "$step" '.["if"] // "none"')" - if [ "$guard" != "none" ]; then - echo "FAIL: '$step' is guarded by: $guard" >&2 - return 1 - fi - done +# Without pipefail bash reports tee's status, so a failing generator would pass +# the step it just failed. +@test "every teed step sets pipefail" { + local n_teed n_pipefail + n_teed="$(yq -r '[.jobs.copy-artifacts.steps[] | select(.run) | select(.run | contains("tee -a"))] | length' "$workflow")" + n_pipefail="$(yq -r '[.jobs.copy-artifacts.steps[] | select(.run) | select(.run | contains("tee -a")) | select(.run | contains("set -euo pipefail"))] | length' "$workflow")" + [ "$n_teed" -gt 0 ] + [ "$n_pipefail" -eq "$n_teed" ] } -@test "the binary's hook list is exactly the hooks the workflow runs" { - local declared invoked - declared="$(sed -n '/pub(crate) const HOOKS/,/^];/p' "$witness" | - grep -o '"script/[^"]*"' | tr -d '"' | sort)" - invoked="$(echo "$runs" | grep -oE '\./script/[A-Za-z0-9_-]+\.(sol|sh)' | - sed 's|^\./||' | sort -u)" - [ -n "$declared" ] - if [ "$declared" != "$invoked" ]; then - echo "FAIL: codegen_witness.rs HOOKS and the hooks the workflow invokes disagree" >&2 - diff <(echo "$declared") <(echo "$invoked") >&2 || true - return 1 - fi +@test "the declaration is checked after the last codegen hook and before the diff" { + local last_hook check diff + last_hook="$(yq -r '[.jobs.copy-artifacts.steps | to_entries[] | select(.value.run) | select(.value.run | contains("tee -a")) | .key] | max' "$workflow")" + check="$(yq -r '[.jobs.copy-artifacts.steps | to_entries[] | select(.value.uses) | select(.value.uses | contains("codegen-declaration")) | .key] | .[0]' "$workflow")" + diff="$(yq -r '[.jobs.copy-artifacts.steps | to_entries[] | select(.value.run) | select(.value.run | contains("git diff --exit-code")) | .key] | .[0]' "$workflow")" + [ "$last_hook" -lt "$check" ] + [ "$check" -lt "$diff" ] } @test "every rainix-copy-artifacts run step resolves rainix through the pinned sha" { - local sha unpinned - sha="$(yq -r '.env.RAINIX_SHA' "$workflow")" [ -n "$sha" ] [ "$sha" != "null" ] + local unpinned unpinned="$(echo "$runs" | grep 'github:rainlanguage/rainix' | grep -v 'env.RAINIX_SHA' || true)" if [ -n "$unpinned" ]; then echo "FAIL: unpinned rainix refs in rainix-copy-artifacts.yaml:" >&2