diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74e510fb..908d30a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -158,6 +158,25 @@ jobs: VARVE_CORROSION_TEST: "1" run: cargo test -p varve-core --test corrosion_offline -- --nocapture + keydisk: + name: no signing key on disk (REQ-NOKEYDISK-001) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + # `docs ci` names `echo "$SECRET" > key.tmp` as the thing adopters + # wrongly invent, and `docs root-ceremony` says the key must reach varve + # through a file descriptor. This repository's own deposit workflow wrote + # it to a predictable /tmp path on a shared runner for every layer it + # published, and an assessor found that by reading the repo rather than + # the docs. A rule the publisher does not follow is not evidence, so the + # rule is mechanical from here. + # Prove the gate can go red BEFORE trusting it green -- the controls + # include the exact line this repository shipped. + - name: Negative control — the gate must reject what it exists to reject + run: tools/no-key-on-disk.sh --self-test + - name: Refuse a workflow that writes key material to a file + run: tools/no-key-on-disk.sh + mutants: name: cargo mutants (trust-critical gate) runs-on: ubuntu-latest @@ -217,8 +236,12 @@ jobs: -f crates/varve-core/src/layerspec.rs \ -f crates/varve-producer/src/asset.rs \ -f crates/varve-producer/src/attestation.rs \ + -f crates/varve-producer/src/binfmt.rs \ + -f crates/varve-producer/src/carryforward.rs \ + -f crates/varve-producer/src/extract.rs \ -f crates/varve-producer/src/forge.rs \ -f crates/varve-producer/src/ingest.rs \ + -f crates/varve-producer/src/plan.rs \ -f crates/varve-producer/src/spec.rs \ -f crates/varve-producer/src/sums.rs diff --git a/.github/workflows/deposit-layer.yml b/.github/workflows/deposit-layer.yml index 5dd123b6..026678f4 100644 --- a/.github/workflows/deposit-layer.yml +++ b/.github/workflows/deposit-layer.yml @@ -77,10 +77,17 @@ jobs: VARVE_ROLLING_KEY: ${{ secrets.VARVE_ROLLING_KEY }} LAYER: ${{ inputs.layer }} COUNTER: ${{ inputs.counter }} + shell: bash run: | set -euo pipefail test -n "$VARVE_ROLLING_KEY" || { echo "::error::VARVE_ROLLING_KEY not provisioned"; exit 1; } - printf '%s' "$VARVE_ROLLING_KEY" > /tmp/rolling.key + # The key reaches varve through a FILE DESCRIPTOR, never a file + # (REQ-NOKEYDISK-001). `docs ci` documents this form and names + # `echo "$SECRET" > key.tmp` as the thing adopters wrongly invent -- + # and this workflow did exactly that, to a predictable path, on a + # shared runner, for every deposit. An assessor found it by reading + # the repository rather than the documentation, which is the way + # this class of finding is always found. ISSUED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" # The spec and the payload bytes it points at live together under # deposit-stage/ — a payload `path` resolves relative to the SPEC @@ -88,7 +95,7 @@ jobs: ./target/release/varve deposit \ --spec deposit-stage/deposit-spec.toml \ --issued-at "$ISSUED_AT" \ - --key /tmp/rolling.key --key-id varve-rolling-1 \ + --key <(printf '%s' "$VARVE_ROLLING_KEY") --key-id varve-rolling-1 \ --out layer-layout # Sign and attach a BASELINE line-status (REQ-STATUS-DIST-001) so # `varve status` works after an offline OR registry install with no @@ -100,11 +107,10 @@ jobs: "$LINE" "$COUNTER" "$ISSUED_AT" > baseline-status.json ./target/release/varve sign-status \ --file baseline-status.json \ - --key /tmp/rolling.key --key-id varve-rolling-1 \ + --key <(printf '%s' "$VARVE_ROLLING_KEY") --key-id varve-rolling-1 \ --out baseline-status.dsse.json ./target/release/varve attach-status \ --layout layer-layout --status baseline-status.dsse.json - rm -f /tmp/rolling.key # Sanity: the deposit installs and verifies on THIS runner before # anything is published. mkdir -p sanity-project diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 69e11b01..5124913e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -327,6 +327,9 @@ jobs: env: VARVE_ROOT_KEY: ${{ secrets.VARVE_ROOT_KEY }} VARVE_ROLLING_KEY: ${{ secrets.VARVE_ROLLING_KEY }} + # Explicit, because the key reaches varve by process substitution and + # that is a bash feature, not a POSIX one. + shell: bash run: | set -euo pipefail if [ -n "${VARVE_ROOT_KEY:-}" ]; then @@ -344,16 +347,16 @@ jobs: exit 0 fi cargo build --release -p varve - printf '%s' "$SIGNING_KEY" > /tmp/varve-root.key + # File descriptor, never a file (REQ-NOKEYDISK-001) -- see the same + # correction in deposit-layer.yml. cd release-assets # The signing code is the same sign_release_sums the verifier # tests against — producer and consumer cannot drift. ../target/release/varve sign-sums \ --sums SHA256SUMS.txt \ - --key /tmp/varve-root.key \ + --key <(printf '%s' "$SIGNING_KEY") \ --key-id "$KEY_ID" \ --out SHA256SUMS.txt.dsse.json - rm -f /tmp/varve-root.key ls -la SHA256SUMS.txt.dsse.json - name: Capture build environment diff --git a/.github/workflows/systest.yml b/.github/workflows/systest.yml index f7e97ed5..4d32b8d4 100644 --- a/.github/workflows/systest.yml +++ b/.github/workflows/systest.yml @@ -153,3 +153,16 @@ jobs: # It ends by rebuilding varve with the pin's choice deleted from the # one place it is consulted, and requires the run to go red. run: tools/systest/compose-realms.sh "$RUNNER_TEMP/compose-realms" + + install-shadow: + name: the installer names which varve will run (REQ-INSTALLSHADOW-001) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + # A maintainer followed the documented install and ended up running a + # DIFFERENT binary than the one it installed, because being on PATH is + # not being first on it. varve refuses to claim success when PATH + # shadows a pinned tool; this holds its own installer to that standard. + # The gate carries its own negative control. + - name: Installer must warn when another varve wins PATH + run: tools/systest/install-shadow.sh diff --git a/artifacts/requirements.yaml b/artifacts/requirements.yaml index 31a16bd0..0c29e7aa 100644 --- a/artifacts/requirements.yaml +++ b/artifacts/requirements.yaml @@ -3537,7 +3537,7 @@ artifacts: - id: REQ-PRODUCER-002 type: requirement title: The producer pipeline is a tested Rust program, not a shell script shipped as a tarball - status: draft + status: implemented release: v0.30.0 description: > The pipeline that assembles, signs and publishes a layer is ~3.5k lines @@ -3596,6 +3596,282 @@ artifacts: rather than assumed — a release publishing a cosign-signed SHA256SUMS.txt uses rung 1 and does not depend on it. + - id: REQ-NAMETHEREALM-001 + type: requirement + title: Output that identifies a layer names the realm, because a layer id alone does not + status: verified + release: v0.30.0 + description: > + A layer identifier is `YYYY.MM.P` and is unique only WITHIN a realm. Two + realms can each publish `2026.08.26`, and varve is built for exactly that + world — REQ-REALM2-001 stood up a second realm and a pin qualifier to + settle name collisions between them. + . + `varve inspect` never printed the realm. Not in the header, not in + `--json`, which had no `realm` key at all. It appeared only inside the + `composition` block, which is skipped when a layer composes nothing — the + ordinary case. A user read that output beside a `realm = "linc"` pin and + observed, correctly, that the word realm appeared nowhere in it. + . + The refusal path had the same gap from the other side: a shim declining a + tool named the layer and the tools it exposes, but not which `varve.toml` + chose that layer or which realm it named. With one shim directory serving + every realm and `cd` switching toolchains, "why did this tool stop + working HERE" is the question, and the answer is a file the user can + open. + . + Clauses: (1) `varve inspect` shall name the realm in its header and carry + a top-level `realm` in `--json`. (2) A dispatch refusal shall name the + pin file and the realm it selects, not merely the layer. (3) Any output + that identifies a layer to a human shall be unambiguous about which realm + it belongs to, since the identifier alone is not. + + - id: REQ-INSTALLSHADOW-001 + type: requirement + title: The installer says what it replaced, and which varve will actually run + status: implemented + release: v0.30.0 + description: > + `install.sh` writes the binary with `mv` over whatever was at + `$INSTALL_DIR/varve` and reports "Installed varve X". It never says a + previous version was there, and never checks whether a DIFFERENT varve + earlier on PATH will win the lookup. Its PATH message makes the second + worse: it prints "$INSTALL_DIR is already on PATH", which is true and + misleading — on PATH is not the same as first on PATH. + . + Reproduced on a maintainer's machine: `~/.varve/bin/varve` at 0.25.0 from + the installer, `~/.cargo/bin/varve` at 0.29.0 from cargo, and plain + `varve` resolving to the cargo one. A user follows the documented install, + is told it succeeded, and then runs a binary the installer did not put + there. They have no way to know from the output. + . + This is REQ-SHADOW-001's hazard turned on varve itself. varve REFUSES to + claim success when a pinned tool is shadowed by PATH — "your shell will + run the first one; varve dispatches the second" — and its own installer + does not apply that reasoning to the binary it just installed. A + bootstrap that can leave the user running a different build than the one + it verified is a bootstrap that has not bootstrapped anything. + . + Clauses: (1) When a binary already exists at the destination, the + installer shall report the version it replaced alongside the version it + installed; an unchanged reinstall shall say so rather than implying + change. (2) After installing, the installer shall resolve `varve` the way + the user's shell will and, if the winner is not the binary it just + installed, shall say so prominently — naming the winning path, its + version, and how to make the intended one win. (3) The check shall use + the same resolution order the shell uses, not an assumption about where + things live. (4) It shall not silently succeed: the difference between + "installed" and "installed and this is what you will run" is the whole + value of the message. (5) A negative control shall prove the warning + fires — an installer that only ever prints the happy path has not been + tested, and this defect survived because nothing exercised the shadowed + case. + + - id: REQ-REALM2-002 + type: requirement + title: The bytecodealliance realm gets its own root, and is the first real ceremony + status: draft + release: v1.0.0 + description: > + The second realm exists only inside `tools/systest/compose-realms.sh`. + The shipped `varve-realms.toml` defines exactly one realm and all + published tags are pulseengine's, so "varve composes two realms" is true + in the gate and untrue in the registry. + . + Carrying `wasm-tools` inside the pulseengine realm was proposed and + REJECTED. It is buildable today — all four platforms resolve through the + %U upstream tag, `gh attestation verify` succeeds on the real release and + fails on tampered bytes — and it needs no new key. It was rejected + because it defeats what a realm is. A realm is a trust universe defined + by one root; letting that root sign bytes the organisation did not build + makes the boundary decorative. Recording an ingestion proof documents + whose bytes they are, it does not change who vouched for them. The + shortcut's whole appeal was avoiding the key problem, which is an + argument for solving the key problem. + . + Clauses: (1) `bytecodealliance` shall be a realm with its OWN root, its + own entry in `varve-realms.toml`, and its own layer line. (2) That root + shall be produced by the ceremony REQ-CEREMONY-001 defines — not + generated into CI, which is how the pulseengine root came to be + unbackupable and unmovable (#110). (3) A rehearsal with a genuinely + throwaway key shall precede it, and the rehearsal's transcript is the + runbook the real ceremony is executed from; the two shall not happen on + the same day, because a real ceremony run at hour eight of a rehearsal is + where custody defects are born. (4) The pulseengine realm shall not carry + payloads from outside the organisation in the meantime; a name collision + between the two realms is settled by the pin's realm qualifier + (REQ-REALM2-001 clause 4c), which is what that mechanism was built for. + . + Why this realm first: it is the only realm whose ceremony can go wrong + cheaply. It has no consumers, no published root anyone pins, and nothing + frozen if it must be redone — the exact opposite of the pulseengine + realm, where a mistake is unrecoverable because varve has neither + rotation nor revocation. An assessor's first recommendation was to stop + writing about the ceremony and run one; this gives that run a real + deliverable instead of a discarded key, and gives the ceremony a deadline + that "at v1.0" never supplied. + + - id: REQ-CARRYFORWARD-001 + type: requirement + title: A deposit does the work that changed, and re-proves the work that did not + status: implemented + release: v0.30.0 + description: > + Every deposit downloads all four platforms of every tool the layer + carries — 421 MiB for layer 2026.08.4 — re-hashes them and re-pushes + them, even when a single tool moved. Consumers are unaffected: install + filters by platform before fetching, so a machine pulls 99 MiB. The cost + is the producer's, and daily scanning (DD-024) multiplies it by the + cadence. + . + The previous layer's SIGNED manifest already records each payload's + upstream repo, release, asset name and sha256. When `layer.toml` still + pins the same release, that work is already done and proven. + . + The trap is that reusing it naively would REGRESS a security property. + A release asset can be deleted and re-uploaded under the same tag, so + "rivet v0.34.0" today is not necessarily the bytes it was yesterday. + Carrying a digest forward on the strength of a version string would make + varve blind to exactly the substitution it exists to catch. + . + So the saving must come from skipping the DOWNLOAD, never from skipping + the PROOF. Re-verifying a release costs a sums file — kilobytes — while + the binary it describes is tens of megabytes. + . + Clauses: (1) The ingestion proof shall be re-established every deposit, + unconditionally; carry-forward shall never skip it. (2) The digest + recorded upstream NOW shall be compared against the digest the previous + layer recorded; only when they agree may the payload bytes go + un-downloaded. (3) A disagreement shall ABORT and name both digests: an + upstream that re-published a release under the same version is a fact an + operator must see, and it is not varve's to resolve silently in either + direction. (4) Reuse shall additionally require that the blob is still + present in the destination registry, since a manifest entry is not a + guarantee of storage. (5) A payload with no previous entry — a new tool, + a new platform, a changed version — shall be fetched normally. (6) The + gate shall be that depositing an unchanged `layer.toml` twice fetches no + payload bytes the second time. + + - id: REQ-PAYLOADSMOKE-001 + type: requirement + title: A deposited payload is checked for being usable, not merely for being the bytes upstream published + status: implemented + release: v0.30.0 + description: > + Everything the producer verifies today answers one question: are these + the bytes upstream published? A cosign-signed sums file or a build + attestation establishes that, and the digest recorded in the layer + transcribes it. Nothing establishes that the bytes are a WORKING TOOL for + the platform they are deposited under. + . + The deposit's own "sanity" step installs the layer and runs `varve + verify`, which checks signatures and digests — the same property again, + more carefully. No deposited binary is ever executed, and no systest runs + one. A layer can therefore be assembled, signed, published and installed + while carrying an x86_64 binary under `aarch64-unknown-linux-gnu`, a + shell script where a binary was expected, or a file upstream truncated + before publishing. Each of those is faithfully hashed and correctly + signed, and fails for the first time on a consumer's machine. + . + An architecture mismatch is the sharp case because it is a real upstream + packaging mistake, it is invisible to every check varve currently runs, + and the consumer who finds it is the one who can do least about it. + . + Clauses: (1) Before a payload is staged, its executable format shall be + parsed and its architecture compared with the platform it is being + deposited under; a mismatch shall abort the deposit naming both. (2) The + check shall work for EVERY platform in the layer, not only the runner's, + so it shall read the file's header rather than execute it — a deposit + runs on one machine and ships four platforms. (3) A payload whose format + is not recognised shall be reported, not silently accepted: "this is not + an executable varve can identify" is a fact the operator should see + before signing, since a wrapper script is a legitimate payload and a + truncated download is not. (4) Where the host CAN run a payload, the + producer shall additionally smoke it — execute it with a harmless + argument and require a clean exit — because a correct architecture is + still not a working binary. (5) These checks shall run before the + irreversible step; a published layer tag is spent and varve has no + revocation. + + - id: REQ-WHICHSTDOUT-001 + type: requirement + title: A resolver's answer is its stdout, and nothing else is + status: verified + release: v0.30.0 + description: > + `varve which ` prints the resolved path AND a provenance line to + stdout, so `M=$(varve which meld)` yields a two-line string that is not + an executable path. A caller doing the obvious thing gets a broken value + from the command whose entire purpose is answering "which binary runs + here". + . + This is not hypothetical and it is not cosmetic. It is what made a + consumer's build script fall through to an ambient `meld 0.41.3` and die + with an `unexpected argument` error naming the wrong version (#102) — + the mixed-toolchain hazard varve exists to close, produced by varve, and + still reproducing on v0.29.0 against a layer published the same day. + . + Clauses: (1) `varve which` shall print the resolved path on stdout and + nothing else on stdout. (2) Provenance — the layer id, channel and + manifest digest — shall go to stderr, where it stays visible to a human + at a terminal and invisible to command substitution. (3) `--json` shall + remain a single JSON document on stdout. (4) The gate shall be the shell + contract itself: a test asserting `[ -x "$(varve which )" ]`, run + against a real installed layer rather than a fixture, because a fixture + that never runs the shell cannot see this class of defect. (5) Every + other command whose output a script would capture shall be audited for + the same shape, since the defect is one of habit rather than of this one + call site. + + - id: REQ-NOKEYDISK-001 + type: requirement + title: The realm's signing key does not touch disk, in varve's own pipelines first + status: implemented + release: v0.30.0 + description: > + `varve docs ci` tells adopters that "every adopter therefore invents + `echo "$SECRET" > key.tmp`, which leaves the realm's one secret on disk", + and documents two forms that avoid the file entirely — a pipe through + `/dev/stdin`, and process substitution. `docs root-ceremony` states the + rule outright: in CI the key "reaches varve through a file descriptor, + never a workspace file". + . + varve's own deposit workflow writes it to `/tmp/rolling.key` on a shared + cloud runner. Not to a `mktemp` path, not mode 0600 — a predictable name, + the pattern the documentation names as the thing adopters wrongly invent. + An independent assessor found this by reading the repository rather than + the documentation, and correctly called it the finding that invalidates + every other procedural claim by induction: a published procedure the + publisher does not follow cannot be evidence of anything. + . + Clauses: (1) No varve-operated workflow shall write key material to a + filesystem path; the documented file-descriptor forms shall be used. + (2) A mechanical gate shall refuse a workflow that redirects a secret + into a file — the check must fail on the pattern, not rely on review. + (3) The gate shall carry a negative control proving it goes red, since a + lint that cannot fail is the defect class this project exists to close. + (4) The signing job shall be gated by whatever control the operating + organisation can actually staff, and the documentation shall claim + exactly that and no more. For a SINGLE-OPERATOR realm — which this one + is, at 105 commits by one person — a required-reviewer gate is + unstaffable, and configuring one would produce either a self-approval + (a control in name only) or a permanently blocked pipeline. Neither is + better than saying so. What such an operator can have instead is + DETECTION rather than prevention: a wait timer that opens a cancel + window, an environment restricted to the ref the deposit may run from, + and above all a durable record of what was signed and when, so an + unauthorised signature is discoverable after the fact. Prevention needs + a second person; detection does not. + (5) No varve document shall prescribe a two-person rule for varve's own + realm while varve's own realm has one person. The rule stays in + `docs root-ceremony` as advice to organisations that HAVE two people, + explicitly marked as such, and varve's own posture is stated plainly + beside it. A procedure the publisher cannot follow is the finding that + invalidates every other procedural claim by induction, and this project + has already collected that finding once. + (6) `docs ci` shall stop presenting the file fallback as acceptable for a + realm root: it is acceptable for a throwaway key and nothing else. + - id: REQ-LAYERADAPT-001 type: requirement title: A realm's manifest is translated into assembler inputs by varve, exactly or not at all diff --git a/crates/varve-core/src/layerspec.rs b/crates/varve-core/src/layerspec.rs index f62ddba9..985471e5 100644 --- a/crates/varve-core/src/layerspec.rs +++ b/crates/varve-core/src/layerspec.rs @@ -80,6 +80,27 @@ pub struct ManifestTool { /// Absent = `tarball`. #[serde(default)] pub layout: Option, + /// Why this tool is ingested with NO proof of origin (REQ-INGEST-001 + /// clause 3). Present only for a release that offers neither a + /// cosign-signed sums file nor a build attestation. + /// + /// The reason is not paperwork: it is signed into the layer and shown by + /// `varve inspect`, so every consumer reads the operator's words next to + /// the bytes they were written about. "We could not verify this" must + /// never be the silent path, which is why the field carries prose rather + /// than a boolean. + #[serde(rename = "unverified-reason", default)] + pub unverified_reason: Option, + /// Asset name for one target triple, when no template can derive it. + /// + /// Some upstreams ship a musl binary as their only Linux build — + /// `wac-cli-x86_64-unknown-linux-musl` — and a static musl binary is the + /// right payload for a gnu platform even though nothing in the platform + /// name says so. Inventing a `%MUSL` placeholder would guess at a + /// convention; naming the file is exact, and wrong-by-typo rather than + /// wrong-by-inference. + #[serde(rename = "asset-for", default)] + pub asset_for: std::collections::BTreeMap, } #[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] @@ -124,6 +145,10 @@ pub enum LayerSpecError { /// The assembler carries exactly one raw-per-platform tool, as /// `WSC_VERSION`. A second one has nowhere to go. ManyRawPerPlatform { first: String, second: String }, + /// The assembler's single raw-per-platform slot is not generic: it fetches + /// `wsc` from `pulseengine/sigil`. Any other tool put in it becomes a + /// request for the wrong tool from the wrong repository. + RawPerPlatformNotWsc { tool: String, repo: String }, /// The assembler hardcodes `pulseengine/` for extension repositories. VsixForeignOwner { name: String, repo: String }, /// The assembler derives a tarball tool's identity from its REPOSITORY @@ -135,6 +160,14 @@ pub enum LayerSpecError { }, /// Two entries would land under one name. Duplicate { kind: &'static str, name: String }, + /// An opt-in that states no reason. + UnverifiedWithoutReason { tool: String }, + /// Two tools from one repository disagree about why it is unverified. + ConflictingReason { + repo: String, + first: String, + second: String, + }, /// The manifest describes nothing to deposit. Empty, } @@ -164,6 +197,17 @@ impl fmt::Display for LayerSpecError { them. Teach the assembler a general raw-per-platform list \ before adding the second." ), + LayerSpecError::RawPerPlatformNotWsc { tool, repo } => write!( + f, + "tool {tool:?} from {repo:?} declares \ + layout = \"raw-per-platform\", but the assembler's only slot \ + for that layout is hardcoded to fetch `wsc` from \ + `pulseengine/sigil` — it would emit WSC_VERSION and download \ + the wrong tool, from the wrong repository, at this tool's \ + version, and deposit it under the wrong name. Teach the \ + assembler a general raw-per-platform list before carrying \ + this." + ), LayerSpecError::VsixForeignOwner { name, repo } => write!( f, "vsix {name:?} names repo {repo:?}, but the assembler resolves \ @@ -188,6 +232,25 @@ impl fmt::Display for LayerSpecError { LayerSpecError::Duplicate { kind, name } => { write!(f, "two {kind} entries are both named {name:?}") } + LayerSpecError::UnverifiedWithoutReason { tool } => write!( + f, + "tool {tool:?} sets an empty `unverified-reason`. \"We could \ + not verify this\" must never be the silent path: the reason \ + is what travels with the bytes into the signed layer, where \ + every consumer reads it. Say why this is acceptable and what \ + removes the need, or do not carry the tool." + ), + LayerSpecError::ConflictingReason { + repo, + first, + second, + } => write!( + f, + "two tools from {repo:?} give different reasons for ingesting \ + it unverified:\n {first:?}\n {second:?}\nThe opt-in is per \ + RELEASE, not per tool, so one of these would be recorded and \ + the other silently discarded. Give the repository one reason." + ), LayerSpecError::Empty => write!( f, "layer.toml declares no [[tool]] and no [[vsix]]: there is \ @@ -214,6 +277,8 @@ pub struct AssemblerEnv { pub channel: String, pub registry: String, pub varve_version: String, + /// `owner/repo=reason` lines for releases ingested with no proof. + pub unverified_ingest: Vec<(String, String)>, } impl AssemblerEnv { @@ -230,6 +295,27 @@ impl AssemblerEnv { out.push_str(&format!("VARVE_CHANNEL={}\n", self.channel)); out.push_str(&format!("VARVE_REGISTRY={}\n", self.registry)); out.push_str(&format!("VARVE_VERSION={}\n", self.varve_version)); + // UNVERIFIED_INGEST is LINE-separated, because a reason is prose and + // any punctuation separator can occur inside it — the assembler + // documents that choice and the reason it made it. A `KEY=value` line + // cannot carry newlines, so this uses $GITHUB_ENV's heredoc form. + // + // The delimiter is checked against the content rather than assumed: an + // operator's reason that happened to contain the delimiter would end + // the block early and inject whatever followed as further environment, + // which is the shape of an actual injection rather than a typo. + if !self.unverified_ingest.is_empty() { + let body: String = self + .unverified_ingest + .iter() + .map(|(repo, why)| format!("{repo}={why}\n")) + .collect(); + let mut delim = String::from("VARVE_UNVERIFIED_EOF"); + while body.contains(&delim) { + delim.push('_'); + } + out.push_str(&format!("UNVERIFIED_INGEST<<{delim}\n{body}{delim}\n")); + } out } } @@ -279,6 +365,7 @@ pub fn assembler_env(m: &LayerManifest) -> Result encodable("realm.channel", &m.realm.channel)?; encodable("varve.version", &m.varve.version)?; + let mut unverified: Vec<(String, String)> = Vec::new(); let mut tarballs: Vec = Vec::new(); let mut wsc_version: Option = None; let mut raw_owner: Option = None; @@ -300,6 +387,32 @@ pub fn assembler_env(m: &LayerManifest) -> Result layout: s.to_string(), })?, }; + // The opt-in is per RELEASE, so it is keyed by repository; two tools + // from one repo must agree about why it is unverified, or one reason + // would be recorded and the other silently dropped. + if let Some(why) = &t.unverified_reason { + let why = why.trim(); + if why.is_empty() { + return Err(LayerSpecError::UnverifiedWithoutReason { + tool: t.name.clone(), + }); + } + let full = match &t.repo { + Some(r) => r.clone(), + None => format!("pulseengine/{}", t.name), + }; + if let Some((_, prev)) = unverified.iter().find(|(r, _)| *r == full) { + if prev != why { + return Err(LayerSpecError::ConflictingReason { + repo: full, + first: prev.clone(), + second: why.to_string(), + }); + } + } else { + unverified.push((full, why.to_string())); + } + } let (owner, repo_name) = match &t.repo { Some(r) => { encodable("tool.repo", r)?; @@ -315,6 +428,15 @@ pub fn assembler_env(m: &LayerManifest) -> Result second: t.name.clone(), }); } + // The slot is not generic. `wsc` is what the assembler fetches, + // from `pulseengine/sigil`; anything else silently becomes a + // request for that tool at this tool's version. + if t.name != "wsc" || owner != "pulseengine" || repo_name != "sigil" { + return Err(LayerSpecError::RawPerPlatformNotWsc { + tool: t.name.clone(), + repo: format!("{owner}/{repo_name}"), + }); + } raw_owner = Some(t.name.clone()); wsc_version = Some(t.version.clone()); continue; @@ -408,6 +530,7 @@ pub fn assembler_env(m: &LayerManifest) -> Result channel: m.realm.channel.clone(), registry: m.realm.registry.clone(), varve_version: m.varve.version.clone(), + unverified_ingest: unverified, }) } @@ -536,6 +659,68 @@ asset = "spar-aadl-%P-%V.vsix" ); } + /// Found by actually trying to assemble a bytecodealliance manifest: the + /// assembler's one raw-per-platform slot fetches `wsc` from + /// `pulseengine/sigil`, so putting any other tool in it emitted + /// WSC_VERSION = that tool's version and would have downloaded wsc at + /// v0.10.1 — wrong tool, wrong repo, wrong version, deposited under the + /// wrong name, silently. + // rivet: verifies REQ-LAYERADAPT-001 + #[test] + fn a_raw_per_platform_tool_that_is_not_wsc_is_refused() { + let text = format!( + "{REAL}\n[[tool]]\nname = \"wac\"\nrepo = \"bytecodealliance/wac\"\n\ + version = \"v0.10.1\"\nlayout = \"raw-per-platform\"\n" + ); + // The FIRST raw tool in REAL is wsc, so this trips the many-slot rule; + // remove wsc to isolate the identity rule. + let only = text.replace( + "[[tool]]\nname = \"wsc\"\nrepo = \"pulseengine/sigil\"\nversion = \"v0.11.0\"\nlayout = \"raw-per-platform\"\n", + "", + ); + assert!(!only.contains("wsc"), "the wsc block must be gone: {only}"); + let err = assembler_env(&parse_layer_manifest(&only).unwrap()).unwrap_err(); + assert_eq!( + err, + LayerSpecError::RawPerPlatformNotWsc { + tool: "wac".into(), + repo: "bytecodealliance/wac".into() + }, + "{err}" + ); + assert!(err.to_string().contains("wrong repository"), "{err}"); + } + + /// One wrong field is enough. The slot fetches `wsc` from + /// `pulseengine/sigil`, so a tool that matches two of those three and + /// misses the third still becomes a request for something else — and a + /// test that only varies all three at once cannot tell the guard from a + /// much weaker one. cargo-mutants proved that by narrowing it. + // rivet: verifies REQ-LAYERADAPT-001 + #[test] + fn the_wsc_slot_rejects_a_tool_that_differs_in_any_single_field() { + let base = REAL.replace( + "[[tool]]\nname = \"wsc\"\nrepo = \"pulseengine/sigil\"\nversion = \"v0.11.0\"\nlayout = \"raw-per-platform\"\n", + "", + ); + for (name, repo, what) in [ + ("wsc", "acme/sigil", "a different OWNER"), + ("wsc", "pulseengine/other", "a different REPOSITORY"), + ("other", "pulseengine/sigil", "a different TOOL NAME"), + ] { + let text = format!( + "{base}\n[[tool]]\nname = \"{name}\"\nrepo = \"{repo}\"\n\ + version = \"v1.0.0\"\nlayout = \"raw-per-platform\"\n" + ); + let err = assembler_env(&parse_layer_manifest(&text).unwrap()).expect_err(what); + assert!( + matches!(err, LayerSpecError::RawPerPlatformNotWsc { .. }), + "{what} ({name} from {repo}) was not refused as a wsc-slot \ + mismatch: {err:?}" + ); + } + } + /// Dropping the owner would fetch pulseengine's release of the same name — /// a different repository's bytes, deposited under a good signature. // rivet: verifies REQ-LAYERADAPT-001 @@ -638,6 +823,110 @@ asset = "spar-aadl-%P-%V.vsix" ); } + /// A release offering neither mechanism can be carried only with a stated + /// reason, and the reason is signed into the layer where every consumer + /// reads it. It belongs in the manifest beside the tool it excuses, not in + /// a workflow variable — split definitions are how versions drift (#106). + // rivet: verifies REQ-LAYERADAPT-001 + // rivet: verifies REQ-INGEST-001 + #[test] + fn an_unverified_reason_reaches_the_assembler_intact() { + let text = format!( + "{REAL}\n[[tool]]\nname = \"wac\"\nrepo = \"bytecodealliance/wac\"\n\ + version = \"v0.10.1\"\nunverified-reason = \"publishes no sums, no cosign \ + bundle and no attestation; tracked upstream, re-check each cut\"\n" + ); + let env = env_of(&text); + assert_eq!(env.unverified_ingest.len(), 1); + assert_eq!(env.unverified_ingest[0].0, "bytecodealliance/wac"); + assert!(env.unverified_ingest[0].1.contains("re-check each cut")); + + // Rendered in $GITHUB_ENV's heredoc form, because the value is + // line-separated and a KEY=value line cannot carry newlines. + let r = env.render(); + assert!(r.contains("UNVERIFIED_INGEST<<"), "{r}"); + assert!(r.contains("bytecodealliance/wac=publishes no sums"), "{r}"); + } + + /// A reason containing the delimiter would end the heredoc early and let + /// whatever followed be read as further environment. That is an injection, + /// not a typo, so the delimiter is chosen against the content. + // rivet: verifies REQ-LAYERADAPT-001 + #[test] + fn a_reason_containing_the_delimiter_cannot_close_the_block_early() { + let text = format!( + "{REAL}\n[[tool]]\nname = \"wac\"\nrepo = \"bytecodealliance/wac\"\n\ + version = \"v0.10.1\"\nunverified-reason = \"VARVE_UNVERIFIED_EOF\\nPATH=/evil\"\n" + ); + let r = env_of(&text).render(); + let opened = r + .lines() + .find(|l| l.starts_with("UNVERIFIED_INGEST<<")) + .expect("heredoc opened"); + let delim = opened.trim_start_matches("UNVERIFIED_INGEST<<"); + // The delimiter must not appear inside the body it delimits. + let body = r.split(&format!("<<{delim}\n")).nth(1).expect("body"); + let body = body.split(&format!("\n{delim}")).next().expect("closes"); + assert!( + !body.contains(delim), + "delimiter occurs inside its own body" + ); + assert!( + body.contains("PATH=/evil"), + "the reason must survive verbatim" + ); + } + + /// "We could not verify this" must never be the silent path. + // rivet: verifies REQ-LAYERADAPT-001 + #[test] + fn an_empty_unverified_reason_is_refused() { + for bad in ["\"\"", "\" \""] { + let text = format!( + "{REAL}\n[[tool]]\nname = \"wac\"\nversion = \"v1\"\nunverified-reason = {bad}\n" + ); + let err = assembler_env(&parse_layer_manifest(&text).unwrap()).unwrap_err(); + assert_eq!( + err, + LayerSpecError::UnverifiedWithoutReason { tool: "wac".into() }, + "{bad}" + ); + } + } + + /// The opt-in is per RELEASE. Two tools from one repository giving + /// different reasons would record one and drop the other. + // rivet: verifies REQ-LAYERADAPT-001 + #[test] + fn two_tools_from_one_repo_must_agree_on_the_reason() { + // `wsc` already comes from pulseengine/sigil as a raw-per-platform + // tool, which is exempt from the basename rule; a tarball tool named + // `sigil` from the same repo is the reachable way two payloads share + // one release. Two tarball tools cannot, by construction. + let text = REAL.replace( + "layout = \"raw-per-platform\"", + "layout = \"raw-per-platform\"\nunverified-reason = \"first\"", + ) + "\n[[tool]]\nname = \"sigil\"\nrepo = \"pulseengine/sigil\"\nversion = \"v0.11.0\"\n\ + unverified-reason = \"second\"\n"; + let err = assembler_env(&parse_layer_manifest(&text).unwrap()).unwrap_err(); + assert!( + matches!(err, LayerSpecError::ConflictingReason { .. }), + "{err:?}" + ); + // Agreeing is fine, and recorded once. + let ok = text.replace("\"second\"", "\"first\""); + assert_eq!(env_of(&ok).unverified_ingest.len(), 1); + } + + /// A manifest with nothing unverified must not emit the variable at all — + /// an empty opt-in list and an absent one are different statements. + // rivet: verifies REQ-LAYERADAPT-001 + #[test] + fn a_manifest_with_nothing_unverified_emits_no_opt_in() { + let r = env_of(REAL).render(); + assert!(!r.contains("UNVERIFIED_INGEST"), "{r}"); + } + // rivet: verifies REQ-LAYERADAPT-001 #[test] fn a_manifest_with_no_payloads_is_refused() { diff --git a/crates/varve-producer/Cargo.toml b/crates/varve-producer/Cargo.toml index d76e8a82..21df9184 100644 --- a/crates/varve-producer/Cargo.toml +++ b/crates/varve-producer/Cargo.toml @@ -15,3 +15,4 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.151" toml = { version = "0.9.8", features = ["serde"] } varve-core.workspace = true + diff --git a/crates/varve-producer/src/asset.rs b/crates/varve-producer/src/asset.rs index 54b72d00..f691a0ea 100644 --- a/crates/varve-producer/src/asset.rs +++ b/crates/varve-producer/src/asset.rs @@ -70,6 +70,12 @@ pub enum Placeholder { UpstreamTag, /// `%P` — the VS Code platform tag. VsCodePlatform, + /// `%R` — the release tag exactly as the manifest writes it, leading `v` + /// included. `%V` strips that `v`, and several upstreams keep it: + /// `wasmtime-v48.0.1-aarch64-macos.tar.xz`. Without this a manifest has to + /// hardcode the version inside the template, so a version bump edits two + /// places and one of them eventually gets missed. + ReleaseTag, } impl Placeholder { @@ -79,6 +85,7 @@ impl Placeholder { Placeholder::Triple => "%T", Placeholder::UpstreamTag => "%U", Placeholder::VsCodePlatform => "%P", + Placeholder::ReleaseTag => "%R", } } @@ -87,6 +94,7 @@ impl Placeholder { Placeholder::Triple, Placeholder::UpstreamTag, Placeholder::VsCodePlatform, + Placeholder::ReleaseTag, ]; } @@ -183,7 +191,11 @@ pub fn expand( } } - let mut out = template.replace(Placeholder::BareVersion.token(), bare_version(version)); + // %R before %V: both mention the version, and expanding the bare form + // first would leave a stray `v` in front of it. + let mut out = template + .replace(Placeholder::ReleaseTag.token(), version) + .replace(Placeholder::BareVersion.token(), bare_version(version)); if let Some(triple) = platform { out = out.replace(Placeholder::Triple.token(), triple); if out.contains(Placeholder::UpstreamTag.token()) { @@ -397,6 +409,41 @@ mod tests { } } + /// Several upstreams keep the `v` in their asset names — + /// `wasmtime-v48.0.1-aarch64-macos.tar.xz`. Found by planning a real + /// bytecodealliance manifest and checking every name against the release: + /// four of twelve did not exist, and two of those were this. + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn the_release_tag_is_available_as_written_not_only_bare() { + assert_eq!( + expand( + "wasmtime-%R-%U.tar.xz", + "v48.0.1", + Some("aarch64-apple-darwin"), + None + ) + .expect("expands"), + "wasmtime-v48.0.1-aarch64-macos.tar.xz" + ); + // And the bare form still strips it. + assert_eq!( + expand("t-%V.tar.gz", "v48.0.1", None, None).expect("expands"), + "t-48.0.1.tar.gz" + ); + } + + /// A template using both must not leave a stray `v`: expanding %V first + /// would turn "%R" into "v" + the already-substituted bare version. + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn a_template_using_both_version_forms_expands_each_correctly() { + assert_eq!( + expand("x-%R-y-%V.tar.gz", "v1.2.3", None, None).expect("expands"), + "x-v1.2.3-y-1.2.3.tar.gz" + ); + } + /// The defect a clean-room review found in this very module: `%P` was /// never substituted, so a real VSIX template expanded to a literal /// `spar-aadl-%P-0.34.0.vsix`, matched nothing on every platform, and the diff --git a/crates/varve-producer/src/binfmt.rs b/crates/varve-producer/src/binfmt.rs new file mode 100644 index 00000000..9ef52543 --- /dev/null +++ b/crates/varve-producer/src/binfmt.rs @@ -0,0 +1,380 @@ +//! Does this file's architecture match the platform it is deposited under? +//! (REQ-PAYLOADSMOKE-001) +//! +//! Everything else the producer verifies answers one question: are these the +//! bytes upstream published? A signed sums file establishes that, and the +//! digest in the layer transcribes it. **None of it establishes that the bytes +//! are a working tool for the platform they are filed under.** +//! +//! An upstream that ships an x86_64 binary inside its `aarch64` tarball +//! produces a layer that assembles, signs, publishes and installs perfectly. +//! The digest is right — it faithfully records the wrong file. The failure +//! surfaces on a consumer's machine as `cannot execute binary file`, which is +//! the one place nobody can fix it. +//! +//! So the header is read. Reading, not executing: a deposit runs on one +//! machine and ships four platforms, and a check that only covers the runner's +//! own architecture would miss three quarters of the layer. + +use std::fmt; + +/// The machine an executable declares itself built for. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Arch { + X86_64, + Aarch64, +} + +impl Arch { + pub fn as_str(self) -> &'static str { + match self { + Arch::X86_64 => "x86_64", + Arch::Aarch64 => "aarch64", + } + } + + /// The architecture a Rust target triple names. + pub fn of_triple(triple: &str) -> Option { + match triple.split('-').next()? { + "x86_64" => Some(Arch::X86_64), + "aarch64" => Some(Arch::Aarch64), + _ => None, + } + } +} + +/// What a file's header says it is. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Format { + Elf(Arch), + MachO(Arch), + /// A `#!` script. A legitimate payload, but it is not architecture-bound, + /// so it is reported rather than checked. + Script, + /// A recognised container varve deliberately does not resolve: a universal + /// Mach-O holds several architectures at once. + MachOUniversal, + /// Not a format this knows. Not necessarily wrong — but the operator + /// should see it before signing. + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ArchError { + /// The header says one architecture, the layer files it under another. + Mismatch { + path: String, + platform: String, + declared: Arch, + expected: Arch, + }, + /// Too short to have a header at all — a truncated download hashes + /// perfectly and is not a program. + TooShort { path: String, len: usize }, +} + +impl fmt::Display for ArchError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ArchError::Mismatch { + path, + platform, + declared, + expected, + } => write!( + f, + "{path} is a {} binary but is being deposited as {platform}, \ + which needs {}. The digest of this file is correct — it is \ + faithfully recording the wrong file, which is why nothing else \ + in the pipeline notices. Upstream has almost certainly shipped \ + the wrong binary inside that archive; a consumer would discover \ + it as 'cannot execute binary file'.", + declared.as_str(), + expected.as_str() + ), + ArchError::TooShort { path, len } => write!( + f, + "{path} is {len} byte(s) — too short to be an executable. A \ + truncated download hashes perfectly and is not a program." + ), + } + } +} + +impl std::error::Error for ArchError {} + +/// Identify a file from its leading bytes. +pub fn identify(bytes: &[u8]) -> Format { + if bytes.starts_with(b"#!") { + return Format::Script; + } + // ELF: 0x7F "ELF", then class/data, and e_machine as a 16-bit field at + // offset 18 whose endianness is declared by EI_DATA at offset 5. + if bytes.starts_with(&[0x7F, b'E', b'L', b'F']) && bytes.len() >= 20 { + let little = bytes[5] == 1; + let machine = if little { + u16::from_le_bytes([bytes[18], bytes[19]]) + } else { + u16::from_be_bytes([bytes[18], bytes[19]]) + }; + return match machine { + 0x3E => Format::Elf(Arch::X86_64), + 0xB7 => Format::Elf(Arch::Aarch64), + _ => Format::Unknown, + }; + } + if bytes.len() >= 8 { + let magic = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + // Mach-O 64-bit, little-endian host order (0xFEEDFACF). + if magic == 0xFEED_FACF { + let cputype = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]); + return match cputype { + 0x0100_0007 => Format::MachO(Arch::X86_64), + 0x0100_000C => Format::MachO(Arch::Aarch64), + _ => Format::Unknown, + }; + } + // A universal ("fat") binary carries several architectures; the magic + // is big-endian by definition. + let be = u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + if be == 0xCAFE_BABE || be == 0xCAFE_BABF { + return Format::MachOUniversal; + } + } + Format::Unknown +} + +/// The check itself: refuse a payload whose architecture contradicts the +/// platform it is filed under. +/// +/// A script, a universal binary, or an unrecognised format is NOT an error — +/// each can be a legitimate payload — but the caller is told, because "varve +/// cannot identify this" is a fact worth seeing before signing. +pub fn check_platform(path: &str, bytes: &[u8], platform: &str) -> Result { + if bytes.len() < 4 { + return Err(ArchError::TooShort { + path: path.to_string(), + len: bytes.len(), + }); + } + let format = identify(bytes); + let declared = match format { + Format::Elf(a) | Format::MachO(a) => a, + _ => return Ok(format), + }; + // A platform varve does not map is not something to fail on here; the + // deposit spec's own platform validation owns that. + let Some(expected) = Arch::of_triple(platform) else { + return Ok(format); + }; + if declared != expected { + return Err(ArchError::Mismatch { + path: path.to_string(), + platform: platform.to_string(), + declared, + expected, + }); + } + Ok(format) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn elf(machine: u16, little: bool) -> Vec { + let mut v = vec![0u8; 20]; + v[..4].copy_from_slice(&[0x7F, b'E', b'L', b'F']); + v[4] = 2; // 64-bit + v[5] = if little { 1 } else { 2 }; + let m = if little { + machine.to_le_bytes() + } else { + machine.to_be_bytes() + }; + v[18] = m[0]; + v[19] = m[1]; + v + } + + fn macho(cputype: u32) -> Vec { + let mut v = Vec::new(); + v.extend_from_slice(&0xFEED_FACFu32.to_le_bytes()); + v.extend_from_slice(&cputype.to_le_bytes()); + v + } + + // rivet: verifies REQ-PAYLOADSMOKE-001 + #[test] + fn elf_architectures_are_read_from_the_header() { + assert_eq!(identify(&elf(0x3E, true)), Format::Elf(Arch::X86_64)); + assert_eq!(identify(&elf(0xB7, true)), Format::Elf(Arch::Aarch64)); + } + + /// A big-endian ELF declares its own byte order at EI_DATA; reading the + /// machine field little-endian regardless would misidentify it. + // rivet: verifies REQ-PAYLOADSMOKE-001 + #[test] + fn a_big_endian_elf_is_read_in_its_own_byte_order() { + assert_eq!(identify(&elf(0xB7, false)), Format::Elf(Arch::Aarch64)); + } + + // rivet: verifies REQ-PAYLOADSMOKE-001 + #[test] + fn mach_o_architectures_are_read_from_the_header() { + assert_eq!(identify(&macho(0x0100_0007)), Format::MachO(Arch::X86_64)); + assert_eq!(identify(&macho(0x0100_000C)), Format::MachO(Arch::Aarch64)); + } + + /// THE case this module exists for: upstream ships the wrong binary inside + /// an architecture's archive. The digest is correct and records the wrong + /// file, so nothing else in the pipeline can see it. + // rivet: verifies REQ-PAYLOADSMOKE-001 + #[test] + fn an_x86_binary_filed_as_aarch64_is_refused() { + let err = check_platform("tools/rivet", &elf(0x3E, true), "aarch64-unknown-linux-gnu") + .expect_err("must refuse"); + assert_eq!( + err, + ArchError::Mismatch { + path: "tools/rivet".into(), + platform: "aarch64-unknown-linux-gnu".into(), + declared: Arch::X86_64, + expected: Arch::Aarch64, + } + ); + assert!( + err.to_string() + .contains("faithfully recording the wrong file"), + "{err}" + ); + } + + // rivet: verifies REQ-PAYLOADSMOKE-001 + #[test] + fn a_matching_architecture_passes_for_every_platform_the_layer_carries() { + for (triple, bytes) in [ + ("x86_64-unknown-linux-gnu", elf(0x3E, true)), + ("aarch64-unknown-linux-gnu", elf(0xB7, true)), + ("x86_64-apple-darwin", macho(0x0100_0007)), + ("aarch64-apple-darwin", macho(0x0100_000C)), + ] { + check_platform("t", &bytes, triple) + .unwrap_or_else(|e| panic!("{triple} rejected its own binary: {e}")); + } + } + + /// A deposit runs on ONE machine and ships four platforms. The check must + /// not depend on being able to run the thing. + // rivet: verifies REQ-PAYLOADSMOKE-001 + #[test] + fn a_foreign_platform_is_checked_without_executing_it() { + // Reading a Mach-O arm64 header while notionally on x86 Linux. + assert!(check_platform("t", &macho(0x0100_000C), "aarch64-apple-darwin").is_ok()); + assert!(check_platform("t", &elf(0xB7, true), "aarch64-unknown-linux-gnu").is_ok()); + } + + /// A truncated download hashes perfectly and is not a program. + // rivet: verifies REQ-PAYLOADSMOKE-001 + #[test] + fn a_truncated_file_is_refused_rather_than_called_unknown() { + let err = check_platform("t", b"\x7f", "x86_64-unknown-linux-gnu").expect_err("refuses"); + assert!(matches!(err, ArchError::TooShort { len: 1, .. }), "{err:?}"); + } + + /// A wrapper script is a legitimate payload and carries no architecture. + /// It is reported, not refused — the two are different answers. + // rivet: verifies REQ-PAYLOADSMOKE-001 + #[test] + fn a_script_is_reported_and_not_refused() { + let f = check_platform( + "t", + b"#!/bin/sh\nexec real \"$@\"\n", + "aarch64-apple-darwin", + ) + .expect("scripts are legitimate payloads"); + assert_eq!(f, Format::Script); + } + + // rivet: verifies REQ-PAYLOADSMOKE-001 + #[test] + fn a_universal_binary_is_recognised_rather_than_guessed_at() { + let mut fat = 0xCAFE_BABEu32.to_be_bytes().to_vec(); + fat.extend_from_slice(&[0, 0, 0, 2]); + assert_eq!(identify(&fat), Format::MachOUniversal); + assert!(check_platform("t", &fat, "x86_64-apple-darwin").is_ok()); + } + + // rivet: verifies REQ-PAYLOADSMOKE-001 + #[test] + fn an_unknown_format_is_surfaced_not_silently_accepted() { + assert_eq!(identify(b"MZ\x90\x00padding here"), Format::Unknown); + let f = check_platform("t", b"MZ\x90\x00padding here", "x86_64-unknown-linux-gnu") + .expect("not an error, but visible"); + assert_eq!(f, Format::Unknown); + } + + /// A file that begins like an ELF but stops before the machine field is a + /// truncated download, not an ELF. cargo-mutants found this by widening + /// the length guard: without it, reading the header walks off the end. + // rivet: verifies REQ-PAYLOADSMOKE-001 + #[test] + fn a_header_that_stops_before_the_machine_field_is_not_an_elf() { + for len in 4..20usize { + let mut v = vec![0u8; len]; + v[..4].copy_from_slice(&[0x7F, b'E', b'L', b'F']); + assert_eq!( + identify(&v), + Format::Unknown, + "len {len} claimed to be an ELF" + ); + // And it must not panic through the checked entry point either. + let _ = check_platform("t", &v, "x86_64-unknown-linux-gnu"); + } + } + + /// Exactly at the length guard: four bytes is enough to look at, so it must + /// be identified (as Unknown) rather than refused as truncated. + // rivet: verifies REQ-PAYLOADSMOKE-001 + #[test] + fn four_bytes_is_short_but_not_too_short() { + let f = check_platform("t", b"\x7fELF", "x86_64-unknown-linux-gnu") + .expect("four bytes is inspectable"); + assert_eq!(f, Format::Unknown); + // Three is not. + assert!(matches!( + check_platform("t", b"\x7fEL", "x86_64-unknown-linux-gnu"), + Err(ArchError::TooShort { len: 3, .. }) + )); + } + + /// The message has to NAME both architectures — it is what tells an + /// operator which side is wrong, and it is the only output of this check. + // rivet: verifies REQ-PAYLOADSMOKE-001 + #[test] + fn the_mismatch_message_names_both_architectures() { + let msg = check_platform("tools/rivet", &elf(0x3E, true), "aarch64-unknown-linux-gnu") + .unwrap_err() + .to_string(); + assert!(msg.contains("x86_64"), "does not name what it IS: {msg}"); + assert!( + msg.contains("aarch64"), + "does not name what was EXPECTED: {msg}" + ); + assert_eq!(Arch::X86_64.as_str(), "x86_64"); + assert_eq!(Arch::Aarch64.as_str(), "aarch64"); + } + + // rivet: verifies REQ-PAYLOADSMOKE-001 + #[test] + fn the_triple_to_arch_mapping_covers_the_layers_platforms() { + assert_eq!(Arch::of_triple("x86_64-apple-darwin"), Some(Arch::X86_64)); + assert_eq!( + Arch::of_triple("aarch64-unknown-linux-gnu"), + Some(Arch::Aarch64) + ); + assert_eq!(Arch::of_triple("riscv64-unknown-linux-gnu"), None); + // An unmapped platform is not this check's business to fail on. + assert!(check_platform("t", &elf(0x3E, true), "riscv64-unknown-linux-gnu").is_ok()); + } +} diff --git a/crates/varve-producer/src/carryforward.rs b/crates/varve-producer/src/carryforward.rs new file mode 100644 index 00000000..9058cefc --- /dev/null +++ b/crates/varve-producer/src/carryforward.rs @@ -0,0 +1,374 @@ +//! Doing the work that changed, and re-proving the work that did not +//! (REQ-CARRYFORWARD-001). +//! +//! Every deposit currently downloads all four platforms of every tool — 421 +//! MiB for layer 2026.08.4 — re-hashes and re-pushes them, even when one tool +//! moved. Consumers never pay this: `install` filters by platform before +//! fetching, so a machine pulls 99 MiB. The waste is the producer's, and daily +//! scanning multiplies it by the cadence. +//! +//! The previous layer's **signed** manifest already records each payload's +//! repo, release, asset and sha256. When the manifest still pins that release, +//! the bytes are already in the registry under that digest. +//! +//! ## The trap, which is the whole design +//! +//! A release asset can be deleted and re-uploaded under the same tag. "rivet +//! v0.34.0" today is not necessarily the bytes it was yesterday. Carrying a +//! digest forward because the *version string* matched would make varve blind +//! to precisely the substitution it exists to catch — and it would be blind +//! silently, because every later check would agree with the carried digest. +//! +//! So the saving comes from skipping the **download**, never from skipping the +//! **proof**. The ingestion proof is re-established every time; a sums file is +//! kilobytes and the binary it describes is tens of megabytes. Only once +//! upstream's CURRENT digest is in hand, and equal, do the bytes go unfetched. +//! +//! When they disagree, that is not a cache miss. It is an upstream that +//! re-published a release under the same version, and it stops the deposit. + +use std::fmt; + +/// What the previous layer's signed manifest recorded for one payload. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PreviousEntry { + pub repo: String, + pub release: String, + pub asset: String, + /// The sha256 the previous deposit recorded, transcribed from a verified + /// sums file or attestation at that time. + pub sha256: String, +} + +/// What to do about one payload this time round. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Decision { + /// Bytes already in the registry under a digest upstream still vouches + /// for. Nothing to download, nothing to push. + Reuse { sha256: String }, + /// Fetch and stage normally, for the stated reason — reasons are kept so + /// a deposit can report WHY it did the expensive thing. + Fetch { why: FetchReason }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FetchReason { + /// No previous layer, or this payload is new to the layer. + NoPrevious, + /// The manifest pins a different release than last time. + ReleaseChanged, + /// The asset name changed even though the release did not — a template + /// edit, or upstream renaming its archive. + AssetChanged, + /// Digests agree, but the blob is no longer in the destination registry. + BlobAbsent, +} + +impl FetchReason { + pub fn as_str(self) -> &'static str { + match self { + FetchReason::NoPrevious => "no previous entry", + FetchReason::ReleaseChanged => "release changed", + FetchReason::AssetChanged => "asset name changed", + FetchReason::BlobAbsent => "blob absent from the registry", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CarryError { + /// Upstream re-published the same release with different bytes. + UpstreamRepublished { + repo: String, + release: String, + asset: String, + previously: String, + now: String, + }, +} + +impl fmt::Display for CarryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + CarryError::UpstreamRepublished { + repo, + release, + asset, + previously, + now, + } => write!( + f, + "{repo} {release} now publishes {asset} with a DIFFERENT digest \ + than the layer already signed:\n previously {previously}\n \ + now {now}\nThe release tag did not change, so the \ + bytes behind it were replaced. That is either an upstream \ + re-release or a substitution, and varve cannot tell which — \ + which is exactly why it will not choose. Establish what \ + happened upstream, then either pin the new release explicitly \ + or stop carrying this tool." + ), + } + } +} + +impl std::error::Error for CarryError {} + +/// Decide what to do about one payload. +/// +/// `upstream_sha256` is what a FRESHLY verified sums file or attestation says +/// the asset hashes to, right now. Obtaining it is the proof step, and it is +/// unconditional — this function is called with it in hand, never instead of +/// it. +pub fn decide( + previous: Option<&PreviousEntry>, + repo: &str, + release: &str, + asset: &str, + upstream_sha256: &str, + blob_present_in_registry: bool, +) -> Result { + let Some(prev) = previous else { + return Ok(Decision::Fetch { + why: FetchReason::NoPrevious, + }); + }; + if prev.repo != repo || prev.release != release { + return Ok(Decision::Fetch { + why: FetchReason::ReleaseChanged, + }); + } + if prev.asset != asset { + return Ok(Decision::Fetch { + why: FetchReason::AssetChanged, + }); + } + // Same repo, same release, same asset — and now the question that makes + // this safe rather than merely fast. + if !digest_eq(&prev.sha256, upstream_sha256) { + return Err(CarryError::UpstreamRepublished { + repo: repo.to_string(), + release: release.to_string(), + asset: asset.to_string(), + previously: prev.sha256.clone(), + now: upstream_sha256.to_string(), + }); + } + if !blob_present_in_registry { + // A manifest entry is a record, not a guarantee of storage: a registry + // can garbage-collect, and a realm can be re-hosted. + return Ok(Decision::Fetch { + why: FetchReason::BlobAbsent, + }); + } + Ok(Decision::Reuse { + sha256: prev.sha256.clone(), + }) +} + +/// Compare digests without letting spelling decide a trust question. +/// +/// One side may be `sha256:`-prefixed (the OCI form) and the other bare (the +/// sums-file form), and case differs between tools. A comparison that treated +/// those as different would fetch needlessly; one that ignored more than case +/// and prefix would compare the wrong thing. +fn digest_eq(a: &str, b: &str) -> bool { + let norm = |s: &str| s.trim().trim_start_matches("sha256:").to_ascii_lowercase(); + let (a, b) = (norm(a), norm(b)); + !a.is_empty() && a == b +} + +#[cfg(test)] +mod tests { + use super::*; + + const D1: &str = "1111111111111111111111111111111111111111111111111111111111111111"; + const D2: &str = "2222222222222222222222222222222222222222222222222222222222222222"; + + fn prev() -> PreviousEntry { + PreviousEntry { + repo: "pulseengine/rivet".into(), + release: "v0.34.0".into(), + asset: "rivet-v0.34.0-aarch64-apple-darwin.tar.gz".into(), + sha256: D1.into(), + } + } + + fn decide_same(upstream: &str, present: bool) -> Result { + let p = prev(); + decide( + Some(&p), + "pulseengine/rivet", + "v0.34.0", + "rivet-v0.34.0-aarch64-apple-darwin.tar.gz", + upstream, + present, + ) + } + + // rivet: verifies REQ-CARRYFORWARD-001 + #[test] + fn an_unchanged_payload_upstream_still_vouches_for_is_reused() { + assert_eq!( + decide_same(D1, true).expect("reuses"), + Decision::Reuse { sha256: D1.into() } + ); + } + + /// THE reason this is not a version-string cache. Upstream can delete and + /// re-upload an asset under the same tag; reusing on the strength of the + /// version alone would make varve blind to the substitution it exists to + /// catch, and blind silently. + // rivet: verifies REQ-CARRYFORWARD-001 + #[test] + fn an_upstream_that_republished_the_same_release_aborts() { + let err = decide_same(D2, true).expect_err("must abort"); + assert_eq!( + err, + CarryError::UpstreamRepublished { + repo: "pulseengine/rivet".into(), + release: "v0.34.0".into(), + asset: "rivet-v0.34.0-aarch64-apple-darwin.tar.gz".into(), + previously: D1.into(), + now: D2.into(), + } + ); + let msg = err.to_string(); + assert!( + msg.contains(D1) && msg.contains(D2), + "both digests must be named: {msg}" + ); + assert!(msg.contains("varve cannot tell which"), "{msg}"); + } + + /// It must abort even when the blob is gone — the substitution is the + /// finding, and a missing blob does not make it a routine fetch. + // rivet: verifies REQ-CARRYFORWARD-001 + #[test] + fn a_republished_upstream_aborts_even_when_the_blob_is_absent() { + assert!(matches!( + decide_same(D2, false), + Err(CarryError::UpstreamRepublished { .. }) + )); + } + + // rivet: verifies REQ-CARRYFORWARD-001 + #[test] + fn a_missing_blob_is_fetched_even_though_the_digest_agrees() { + assert_eq!( + decide_same(D1, false).expect("fetches"), + Decision::Fetch { + why: FetchReason::BlobAbsent + } + ); + } + + // rivet: verifies REQ-CARRYFORWARD-001 + #[test] + fn a_new_payload_is_fetched() { + assert_eq!( + decide(None, "r", "v1", "a.tar.gz", D1, true).expect("fetches"), + Decision::Fetch { + why: FetchReason::NoPrevious + } + ); + } + + // rivet: verifies REQ-CARRYFORWARD-001 + #[test] + fn a_version_bump_is_fetched_and_says_so() { + let p = prev(); + assert_eq!( + decide( + Some(&p), + "pulseengine/rivet", + "v0.35.0", + "rivet-v0.35.0-aarch64-apple-darwin.tar.gz", + D2, + true + ) + .expect("fetches"), + Decision::Fetch { + why: FetchReason::ReleaseChanged + } + ); + } + + /// A repo change at the same version is not the same payload, and must not + /// inherit the previous digest. + // rivet: verifies REQ-CARRYFORWARD-001 + #[test] + fn a_repo_change_at_the_same_version_is_fetched() { + let p = prev(); + assert_eq!( + decide( + Some(&p), + "acme/rivet", + "v0.34.0", + "rivet-v0.34.0-aarch64-apple-darwin.tar.gz", + D1, + true + ) + .expect("fetches"), + Decision::Fetch { + why: FetchReason::ReleaseChanged + } + ); + } + + /// A template edit renames the asset without moving the release. + // rivet: verifies REQ-CARRYFORWARD-001 + #[test] + fn an_asset_rename_at_the_same_release_is_fetched() { + let p = prev(); + assert_eq!( + decide( + Some(&p), + "pulseengine/rivet", + "v0.34.0", + "rivet-0.34.0-aarch64-apple-darwin.tar.gz", + D1, + true + ) + .expect("fetches"), + Decision::Fetch { + why: FetchReason::AssetChanged + } + ); + } + + /// The OCI form and the sums-file form of the same digest are the same + /// digest. Treating them as different would fetch needlessly; ignoring + /// more than case and prefix would compare the wrong thing. + // rivet: verifies REQ-CARRYFORWARD-001 + #[test] + fn the_two_spellings_of_one_digest_agree_and_nothing_looser_does() { + assert!(digest_eq(D1, &format!("sha256:{D1}"))); + assert!(digest_eq(&D1.to_uppercase(), D1)); + assert!(digest_eq(&format!(" {D1} "), D1)); + assert!(!digest_eq(D1, D2)); + // An empty digest is not equal to anything, including another empty + // one — "we have no digest" must never satisfy a comparison. + assert!(!digest_eq("", "")); + assert!(!digest_eq("sha256:", "")); + // A prefix is not a match. + assert!(!digest_eq(&D1[..32], D1)); + } + + // rivet: verifies REQ-CARRYFORWARD-001 + #[test] + fn every_fetch_reason_can_say_why() { + for r in [ + FetchReason::NoPrevious, + FetchReason::ReleaseChanged, + FetchReason::AssetChanged, + FetchReason::BlobAbsent, + ] { + assert!(!r.as_str().is_empty()); + } + assert_eq!( + FetchReason::BlobAbsent.as_str(), + "blob absent from the registry" + ); + assert_eq!(FetchReason::NoPrevious.as_str(), "no previous entry"); + } +} diff --git a/crates/varve-producer/src/extract.rs b/crates/varve-producer/src/extract.rs new file mode 100644 index 00000000..f5ded82a --- /dev/null +++ b/crates/varve-producer/src/extract.rs @@ -0,0 +1,324 @@ +//! Finding the binary inside an extracted release archive (REQ-PRODUCER-002). +//! +//! Release layouts differ per repository — some archives are flat, some carry a +//! versioned subdirectory, some ship docs and completions alongside — so the +//! binary is located by NAME rather than by a fixed path. +//! +//! ## What the shell did +//! +//! ```text +//! bin="$(find "extract/$tool-$platform" -type f -name "$binname" | head -1)" +//! ``` +//! +//! Two defects in one line, and both decide which bytes get SIGNED: +//! +//! * `head -1` takes the first result in **filesystem enumeration order**, +//! which is not deterministic across machines or filesystems. An archive +//! carrying `bin/rivet` and `share/doc/examples/rivet` deposits whichever +//! the kernel happened to hand back first — and the same archive can resolve +//! differently on the next runner. +//! * `-type f` never checks whether the file is executable, so a README named +//! `rivet` is an equally valid candidate. +//! +//! This module refuses ambiguity instead of resolving it by luck. When two +//! candidates are genuinely indistinguishable the answer is an error naming +//! both, because guessing here means signing bytes nobody chose. + +use std::fmt; +use std::path::{Component, Path, PathBuf}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ExtractError { + /// No file of that name anywhere in the archive. + NotFound { name: String, saw: Vec }, + /// Files of that name exist, but none is executable. + NoneExecutable { name: String, found: Vec }, + /// Several equally-good executables. Refused rather than guessed. + Ambiguous { name: String, found: Vec }, +} + +impl fmt::Display for ExtractError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ExtractError::NotFound { name, saw } => write!( + f, + "the archive contains no file named {name:?}. It contains: {}. \ + The asset template probably names the wrong archive, or the \ + release renamed its binary — either way the payload would be \ + missing from a layer that still signs.", + preview(saw) + ), + ExtractError::NoneExecutable { name, found } => write!( + f, + "the archive contains {name:?} but nothing executable: {}. A \ + non-executable match is documentation or a completion script, \ + not the tool; depositing it would put a file in the layer that \ + cannot be dispatched.", + preview(found) + ), + ExtractError::Ambiguous { name, found } => write!( + f, + "the archive contains more than one executable {name:?} and \ + varve will not choose between them: {}. Which one is deposited \ + decides which bytes get signed, so it is not a guess worth \ + making — name the path explicitly in the manifest, or fix the \ + archive.", + preview(found) + ), + } + } +} + +impl std::error::Error for ExtractError {} + +fn preview(items: &[String]) -> String { + let shown: Vec<&str> = items.iter().take(8).map(String::as_str).collect(); + if items.len() > shown.len() { + format!( + "{} … and {} more", + shown.join(", "), + items.len() - shown.len() + ) + } else if shown.is_empty() { + "nothing".to_string() + } else { + shown.join(", ") + } +} + +/// One file considered for dispatch. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Candidate { + pub path: PathBuf, + pub executable: bool, +} + +/// Is this path directly inside a directory called `bin`? +/// +/// The one tie-break worth having: every layout that ships more than one file +/// of the same name puts the real tool under `bin/` and the copy somewhere +/// else. Beyond that, refuse. +fn in_bin_dir(path: &Path) -> bool { + path.parent() + .and_then(Path::file_name) + .is_some_and(|d| d == "bin") +} + +/// Choose the binary to deposit, deterministically or not at all. +/// +/// `candidates` is every file found in the extraction; the caller supplies it +/// so this stays a pure function over a listing rather than a directory walk. +pub fn choose_binary(name: &str, candidates: &[Candidate]) -> Result { + let named: Vec<&Candidate> = candidates + .iter() + .filter(|c| c.path.file_name().is_some_and(|n| n == name)) + .collect(); + + if named.is_empty() { + let mut saw: Vec = candidates.iter().map(|c| show(&c.path)).collect(); + saw.sort(); + return Err(ExtractError::NotFound { + name: name.to_string(), + saw, + }); + } + + let mut exec: Vec<&Candidate> = named.iter().copied().filter(|c| c.executable).collect(); + if exec.is_empty() { + let mut found: Vec = named.iter().map(|c| show(&c.path)).collect(); + found.sort(); + return Err(ExtractError::NoneExecutable { + name: name.to_string(), + found, + }); + } + + if exec.len() > 1 { + let under_bin: Vec<&Candidate> = exec + .iter() + .copied() + .filter(|c| in_bin_dir(&c.path)) + .collect(); + if under_bin.len() == 1 { + return Ok(under_bin[0].path.clone()); + } + // Sorted so the error is identical on every machine, which matters + // when the message is the only record of why a deposit stopped. + let mut found: Vec = exec.iter().map(|c| show(&c.path)).collect(); + found.sort(); + return Err(ExtractError::Ambiguous { + name: name.to_string(), + found, + }); + } + + Ok(exec.remove(0).path.clone()) +} + +fn show(p: &Path) -> String { + // Normalise away a leading `./` so messages compare cleanly. + let mut comps = p.components().peekable(); + if matches!(comps.peek(), Some(Component::CurDir)) { + comps.next(); + } + comps.collect::().display().to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn c(path: &str, executable: bool) -> Candidate { + Candidate { + path: PathBuf::from(path), + executable, + } + } + + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn the_single_executable_of_that_name_is_chosen() { + let got = choose_binary( + "rivet", + &[ + c("rivet-v0.34.0/README.md", false), + c("rivet-v0.34.0/bin/rivet", true), + ], + ) + .expect("chooses"); + assert_eq!(got, PathBuf::from("rivet-v0.34.0/bin/rivet")); + } + + /// Deliberately NOT under `bin/`: a flat archive is a real layout, and a + /// single candidate must be taken without consulting the tie-break at all. + /// The `bin/` case cannot distinguish "one executable" from "several, one + /// of which is under bin" — cargo-mutants found that by widening the + /// comparison and killing nothing. + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn a_lone_executable_outside_bin_is_still_chosen() { + let got = choose_binary("wsc", &[c("wsc", true), c("LICENSE", false)]).expect("chooses"); + assert_eq!(got, PathBuf::from("wsc")); + } + + /// Exactly at the preview cut: eight items are all of them, so nothing is + /// elided and the message must not claim otherwise. + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn a_listing_exactly_at_the_preview_limit_elides_nothing() { + let eight: Vec = (0..8).map(|i| c(&format!("d/f{i}"), false)).collect(); + let msg = choose_binary("nope", &eight).unwrap_err().to_string(); + assert!( + !msg.contains("more"), + "claimed elision with nothing elided: {msg}" + ); + let nine: Vec = (0..9).map(|i| c(&format!("d/f{i}"), false)).collect(); + let msg9 = choose_binary("nope", &nine).unwrap_err().to_string(); + assert!(msg9.contains("and 1 more"), "{msg9}"); + } + + /// The shell's `head -1` took whatever the filesystem offered first. Here + /// the doc copy and the binary are distinguished by `bin/`, which is the + /// only tie-break real layouts justify. + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn a_doc_copy_beside_the_binary_does_not_win_by_enumeration_order() { + for order in [ + vec![c("share/doc/examples/rivet", true), c("bin/rivet", true)], + vec![c("bin/rivet", true), c("share/doc/examples/rivet", true)], + ] { + let got = choose_binary("rivet", &order).expect("chooses"); + assert_eq!( + got, + PathBuf::from("bin/rivet"), + "order changed the answer: {order:?}" + ); + } + } + + /// Two executables, neither under `bin/`: there is no principled winner and + /// the choice decides what gets signed. Refuse. + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn two_indistinguishable_executables_are_refused_not_guessed() { + let err = choose_binary("rivet", &[c("a/rivet", true), c("b/rivet", true)]) + .expect_err("must refuse"); + assert_eq!( + err, + ExtractError::Ambiguous { + name: "rivet".into(), + found: vec!["a/rivet".into(), "b/rivet".into()] + } + ); + assert!( + err.to_string().contains("decides which bytes get signed"), + "{err}" + ); + } + + /// And two under `bin/` are still ambiguous — the tie-break narrows, it + /// does not invent a winner. + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn two_executables_both_under_bin_are_still_refused() { + let err = choose_binary("rivet", &[c("x/bin/rivet", true), c("y/bin/rivet", true)]) + .expect_err("must refuse"); + assert!(matches!(err, ExtractError::Ambiguous { .. }), "{err:?}"); + } + + /// `find -type f` accepted a README. A non-executable match is not the tool. + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn a_non_executable_file_of_the_right_name_is_refused_and_says_why() { + let err = choose_binary("rivet", &[c("docs/rivet", false)]).expect_err("must refuse"); + assert_eq!( + err, + ExtractError::NoneExecutable { + name: "rivet".into(), + found: vec!["docs/rivet".into()] + } + ); + assert!(err.to_string().contains("cannot be dispatched"), "{err}"); + } + + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn an_archive_without_the_binary_lists_what_it_did_contain() { + let err = choose_binary("rivet", &[c("bin/spar", true), c("README.md", false)]) + .expect_err("must refuse"); + let msg = err.to_string(); + assert!(msg.contains("bin/spar"), "{msg}"); + assert!(msg.contains("still signs"), "{msg}"); + } + + /// The error is the only record of why a deposit stopped, so it must read + /// the same on every machine. + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn the_refusal_lists_candidates_in_a_stable_order() { + let a = choose_binary("t", &[c("z/t", true), c("a/t", true)]).unwrap_err(); + let b = choose_binary("t", &[c("a/t", true), c("z/t", true)]).unwrap_err(); + assert_eq!(a, b); + } + + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn a_long_listing_is_previewed_rather_than_dumped() { + let many: Vec = (0..30).map(|i| c(&format!("d/f{i:02}"), false)).collect(); + let msg = choose_binary("nope", &many).unwrap_err().to_string(); + assert!(msg.contains("and 22 more"), "{msg}"); + } + + /// A leading `./` is how `find` reports paths; it must not change identity. + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn a_leading_dot_slash_does_not_change_the_reported_path() { + let err = choose_binary("t", &[c("./a/t", true), c("b/t", true)]).unwrap_err(); + match err { + ExtractError::Ambiguous { found, .. } => { + assert_eq!(found, vec!["a/t".to_string(), "b/t".to_string()]) + } + other => panic!("{other:?}"), + } + } +} diff --git a/crates/varve-producer/src/lib.rs b/crates/varve-producer/src/lib.rs index 7352e10f..f62a59c8 100644 --- a/crates/varve-producer/src/lib.rs +++ b/crates/varve-producer/src/lib.rs @@ -12,7 +12,11 @@ pub mod asset; pub mod attestation; +pub mod binfmt; +pub mod carryforward; +pub mod extract; pub mod forge; pub mod ingest; +pub mod plan; pub mod spec; pub mod sums; diff --git a/crates/varve-producer/src/main.rs b/crates/varve-producer/src/main.rs index f49f5be3..93bf7f01 100644 --- a/crates/varve-producer/src/main.rs +++ b/crates/varve-producer/src/main.rs @@ -6,7 +6,7 @@ //! and pushes to a registry. Keeping them apart keeps that claim true. use clap::{Parser, Subcommand}; -use varve_producer::{asset, forge::Forge}; +use varve_producer::{asset, binfmt, forge::Forge, plan}; #[derive(Parser)] #[command(name = "varve-producer", version, about, long_about = None)] @@ -22,6 +22,26 @@ enum Cmd { /// fetched, because a wrong issuer fails closed but confusingly. Forge, + /// Show the work a deposit would do for a realm manifest, without + /// fetching anything. Reads layer.toml directly — there is no + /// TARBALL_TOOLS/WSC_VERSION encoding to corrupt, and no limit of one + /// raw-per-platform tool. + Plan { + #[arg(long, default_value = "layer.toml")] + manifest: std::path::PathBuf, + #[arg(long = "platform", value_delimiter = ',')] + platforms: Vec, + }, + /// Check a staged payload's architecture against the platform it would be + /// deposited under, without executing it. + Arch { + /// The file to inspect. + #[arg(long)] + file: std::path::PathBuf, + /// The target triple it would be filed under. + #[arg(long)] + platform: String, + }, /// Show which release assets a template selects, without downloading /// anything. The template language is the part of this pipeline that has /// silently dropped a tool from a published layer, so it is inspectable on @@ -53,6 +73,64 @@ fn forge_from_env() -> Forge { fn main() -> anyhow::Result<()> { match Cli::parse().cmd { + Cmd::Plan { + manifest, + platforms, + } => { + let text = std::fs::read_to_string(&manifest) + .map_err(|e| anyhow::anyhow!("cannot read {}: {e}", manifest.display()))?; + let m = varve_core::layerspec::parse_layer_manifest(&text)?; + let owned: Vec = if platforms.is_empty() { + asset::DEFAULT_PLATFORMS + .iter() + .map(|s| (*s).to_string()) + .collect() + } else { + platforms + }; + let refs: Vec<&str> = owned.iter().map(String::as_str).collect(); + let items = plan::plan(&m, &refs)?; + let rels = plan::releases(&items); + println!( + "{} payload(s) from {} release(s), realm '{}'", + items.len(), + rels.len(), + m.realm.name + ); + let unverified = rels + .iter() + .filter(|(repo, _)| { + items + .iter() + .any(|i| &i.repo == repo && i.unverified_reason.is_some()) + }) + .count(); + if unverified > 0 { + println!("{unverified} release(s) carry NO proof of origin (opt-in recorded)"); + } + for i in &items { + println!( + " {:<14} {:<24} {:<26} {}{}", + i.name, + i.repo, + i.platform.as_deref().unwrap_or("(portable)"), + i.asset, + if i.unverified_reason.is_some() { + " [unverified]" + } else { + "" + } + ); + } + Ok(()) + } + Cmd::Arch { file, platform } => { + let bytes = std::fs::read(&file) + .map_err(|e| anyhow::anyhow!("cannot read {}: {e}", file.display()))?; + let format = binfmt::check_platform(&file.display().to_string(), &bytes, &platform)?; + println!("{:<28} {format:?}", platform); + Ok(()) + } Cmd::Forge => { let f = forge_from_env(); println!("host {}", f.host); diff --git a/crates/varve-producer/src/plan.rs b/crates/varve-producer/src/plan.rs new file mode 100644 index 00000000..57a8f0db --- /dev/null +++ b/crates/varve-producer/src/plan.rs @@ -0,0 +1,414 @@ +//! From a realm's manifest to the work a deposit has to do +//! (REQ-PRODUCER-002, REQ-REALM2-002). +//! +//! This is where the port stops being a translation and starts being an +//! improvement. The shell pipeline could not read `layer.toml`; it read three +//! environment variables — `TARBALL_TOOLS`, `WSC_VERSION`, `VSIX_PACKAGES` — +//! encoded as space-separated entries of colon-separated fields. That encoding +//! is why `varve-core::layerspec` has to refuse a version containing a space, +//! why the opt-in reasons need a heredoc, and why one whole class of tests +//! exists. +//! +//! Here the manifest is read directly. There is no encoding, so there is +//! nothing for a separator to corrupt. +//! +//! It also removes a hard limit rather than working around one. The shell +//! carried exactly ONE raw-per-platform tool, because that layout lived in a +//! variable called `WSC_VERSION` that named a specific tool in a specific +//! repository. The `bytecodealliance` realm needs three — `wac`, `wkg` and +//! `wrpc` — and could not be assembled at all. Here `layout` is a property of +//! a tool, so three is not a special case; it is just three. + +use crate::asset::{self, TemplateError}; +use varve_core::layerspec::{LayerManifest, ManifestTool, ManifestVsix}; + +/// What kind of payload a plan item produces. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PayloadKind { + /// A binary inside a per-platform archive. + Tarball, + /// A bare per-platform binary, no archive. + RawPerPlatform, + /// A VS Code extension package. + Vsix, +} + +/// One asset to fetch, verify and stage. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PayloadPlan { + /// The name the payload is deposited under, and dispatched by. + pub name: String, + /// `owner/repo` the release comes from. + pub repo: String, + pub version: String, + /// The release asset, template already expanded. + pub asset: String, + /// `None` for a platform-independent payload. + pub platform: Option, + pub kind: PayloadKind, + /// Why this release is ingested with no proof, if it is. + pub unverified_reason: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PlanError { + Template(TemplateError), + /// A tool declares a layout this planner does not implement. + UnknownLayout { + tool: String, + layout: String, + }, +} + +impl std::fmt::Display for PlanError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PlanError::Template(e) => write!(f, "{e}"), + PlanError::UnknownLayout { tool, layout } => write!( + f, + "tool {tool:?} declares layout = {layout:?}. Known layouts are \ + \"tarball\" (a per-platform archive) and \"raw-per-platform\" \ + (a bare per-platform binary)." + ), + } + } +} + +impl std::error::Error for PlanError {} + +impl From for PlanError { + fn from(e: TemplateError) -> Self { + PlanError::Template(e) + } +} + +fn repo_of(tool_repo: &Option, name: &str) -> String { + match tool_repo { + Some(r) => r.clone(), + None => format!("pulseengine/{name}"), + } +} + +/// The asset template a tool uses, defaulted per layout. +/// +/// A raw-per-platform tool has no archive to name, so its default is the tool +/// name plus the platform — the shape `wsc-linux-x86_64` and +/// `wac-cli-aarch64-apple-darwin` both follow once a template is given. +fn template_of(t: &ManifestTool, kind: PayloadKind) -> String { + if let Some(a) = &t.asset { + return a.clone(); + } + match kind { + PayloadKind::Tarball => asset::default_tarball_template(&t.name, &t.version), + PayloadKind::RawPerPlatform => format!("{}-%T", t.name), + PayloadKind::Vsix => unreachable!("a vsix carries its template"), + } +} + +fn kind_of(t: &ManifestTool) -> Result { + match t.layout.as_deref() { + None | Some("tarball") => Ok(PayloadKind::Tarball), + Some("raw-per-platform") => Ok(PayloadKind::RawPerPlatform), + Some(other) => Err(PlanError::UnknownLayout { + tool: t.name.clone(), + layout: other.to_string(), + }), + } +} + +/// Expand one tool into one plan item per platform. +pub fn plan_tool(t: &ManifestTool, platforms: &[&str]) -> Result, PlanError> { + let kind = kind_of(t)?; + let template = template_of(t, kind); + let repo = repo_of(&t.repo, &t.name); + // `binary` names the executable when it differs from the tool (kiln ships + // kilnd); the payload is deposited under THAT name, which is what a + // consumer dispatches. + let name = t.binary.clone().unwrap_or_else(|| t.name.clone()); + + let mut out = Vec::new(); + if !asset::is_per_platform(&template) { + out.push(PayloadPlan { + name, + repo, + version: t.version.clone(), + asset: asset::expand(&template, &t.version, None, None)?, + platform: None, + kind, + unverified_reason: t.unverified_reason.clone(), + }); + return Ok(out); + } + for p in platforms { + // An explicit name wins over the template. Some upstreams ship only a + // musl Linux build, whose name no template can derive from a gnu + // triple; naming the file is exact where inferring it would guess. + let asset = match t.asset_for.get(*p) { + Some(explicit) => explicit.clone(), + None => asset::expand(&template, &t.version, Some(p), None)?, + }; + out.push(PayloadPlan { + name: name.clone(), + repo: repo.clone(), + version: t.version.clone(), + asset, + platform: Some((*p).to_string()), + kind, + unverified_reason: t.unverified_reason.clone(), + }); + } + Ok(out) +} + +/// Expand one extension entry. +pub fn plan_vsix(v: &ManifestVsix, platforms: &[&str]) -> Result, PlanError> { + let repo = repo_of(&v.repo, &v.name); + let mut out = Vec::new(); + if !asset::is_per_platform(&v.asset) { + out.push(PayloadPlan { + name: v.name.clone(), + repo, + version: v.version.clone(), + asset: asset::expand(&v.asset, &v.version, None, None)?, + platform: None, + kind: PayloadKind::Vsix, + unverified_reason: None, + }); + return Ok(out); + } + for p in platforms { + out.push(PayloadPlan { + name: v.name.clone(), + repo: repo.clone(), + version: v.version.clone(), + asset: asset::expand(&v.asset, &v.version, Some(p), None)?, + platform: Some((*p).to_string()), + kind: PayloadKind::Vsix, + unverified_reason: None, + }); + } + Ok(out) +} + +/// The whole manifest as work items. +pub fn plan(m: &LayerManifest, platforms: &[&str]) -> Result, PlanError> { + let mut out = Vec::new(); + for t in &m.tools { + out.extend(plan_tool(t, platforms)?); + } + for v in &m.vsix { + out.extend(plan_vsix(v, platforms)?); + } + Ok(out) +} + +/// Distinct releases the plan touches, in first-seen order. +/// +/// The ingestion proof is established per RELEASE, not per payload: rivet and +/// spar each appear as a tool and an extension from one release, and verifying +/// twice is what killed the 2026.08.3 deposit. +pub fn releases(plans: &[PayloadPlan]) -> Vec<(String, String)> { + let mut seen: Vec<(String, String)> = Vec::new(); + for p in plans { + let key = (p.repo.clone(), p.version.clone()); + if !seen.contains(&key) { + seen.push(key); + } + } + seen +} + +#[cfg(test)] +mod tests { + use super::*; + use varve_core::layerspec::parse_layer_manifest; + + const PLATFORMS: &[&str] = &["aarch64-apple-darwin", "x86_64-unknown-linux-gnu"]; + + fn manifest(tools: &str) -> LayerManifest { + let text = format!( + "[varve]\nversion = \"v0.29.0\"\n\n[realm]\nname = \"r\"\n\ + channel = \"rolling\"\nregistry = \"oci://x\"\n\n{tools}" + ); + parse_layer_manifest(&text).expect("parses") + } + + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn a_tarball_tool_expands_to_one_item_per_platform() { + let m = manifest("[[tool]]\nname = \"rivet\"\nversion = \"v0.34.0\"\n"); + let p = plan(&m, PLATFORMS).expect("plans"); + assert_eq!(p.len(), 2); + assert_eq!(p[0].asset, "rivet-v0.34.0-aarch64-apple-darwin.tar.gz"); + assert_eq!(p[0].repo, "pulseengine/rivet"); + assert_eq!(p[0].kind, PayloadKind::Tarball); + } + + /// THE thing the shell could not do. `WSC_VERSION` was one variable naming + /// one tool in one repository, so the bytecodealliance realm — which needs + /// wac, wkg and wrpc — could not be assembled at all. Here layout is a + /// property of a tool. + // rivet: verifies REQ-REALM2-002 + #[test] + fn three_raw_per_platform_tools_are_not_a_special_case() { + let m = manifest( + "[[tool]]\nname = \"wac\"\nrepo = \"bytecodealliance/wac\"\n\ + version = \"v0.10.1\"\nlayout = \"raw-per-platform\"\nasset = \"wac-cli-%T\"\n\ + [[tool]]\nname = \"wkg\"\nrepo = \"bytecodealliance/wasm-pkg-tools\"\n\ + version = \"v0.16.1\"\nlayout = \"raw-per-platform\"\nasset = \"wkg-%T\"\n\ + [[tool]]\nname = \"wrpc\"\nrepo = \"bytecodealliance/wrpc\"\n\ + version = \"v0.17.0\"\nlayout = \"raw-per-platform\"\n\ + asset = \"wit-bindgen-wrpc-%T\"\n", + ); + let p = plan(&m, PLATFORMS).expect("plans"); + assert_eq!(p.len(), 6, "three tools x two platforms"); + let names: Vec<&str> = p.iter().map(|x| x.name.as_str()).collect(); + assert!(names.contains(&"wac") && names.contains(&"wkg") && names.contains(&"wrpc")); + assert!(p.iter().all(|x| x.kind == PayloadKind::RawPerPlatform)); + assert_eq!(p[0].asset, "wac-cli-aarch64-apple-darwin"); + } + + /// wac and wrpc ship only a musl Linux build. A static musl binary is the + /// right payload for a gnu platform, but no template can derive + /// `wac-cli-x86_64-unknown-linux-musl` from `x86_64-unknown-linux-gnu` — + /// so the manifest names it. Found by checking a planned manifest against + /// the real releases: four of twelve assets did not exist. + // rivet: verifies REQ-REALM2-002 + #[test] + fn a_platform_whose_asset_name_cannot_be_derived_is_named_explicitly() { + let m = manifest( + "[[tool]]\nname = \"wac\"\nrepo = \"bytecodealliance/wac\"\n\ + version = \"v0.10.1\"\nlayout = \"raw-per-platform\"\n\ + asset = \"wac-cli-%T\"\n\ + [tool.asset-for]\n\ + \"x86_64-unknown-linux-gnu\" = \"wac-cli-x86_64-unknown-linux-musl\"\n", + ); + let p = plan(&m, PLATFORMS).expect("plans"); + let linux = p + .iter() + .find(|x| x.platform.as_deref() == Some("x86_64-unknown-linux-gnu")); + assert_eq!( + linux.map(|x| x.asset.as_str()), + Some("wac-cli-x86_64-unknown-linux-musl") + ); + // The platform with no override still follows the template. + let mac = p + .iter() + .find(|x| x.platform.as_deref() == Some("aarch64-apple-darwin")); + assert_eq!( + mac.map(|x| x.asset.as_str()), + Some("wac-cli-aarch64-apple-darwin") + ); + } + + /// The payload is deposited under the BINARY's name — kiln ships kilnd, + /// and a consumer dispatches `kilnd`. + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn a_differing_binary_name_is_what_the_payload_is_called() { + let m = manifest("[[tool]]\nname = \"kiln\"\nversion = \"v0.4.4\"\nbinary = \"kilnd\"\n"); + let p = plan(&m, PLATFORMS).expect("plans"); + assert!(p.iter().all(|x| x.name == "kilnd")); + // …but the ASSET still follows the tool's own name. + assert!(p[0].asset.starts_with("kiln-v0.4.4-"), "{}", p[0].asset); + } + + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn a_portable_vsix_is_planned_once_and_a_per_platform_one_per_platform() { + let m = manifest( + "[[tool]]\nname = \"t\"\nversion = \"v1\"\n\ + [[vsix]]\nname = \"rivet-sdlc\"\nrepo = \"pulseengine/rivet\"\n\ + version = \"v0.34.0\"\nasset = \"rivet-sdlc-%V.vsix\"\n\ + [[vsix]]\nname = \"spar-aadl\"\nrepo = \"pulseengine/spar\"\n\ + version = \"v0.40.0\"\nasset = \"spar-aadl-%P-%V.vsix\"\n", + ); + let p = plan(&m, PLATFORMS).expect("plans"); + let sdlc: Vec<_> = p.iter().filter(|x| x.name == "rivet-sdlc").collect(); + let aadl: Vec<_> = p.iter().filter(|x| x.name == "spar-aadl").collect(); + assert_eq!(sdlc.len(), 1, "portable package planned once"); + assert_eq!(sdlc[0].platform, None); + assert_eq!(aadl.len(), 2, "per-platform package planned per platform"); + assert_eq!(aadl[0].asset, "spar-aadl-darwin-arm64-0.40.0.vsix"); + } + + /// The opt-in reason travels with every payload of that release, because + /// it is signed beside each of them. + // rivet: verifies REQ-INGEST-001 + #[test] + fn an_unverified_reason_is_carried_onto_every_payload_of_that_release() { + let m = manifest( + "[[tool]]\nname = \"wac\"\nrepo = \"bytecodealliance/wac\"\nversion = \"v0.10.1\"\n\ + layout = \"raw-per-platform\"\nasset = \"wac-cli-%T\"\n\ + unverified-reason = \"publishes nothing verifiable\"\n", + ); + let p = plan(&m, PLATFORMS).expect("plans"); + assert_eq!(p.len(), 2); + assert!( + p.iter() + .all(|x| x.unverified_reason.as_deref() == Some("publishes nothing verifiable")) + ); + } + + /// rivet appears as a tool AND an extension from one release. Verifying + /// that release twice is what killed the 2026.08.3 deposit. + // rivet: verifies REQ-INGEST-001 + #[test] + fn a_release_reached_by_two_payloads_is_listed_once() { + let m = manifest( + "[[tool]]\nname = \"rivet\"\nversion = \"v0.34.0\"\n\ + [[vsix]]\nname = \"rivet-sdlc\"\nrepo = \"pulseengine/rivet\"\n\ + version = \"v0.34.0\"\nasset = \"rivet-sdlc-%V.vsix\"\n", + ); + let p = plan(&m, PLATFORMS).expect("plans"); + let r = releases(&p); + assert_eq!( + r, + vec![("pulseengine/rivet".to_string(), "v0.34.0".to_string())] + ); + } + + /// Distinct versions of one repo are distinct releases — the assembler + /// refuses that case, and it must be able to SEE it first. + // rivet: verifies REQ-INGEST-001 + #[test] + fn one_repo_at_two_versions_is_two_releases() { + let m = manifest( + "[[tool]]\nname = \"rivet\"\nversion = \"v0.34.0\"\n\ + [[vsix]]\nname = \"rivet-sdlc\"\nrepo = \"pulseengine/rivet\"\n\ + version = \"v0.33.1\"\nasset = \"rivet-sdlc-%V.vsix\"\n", + ); + let r = releases(&plan(&m, PLATFORMS).expect("plans")); + assert_eq!(r.len(), 2); + } + + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn an_unknown_layout_is_refused_naming_the_ones_that_work() { + let m = manifest("[[tool]]\nname = \"t\"\nversion = \"v1\"\nlayout = \"zipfile\"\n"); + let err = plan(&m, PLATFORMS).expect_err("refuses"); + let msg = err.to_string(); + assert!( + msg.contains("zipfile") && msg.contains("raw-per-platform"), + "{msg}" + ); + } + + /// A raw-per-platform tool with no explicit template still needs one, and + /// the default has to vary by platform or every platform would fetch the + /// same file. + // rivet: verifies REQ-PRODUCER-002 + #[test] + fn a_raw_tool_without_a_template_still_varies_by_platform() { + let m = manifest( + "[[tool]]\nname = \"thing\"\nversion = \"v1\"\nlayout = \"raw-per-platform\"\n", + ); + let p = plan(&m, PLATFORMS).expect("plans"); + assert_eq!(p.len(), 2); + assert_ne!( + p[0].asset, p[1].asset, + "every platform fetched the same asset" + ); + assert_eq!(p[0].asset, "thing-aarch64-apple-darwin"); + } +} diff --git a/crates/varve/docs/concept-root-ceremony.md b/crates/varve/docs/concept-root-ceremony.md index 11ef8bdd..b548392c 100644 --- a/crates/varve/docs/concept-root-ceremony.md +++ b/crates/varve/docs/concept-root-ceremony.md @@ -21,8 +21,10 @@ will own. Every command here is offline; none of them contacts a registry, an API, or a transparency log (`varve docs air-gap`). So generate on a machine that has never been and never will be connected — a wiped laptop, a live USB image, an -existing air-gapped build host. Two people present, and the transcript recorded -on paper. +existing air-gapped build host. Two people present where you have two people, +and the transcript recorded on paper. If you are one person, see +"When you are one person" below before you read the rest as a checklist you +have failed. ```sh varve keygen --out root.key --pub root.pub @@ -106,9 +108,10 @@ The key is offline media plus paper, and nothing else: * Two or more geographically separate safes, each holding one share (or one full copy under a different control, if you accepted single custody). -* An access log with two-person rule. Every use of the key is a ceremony - entry, because there is no revocation to fall back on if a use was not - authorised. +* An access log. Every use of the key is a ceremony entry, because there is no + revocation to fall back on if a use was not authorised. Add a two-person + rule if you have two people; if you do not, the log is what you have, and it + is worth more when it is written by the machine than by the operator. * A **read test on a schedule** — annually is a reasonable floor. Restore from the medium onto an air-gapped machine and run `varve pubkey`; the failure mode you are looking for is a safe full of unreadable USB sticks, discovered @@ -118,6 +121,40 @@ The key is offline media plus paper, and nothing else: descriptor, never a workspace file — `varve docs ci`, "Getting the key into CI", has the two patterns that avoid writing it to disk at all. +## When you are one person + +Most of this topic assumes an organisation with several people who owe each +other a duty of care. Plenty of realms will not have that, and **varve's own +does not**: the `pulseengine` realm is operated by one person. Saying so is not +a disclaimer, it is the point — a document that prescribes controls its own +author cannot staff is a document an assessor will use to discount everything +else in it. + +So, honestly, for a single operator: + +* **The two-person rule is unavailable, and a fake one is worse than none.** + A required-reviewer gate you approve yourself is a control in name only, and + it will read as an attempt to appear compliant. Do not configure one. State + that the realm is single-operator instead. +* **Trade prevention for detection.** Prevention is what a second person buys. + Detection you can have alone: a wait timer before a signing job runs so there + is a window to cancel it, an environment restricted to the one ref a deposit + may run from, and a durable record of what was signed and when — written by + the pipeline, not by you. An unauthorised signature you can discover + afterwards is enormously better than one you cannot. +* **Split custody is still available, and matters more, not less.** M-of-N + shares do not need colleagues; they need parties with continuity. Prefer at + least one holder that is an institution — a bank deposit box, a notary — + because an institution survives you moving house, changing jobs, or dying, + and a friend's desk drawer does not. +* **Write down what happens if you stop.** The likeliest end of a + single-operator realm is not compromise, it is the operator stopping. A + realm whose root is unreachable while consumers still pin it is a + supply-chain hazard that decays quietly. Decide now, in writing, what + happens: who inherits, and failing that, that a final layer is published + declaring end-of-life so consumers know to unpin. That decision costs + nothing today and is unmakeable later. + ## 5. Use it as rarely as possible Signing is the only thing the key is for, and each of those commands is a diff --git a/crates/varve/src/inspect.rs b/crates/varve/src/inspect.rs index 241dba67..a16fada9 100644 --- a/crates/varve/src/inspect.rs +++ b/crates/varve/src/inspect.rs @@ -157,7 +157,10 @@ fn store_of(l: &crate::ComposedLayer) -> &Store { /// ```text /// { /// "command": "inspect", -/// "layer", "channel", "manifest_digest", the layer the pin resolved to +/// "layer", "channel", "realm", "manifest_digest", the layer the pin +/// resolved to, and the realm that +/// vouched for it (a layer id is +/// unique only within a realm) /// "host_platform", what `present` was decided against /// "composition": [ {"layer","manifest_digest","realm","root"} ], /// "payloads": [ {"name","version","kind","known_kind","platform", @@ -213,6 +216,13 @@ fn print_json( "command": "inspect", "layer": target.entry.layer.to_string(), "channel": target.entry.channel, + // A layer id is YYYY.MM.P and is only unique WITHIN a realm; a + // consumer diffing two inspect reports needs to know which realm each + // came from. It was in `composition[]` and absent from the top level. + "realm": layers + .iter() + .find(|l| l.entry.digest == target.entry.digest) + .map(|l| l.realm.clone()), "manifest_digest": target.entry.digest, "host_platform": host, "composition": composition, @@ -243,10 +253,26 @@ fn print_text( rows: &[Row], host: &str, ) { - println!( - "layer {} ({}) {}", - target.entry.layer, target.entry.channel, target.entry.digest - ); + // The realm belongs on this line, not only in the composition block below. + // Layer ids are YYYY.MM.P, so two realms can publish the same one -- + // "layer 2026.08.26" alone does not say WHICH 2026.08.26, and the realm is + // what the trust root, the store partition and the pin all hang on. + // Reported from a real session: a user read this output beside a + // `realm = "linc"` pin and the word realm appeared nowhere in it. + let realm = layers + .iter() + .find(|l| l.entry.digest == target.entry.digest) + .map(|l| l.realm.as_str()); + match realm { + Some(r) => println!( + "layer {} ({}) realm '{}' {}", + target.entry.layer, target.entry.channel, r, target.entry.digest + ), + None => println!( + "layer {} ({}) {}", + target.entry.layer, target.entry.channel, target.entry.digest + ), + } if layers.len() > 1 { println!("composition: {} layers —", layers.len()); for l in layers { diff --git a/crates/varve/src/main.rs b/crates/varve/src/main.rs index 2bce35da..b2cdf09b 100644 --- a/crates/varve/src/main.rs +++ b/crates/varve/src/main.rs @@ -1574,6 +1574,10 @@ fn run_tool( args: &[std::ffi::OsString], ) -> anyhow::Result<()> { let ctx = project_ctx(store)?; + // Kept before the move: a refusal below names the pin that caused it, and + // the realm is what makes that sentence answer the user's real question. + let pin_file = ctx.root.join("varve.toml"); + let pinned_realm = ctx.pin.realm.clone(); let mut pin = ctx.pin; if let Some(layer) = override_layer { // A one-off: resolve another layer for this invocation only. The @@ -1599,11 +1603,26 @@ fn run_tool( }, ); } + // Name the PIN, not just the layer. A shim directory serves every + // realm (one `cd` switches toolchains), so the honest question when a + // familiar tool stops working is not "what is in this layer" but "why + // is this directory pinned to that layer" — and the answer is a file + // the user can open. Reported from a real session: a `rivet` shim + // refused inside a project pinned to an unrelated realm, and the + // message named neither the realm nor the varve.toml that chose it. bail!( - "tool '{tool}' is not part of layer {} — it exposes: {}. `varve inspect` lists \ - every payload, dispatched and held.", + "tool '{tool}' is not part of layer {} — it exposes: {}.\n\ + This directory is pinned by {}{}, which is why '{tool}' does not \ + resolve here even though a shim for it exists (one shim directory \ + serves every realm). `varve inspect` lists every payload in the \ + pinned layer; `varve docs pins` explains switching.", resolved.layer.layer, - addressable(&resolved) + addressable(&resolved), + pin_file.display(), + match pinned_realm.as_deref() { + Some(r) => format!(" to realm '{r}'"), + None => String::new(), + } ); }; // Runnered entries (portable wasm) execute through their runner — from @@ -3954,24 +3973,36 @@ fn which(store: &Store, tool: &str) -> anyhow::Result<()> { addressable(&resolved) ); }; - // STDOUT is the dispatched path, unchanged, so scripts that capture it - // keep working (REQ-SHADOW-001 clause 2). + // STDOUT is the path and NOTHING else, because that is what a caller + // captures: `M=$(varve which synth)` must yield something executable + // (REQ-WHICHSTDOUT-001). + // + // This used to print the provenance to stdout too, under a comment + // claiming scripts kept working and another saying "the first two lines + // are what scripts capture". A script captures ALL of them. A consumer's + // build script took the two-line value, found it was not a binary, fell + // through to whatever was on PATH, and died naming a version nobody had + // pinned (#102) — the mixed-toolchain failure varve exists to close, + // caused by the command whose job is closing it. + // + // Provenance moves to stderr: still in front of a human at a terminal, + // out of the way of command substitution. println!("{}", path.display()); - println!( + eprintln!( "layer {} ({}) {}", resolved.layer.layer, resolved.layer.channel, resolved.layer.digest ); // The line above names the layer the PIN resolves to, which for a composed // tool is not the layer that owns the binary (`docs composition` says so). // Someone who ASKED qualified is asking precisely about a provider, so name - // it — a third line, because the first two are what scripts capture. + // it — also on stderr, for the same reason. if asked.contains('/') && let Some((provider, _)) = resolved .qualified .iter() .find(|(p, _)| p.qualified().as_deref() == Some(asked.as_str())) { - println!( + eprintln!( "provided by realm '{}' layer {} {}", provider.realm, provider.layer, provider.digest ); diff --git a/crates/varve/tests/cli.rs b/crates/varve/tests/cli.rs index 64df87e6..e855b558 100644 --- a/crates/varve/tests/cli.rs +++ b/crates/varve/tests/cli.rs @@ -148,7 +148,56 @@ fn verify_fails_when_path_runs_a_different_binary_than_the_pin() { .success(); } +/// The shell contract, which is the whole point of the command: what a caller +/// captures must BE the path. `predicate::str::contains` cannot see this — it +/// passes just as happily on a two-line stdout, which is how the defect +/// survived a test named for the behaviour it broke. A consumer's build script +/// captured this value, got a non-executable string, fell through to an ambient +/// binary, and died naming the wrong version (#102). +// rivet: verifies REQ-WHICHSTDOUT-001 +#[test] +fn what_a_script_captures_from_which_is_exactly_the_path() { + let fx = fixture(Some(PIN_JULY), &[(MANIFEST_JULY, &[("synth", b"s")])]); + let out = varve(&fx).args(["which", "synth"]).assert().success(); + let stdout = String::from_utf8_lossy(&out.get_output().stdout).to_string(); + + // Exactly what `M=$(varve which synth)` yields, after the shell strips + // trailing newlines. + let captured = stdout.trim_end_matches('\n'); + assert!( + !captured.contains('\n'), + "stdout is {} lines; a script capturing it gets a string that is not a \ + path:\n{stdout}", + captured.lines().count() + ); + assert!( + std::path::Path::new(captured).is_absolute(), + "captured value is not an absolute path: {captured:?}" + ); + assert!( + captured.ends_with("bin/synth"), + "captured value is not the tool's path: {captured:?}" + ); + + // The provenance is not lost — it moves to stderr, where a human at a + // terminal still reads it and command substitution does not. + let stderr = String::from_utf8_lossy(&out.get_output().stderr).to_string(); + assert!( + stderr.contains("2026.07.0"), + "layer id missing from stderr: {stderr}" + ); + assert!( + stderr.contains("sha256:"), + "digest missing from stderr: {stderr}" + ); +} + +/// The human still gets everything — the path to act on and the layer it came +/// from — but on the two streams that mean different things. This test used to +/// assert BOTH on stdout, which is how it passed while the command returned a +/// value no script could use. // rivet: verifies REQ-PIN-001 +// rivet: verifies REQ-WHICHSTDOUT-001 #[test] fn which_prints_the_resolved_binary_and_its_layer() { let fx = fixture(Some(PIN_JULY), &[(MANIFEST_JULY, &[("synth", b"s")])]); @@ -156,11 +205,9 @@ fn which_prints_the_resolved_binary_and_its_layer() { .args(["which", "synth"]) .assert() .success() - .stdout( - predicate::str::contains("bin/synth") - .and(predicate::str::contains("2026.07.0")) - .and(predicate::str::contains("sha256:")), - ); + .stdout(predicate::str::contains("bin/synth")) + .stdout(predicate::str::contains("2026.07.0").not()) + .stderr(predicate::str::contains("2026.07.0").and(predicate::str::contains("sha256:"))); } // rivet: verifies REQ-PIN-001 @@ -6366,6 +6413,44 @@ fn inspect_reports_name_version_kind_and_platform_for_every_payload() { ); } +/// A layer identifier is YYYY.MM.P and is unique only WITHIN a realm — two +/// realms can each publish 2026.08.26. varve is built for that world, and +/// `inspect` said layer, channel, digest, platform and never the realm. A user +/// read that output beside a `realm = "linc"` pin and observed, correctly, that +/// the word appeared nowhere in it. +// rivet: verifies REQ-NAMETHEREALM-001 +#[test] +fn inspect_names_the_realm_because_a_layer_id_alone_does_not() { + let fx = fixture(Some(PIN_JULY), &[]); + let trust = inspectable_composition(&fx); + + let assert = varve(&fx) + .env("VARVE_TRUST_ROOT", &trust) + .arg("inspect") + .assert() + .success(); + let stdout = String::from_utf8(assert.get_output().stdout.clone()).unwrap(); + let header = stdout.lines().next().unwrap_or_default(); + assert!( + header.contains("realm"), + "the header identifies a layer without saying which realm it belongs \ + to: {header}" + ); + + // And a pipeline gets it as a field rather than by scraping the header. + let assert = varve(&fx) + .env("VARVE_TRUST_ROOT", &trust) + .args(["inspect", "--json"]) + .assert() + .success(); + let doc: serde_json::Value = + serde_json::from_slice(&assert.get_output().stdout).expect("inspect --json parses"); + assert!( + doc.get("realm").and_then(|v| v.as_str()).is_some(), + "inspect --json carries no top-level realm: {doc}" + ); +} + // rivet: verifies REQ-INSPECT-001 #[test] fn inspect_json_is_the_shape_a_pipeline_was_promised() { @@ -6776,15 +6861,13 @@ fn the_unselected_layer_stays_installed_verified_and_addressable() { .args(["which", "pulseengine/wasm-tools"]) .assert() .success() - .stdout( - predicate::str::contains("/bin/wasm-tools") - .and(predicate::str::contains("provided by realm 'pulseengine'")), - ); + .stdout(predicate::str::contains("/bin/wasm-tools")) + .stderr(predicate::str::contains("provided by realm 'pulseengine'")); in_realm_project(&fx, &project) .args(["which", "bytecodealliance/wasm-tools"]) .assert() .success() - .stdout(predicate::str::contains( + .stderr(predicate::str::contains( "provided by realm 'bytecodealliance' layer 2026.08.0", )); } diff --git a/install.sh b/install.sh index 02effa6a..06aabdf2 100755 --- a/install.sh +++ b/install.sh @@ -222,13 +222,25 @@ tar -xzf "${WORK}/${ARCHIVE}" -C "${WORK}/unpack" [ -f "${WORK}/unpack/varve" ] || die "the archive does not contain a 'varve' binary." mkdir -p "$INSTALL_DIR" +# What was here before, so the install can say what it REPLACED rather than +# implying the destination was empty (REQ-INSTALLSHADOW-001 clause 1). +previous="" +if [ -x "${INSTALL_DIR}/varve" ]; then + previous="$("${INSTALL_DIR}/varve" --version 2>/dev/null || echo "an unreadable binary")" +fi cp "${WORK}/unpack/varve" "${INSTALL_DIR}/varve.new" chmod 0755 "${INSTALL_DIR}/varve.new" mv "${INSTALL_DIR}/varve.new" "${INSTALL_DIR}/varve" installed="$("${INSTALL_DIR}/varve" --version)" \ || die "the installed binary does not run: ${INSTALL_DIR}/varve" -step "Installed ${installed} to ${INSTALL_DIR}/varve" +if [ -n "$previous" ] && [ "$previous" != "$installed" ]; then + step "Installed ${installed} to ${INSTALL_DIR}/varve (replaced ${previous})" +elif [ -n "$previous" ]; then + step "Reinstalled ${installed} to ${INSTALL_DIR}/varve (unchanged)" +else + step "Installed ${installed} to ${INSTALL_DIR}/varve" +fi # ── What the user has, and what they do not ─────────────────────────────── say "" @@ -247,9 +259,37 @@ if [ "$SIGNATURE_CHECKED" = no ]; then say "" fi +# Which varve will the user's shell ACTUALLY run? Being on PATH is not the +# same as being first on it, and "already on PATH" was a reassuring way to say +# nothing. varve refuses to claim success when PATH shadows a pinned tool +# (REQ-SHADOW-001); the same reasoning has to apply to varve itself, or a +# bootstrap can leave you running a build it did not install +# (REQ-INSTALLSHADOW-001 clause 2). +# +# Resolved the way the shell resolves it, rather than by guessing at locations. +winner="$(command -v varve 2>/dev/null || true)" +if [ -n "$winner" ] && [ "$winner" != "${INSTALL_DIR}/varve" ]; then + winner_version="$("$winner" --version 2>/dev/null || echo "unknown version")" + say "" + say "WARNING: \`varve\` on your PATH is NOT the binary just installed." + say "" + say " your shell runs ${winner} (${winner_version})" + say " this installed ${INSTALL_DIR}/varve (${installed})" + say "" + say "Everything below, and every command in the docs, will use the first one." + say "To make this install win, put its directory FIRST on PATH:" + say "" + say " export PATH=\"${INSTALL_DIR}:\$PATH\"" + say "" + say "…or remove the other one. If it came from cargo: cargo uninstall varve" + say "" +fi + case ":${PATH}:" in *":${INSTALL_DIR}:"*) - say "${INSTALL_DIR} is already on PATH." + if [ -z "$winner" ] || [ "$winner" = "${INSTALL_DIR}/varve" ]; then + say "${INSTALL_DIR} is on PATH, and is what \`varve\` runs." + fi ;; *) say "Add it to PATH:" diff --git a/tools/no-key-on-disk.sh b/tools/no-key-on-disk.sh new file mode 100755 index 00000000..ef2da414 --- /dev/null +++ b/tools/no-key-on-disk.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# REQ-NOKEYDISK-001: no varve-operated workflow may write key material to a file. +# +# `docs ci` tells adopters that "every adopter therefore invents +# `echo "$SECRET" > key.tmp`, which leaves the realm's one secret on disk", and +# `docs root-ceremony` says the key must reach varve "through a file +# descriptor, never a workspace file". varve's own deposit workflow wrote it to +# a predictable /tmp path on a shared runner for every layer it ever published. +# +# An assessor found that by reading the repository rather than the +# documentation, and was right that it is the finding which invalidates every +# other procedural claim by induction: a published procedure the publisher does +# not follow is not evidence of anything. +# +# So the rule is mechanical from here. This refuses a redirect of anything +# key-shaped into a file. It is deliberately about the SHAPE of the line, not +# about whether a human reviewer noticed. +# +# Usage: tools/no-key-on-disk.sh [dir] (default .github/workflows) +set -euo pipefail + +# --self-test: prove this gate can go RED before trusting it green. +# +# A gate admitted without a proof that it can fail is not a gate, and this +# repository has found that class of defect repeatedly -- including in this very +# script, which on its first run flagged its own documentation comment. The +# controls below include the EXACT line this repository shipped for every layer +# it published, so a future refactor that guts the pattern fails here rather +# than silently allowing the thing back. +# rivet: verifies REQ-NOKEYDISK-001 +if [ "${1:-}" = "--self-test" ]; then + work="$(mktemp -d)"; trap 'rm -rf "$work"' EXIT + mkdir -p "$work/wf" + fail=0 + must_reject() { # name, line + printf 'jobs:\n x:\n steps:\n - run: %s\n' "$2" > "$work/wf/a.yml" + if "$0" "$work/wf" >/dev/null 2>&1; then + echo "::error::self-test: gate ACCEPTED what it must reject ($1): $2"; fail=1 + else + echo " rejects: $1" + fi + } + must_accept() { # name, line + printf 'jobs:\n x:\n steps:\n - run: %s\n' "$2" > "$work/wf/a.yml" + if "$0" "$work/wf" >/dev/null 2>&1; then + echo " accepts: $1" + else + echo "::error::self-test: gate REJECTED what it must accept ($1): $2"; fail=1 + fi + } + # The line this repository actually shipped, verbatim. + must_reject "the pattern varve shipped" "printf '%s' \"\$VARVE_ROLLING_KEY\" > /tmp/rolling.key" + must_reject "the adopter mistake docs name" 'echo "$SIGNING_SECRET" > key.tmp' + must_reject "append" 'printf %s "$VARVE_ROOT_KEY" >> /tmp/k' + must_reject "tee" 'printf %s "$MY_TOKEN" | tee /tmp/t' + must_accept "the documented fd form" "varve deposit --key <(printf '%s' \"\$VARVE_ROLLING_KEY\") --out o" + must_accept "a pipe to /dev/stdin" "printf '%s' \"\$K\" | varve sign-status --key /dev/stdin" + [ "$fail" -eq 0 ] || { echo "::error::no-key-on-disk self-test FAILED"; exit 1; } + echo "no-key-on-disk: self-test OK — the gate rejects 4 shapes and accepts 2" + exit 0 +fi + +DIR="${1:-.github/workflows}" + +# A secret-looking variable redirected into a file. Covers `> f`, `>> f`, and +# `tee f`, with or without quotes around the variable. +PATTERN='(\$\{?[A-Za-z_]*(KEY|SECRET|TOKEN)[A-Za-z_]*\}?"?[[:space:]]*(>>?|\|[[:space:]]*tee)[[:space:]])|((>>?|\|[[:space:]]*tee)[[:space:]]*[^[:space:]|]*(key|secret)[^[:space:]|]*$)' + +found=0 +while IFS= read -r hit; do + [ -n "$hit" ] || continue + echo "::error::$hit" + found=1 +# Comments are stripped BEFORE matching, not after: grep -rIn prefixes each hit +# with `file:line:`, so a naive `grep -v '^#'` never sees a commented line -- +# which this script proved on its first run by flagging its own documentation. +done < <(grep -rInE "$PATTERN" "$DIR" 2>/dev/null \ + | awk -F: '{ rest = substr($0, index($0, $3)); sub(/^[[:space:]]+/, "", rest); + if (rest !~ /^#/) print }' || true) + +if [ "$found" -ne 0 ]; then + cat >&2 <<'WHY' + +REQ-NOKEYDISK-001: a signing key must not be written to a filesystem path. +Use the file-descriptor forms `varve docs ci` documents: + + varve deposit --key <(printf '%s' "$VARVE_SIGNING_KEY") ... + printf '%s' "$VARVE_SIGNING_KEY" | varve sign-status --key /dev/stdin ... + +A short-lived `mktemp` file is acceptable for a THROWAWAY key and for nothing +that a realm's consumers pin. +WHY + exit 1 +fi +echo "no-key-on-disk: OK — no workflow writes key material to a file" diff --git a/tools/systest/install-shadow.sh b/tools/systest/install-shadow.sh new file mode 100755 index 00000000..9642d10f --- /dev/null +++ b/tools/systest/install-shadow.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# REQ-INSTALLSHADOW-001: the installer must say which varve will actually run. +# +# A maintainer followed the documented install and ended up running a different +# binary than the one it installed: ~/.varve/bin/varve at 0.25.0 from the +# installer, ~/.cargo/bin/varve at 0.29.0 from cargo, and plain `varve` +# resolving to the cargo one. The installer said "Installed …" and even +# "$INSTALL_DIR is already on PATH" -- true, and useless, because being on PATH +# is not being FIRST on it. +# +# varve refuses to claim success when PATH shadows a pinned tool +# (REQ-SHADOW-001). This gate holds varve's own installer to that standard. +# +# It runs install.sh's real decision logic against a fabricated PATH; the +# download and signature steps are not what is under test here. +set -euo pipefail +HERE="$(cd "$(dirname "$0")/../.." && pwd)" +fail() { echo "FAIL: $*" >&2; exit 1; } + +WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/installed/bin" "$WORK/other/bin" +printf '#!/bin/sh\necho "varve 9.9.9"\n' > "$WORK/installed/bin/varve" +printf '#!/bin/sh\necho "varve 0.25.0"\n' > "$WORK/other/bin/varve" +chmod +x "$WORK/installed/bin/varve" "$WORK/other/bin/varve" + +# The decision under test, lifted verbatim in shape from install.sh. +probe() { # PATH-value -> prints, exit 7 when it warns + INSTALL_DIR="$WORK/installed/bin" installed="varve 9.9.9" \ + PATH="$1" bash -c ' + winner="$(command -v varve 2>/dev/null || true)" + if [ -n "$winner" ] && [ "$winner" != "${INSTALL_DIR}/varve" ]; then + echo "WARNING: varve on PATH is not the binary just installed: $winner" + exit 7 + fi + echo "ok: the install wins" + ' +} + +# rivet: verifies REQ-INSTALLSHADOW-001 +echo "== the warning must FIRE when another varve wins PATH" +if probe "$WORK/other/bin:$WORK/installed/bin:/usr/bin:/bin" >/dev/null 2>&1; then + fail "a shadowing varve did not produce a warning — the exact defect this gate exists for" +fi +echo " fired as required" + +echo "== and must NOT fire when the install wins" +probe "$WORK/installed/bin:$WORK/other/bin:/usr/bin:/bin" >/dev/null 2>&1 \ + || fail "warned when the installed binary is the one that runs (false alarm)" +echo " quiet as required" + +echo "== nor when nothing else is on PATH" +probe "$WORK/installed/bin:/usr/bin:/bin" >/dev/null 2>&1 \ + || fail "warned with no competing varve on PATH" +echo " quiet as required" + +echo "== install.sh actually carries the check" +grep -q 'command -v varve' "$HERE/install.sh" \ + || fail "install.sh no longer resolves varve the way the shell does" +grep -q 'is NOT the binary just installed' "$HERE/install.sh" \ + || fail "install.sh no longer warns about a shadowing varve" +grep -q 'replaced' "$HERE/install.sh" \ + || fail "install.sh no longer reports what it replaced" +echo " present" + +echo "install-shadow systest: PASS — the installer names the winner, and the check can fail"