diff --git a/issues/build/every-build-still-removes-an-llvm-no-build-makes-in-tree-any-more.md b/issues/build/every-build-still-removes-an-llvm-no-build-makes-in-tree-any-more.md new file mode 100644 index 00000000000..8fc7f04ffff --- /dev/null +++ b/issues/build/every-build-still-removes-an-llvm-no-build-makes-in-tree-any-more.md @@ -0,0 +1,24 @@ +--- +status: open +kind: tooling +opened: 2026-09-29 +--- + +# Every build still removes an in-tree LLVM that no build makes any more + +`llvm::retire_in_tree` removes a bootstrap build directory's own `/llvm`, +`/lld`, `/ci-llvm` and `cache/llvm-*`. It runs at four sites on +every build: the primary's `reassemble`, `compiler::place`, `compiler::choose` +on the way back to the primary's compiler, and the std build +(`sysroot::prepare_std_build`). + +Since the LLVM store, no build makes any of these. Every compiler build links +the store's `llvm-config`, and the std build sets `download-ci-llvm = false`. +What the four sites remove is what a build directory kept from before the +store. After a checkout's first build at that code, they remove nothing, but +they keep asking, on every build, a `read_dir` of `cache/` and a `stat` of +each name. + +Exit: delete `retire_in_tree`, `in_tree` and the four call sites once no +checkout that builds holds a build directory made before the store. A host +cannot know that for any other host, so the owner sets the date. diff --git a/issues/build/lld-beside-an-external-llvm-config-is-carried-without-an-upstream-pr.md b/issues/build/lld-beside-an-external-llvm-config-is-carried-without-an-upstream-pr.md new file mode 100644 index 00000000000..64c83dfe567 --- /dev/null +++ b/issues/build/lld-beside-an-external-llvm-config-is-carried-without-an-upstream-pr.md @@ -0,0 +1,22 @@ +--- +status: open +kind: tooling +opened: 2026-09-29 +--- + +# The fork's bootstrap takes LLD from beside an external `llvm-config`, and no upstream pull request numbers it + +`ToyOSOrg/rust` commit `d9f8a5ca8db` ("bootstrap: take LLD from beside an +external llvm-config that is Rust's LLVM") accepts `rust.lld = true` with an +external `llvm-config` for the host when that `llvm-config` has +`llvm-has-rust-patches = true` and an `lld` sits in its `bin/`, and takes that +`lld` as `rust-lld`; before it, bootstrap refused the combination. Every +compiler build ToyOS runs relies on it (`src/llvm.rs`). + +It is a configuration change, which upstream records in +`src/bootstrap/src/utils/change_tracker.rs` under the number of the pull +request that makes it. No such pull request exists, because none is sent for +now, so the fork carries no entry (`src/forkcheck.rs`). + +Exit: an upstream pull request carries the change with its `change_tracker` +entry, and the fork takes it from there. diff --git a/issues/build/python-and-cc-are-declared.md b/issues/build/python-and-cc-are-declared.md index 32923a71fba..14274af8d1f 100644 --- a/issues/build/python-and-cc-are-declared.md +++ b/issues/build/python-and-cc-are-declared.md @@ -59,9 +59,10 @@ Python search list (`src/main.rs:18`, `:28`). The expensive half is untouched. **CMake and Ninja joined the list on 2026-09-27, by the owner's ruling**: rustc's LLVM is built from source, from `ToyOSOrg/llvm-project`, with clang beside it, and rustc's bootstrap builds LLVM with CMake driving Ninja. Both are declared -where the others are — `ALSO_USED` in `src/main.rs`, because a build whose -LLVM commit this host has already built runs neither — and in the README's -Prerequisites. On macOS they come from Homebrew (`brew install cmake ninja`), +where the others are — CMake in `REQUIRED` in `src/main.rs`, because every +build keys the host's LLVM on `cmake --version` (`src/llvm.rs`), and Ninja in +`ALSO_USED`, because a build whose LLVM this host has already built runs it +not — and in the README's Prerequisites. On macOS they come from Homebrew (`brew install cmake ninja`), with no workaround; on the nightly's toolchain runner at the versions `.github/workflows/nightly.yml` pins from Ubuntu 24.04's archive (`cmake 3.28.3-1build7`, `ninja-build 1.11.1-2`); the two portability jobs install diff --git a/rust b/rust index 1b236638a90..9c3eea441d8 160000 --- a/rust +++ b/rust @@ -1 +1 @@ -Subproject commit 1b236638a905cf078618b5739e9d872b69ecc6e8 +Subproject commit 9c3eea441d8eefb4211a4e88a1a8958e7fff2c18 diff --git a/src/buildlock.rs b/src/buildlock.rs index 2b63a4c296a..380b9e1b326 100644 --- a/src/buildlock.rs +++ b/src/buildlock.rs @@ -37,9 +37,11 @@ //! //! **The lock order is a constraint, not a preference:** a compiler //! key's lock → a sysroot key's lock → the worktree build lock → the global one -//! → artifact. A key's lock is taken with the worktree lock put down -//! ([`Held::without_shared`]), because the key's builder takes the worktree lock -//! exclusively. +//! → an LLVM key's lock → artifact. A compiler's or a sysroot's key lock is +//! taken with the worktree lock put down ([`Held::without_shared`]), because the +//! key's builder takes the worktree lock exclusively; an LLVM key's is taken +//! inside the worktree or global lock covering the fork build directory its +//! builder writes. //! //! Holder death: `flock` is released by the kernel when the open file //! description closes, so a builder that is SIGKILLed mid-phase — routine here @@ -239,12 +241,14 @@ fn integration_path(root: &Path) -> PathBuf { git_lock_dir(root).join("integration") } -/// A content-addressed product of the host, locked per key: a sysroot, or a -/// compiler a worktree's fork checkout names (`src/compiler.rs`). +/// A content-addressed product of the host, locked per key: a sysroot, a +/// compiler a worktree's fork checkout names (`src/compiler.rs`), or the LLVM +/// a compiler links (`src/llvm.rs`). #[derive(Clone, Copy)] pub enum Keyed { Sysroot, Compiler, + Llvm, } impl Keyed { @@ -252,13 +256,15 @@ impl Keyed { match self { Keyed::Sysroot => "sysroots", Keyed::Compiler => "compilers", + Keyed::Llvm => "llvm", } } - fn name(self) -> &'static str { + pub(crate) fn name(self) -> &'static str { match self { Keyed::Sysroot => "sysroot", Keyed::Compiler => "compiler", + Keyed::Llvm => "LLVM", } } } @@ -673,7 +679,7 @@ pub(crate) mod tests { } /// This test binary, to run the one `#[ignore]`d test `test` names. - fn rerun(test: &str) -> Command { + pub(crate) fn rerun(test: &str) -> Command { let mut rerun = Command::new(std::env::current_exe().unwrap()); rerun.args(["--exact", test, "--include-ignored", "--nocapture"]); rerun diff --git a/src/clang.rs b/src/clang.rs index 188d0938a3e..8473447f5e3 100644 --- a/src/clang.rs +++ b/src/clang.rs @@ -149,9 +149,9 @@ fn resource_version(llvm: &Path) -> PathBuf { } } -/// Give `stage2` the C toolchain of the LLVM bootstrap installed beside it. -pub(crate) fn provision(stage2: &Path) { - let llvm = &stage2.parent().expect("stage2 is under a build directory").join("llvm"); +/// Give `stage2` the C toolchain of the LLVM at `llvm` (`src/llvm.rs`), the one +/// its rustc links. +pub(crate) fn provision(stage2: &Path, llvm: &Path) { let bin = bin(stage2); let from = llvm.join("bin/clang"); let to = bin.join("clang"); @@ -208,7 +208,7 @@ mod tests { let said = refused.downcast_ref::().expect("a formatted refusal"); assert!(said.contains("clang") && said.contains("ld.lld") && said.contains("include"), "{said}"); - provision(&stage2); + provision(&stage2, &llvm); assert_eq!(defect(&stage2), None); assert_eq!(fs::read_to_string(bin(&stage2).join("clang")).unwrap(), "the clang"); assert!(!fs::symlink_metadata(bin(&stage2).join("clang")).unwrap().file_type().is_symlink(), "clang is a copy, not the link"); @@ -221,7 +221,7 @@ mod tests { // Provisioning again, from another LLVM, replaces what was there. fs::remove_dir_all(llvm.join("lib/clang/22")).unwrap(); write(&llvm.join("lib/clang/23/include/stddef.h"), "v23"); - provision(&stage2); + provision(&stage2, &llvm); assert!(!resource_parent(&stage2).join("22").exists(), "the old headers stayed beside the new"); assert_eq!(fs::read_to_string(resource_parent(&stage2).join("23/include/stddef.h")).unwrap(), "v23"); diff --git a/src/compiler.rs b/src/compiler.rs index e9a53c5c41e..8586f0e566c 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -9,27 +9,31 @@ //! built by bootstrap in its own fork checkout, under that checkout's //! `build/toyos-compiler/`, and placed at `rust/build/compilers//`, where //! the key ([`key`]) is the identity (`src/identity.rs`) of the checkout's -//! `compiler/`, `src/bootstrap/`, `src/tools/`, `src/stage0` and `Cargo.lock`, -//! the `src/llvm-project` commit, and [`RECIPE`]. Nothing writes that directory after its [`SOURCE`] file exists, -//! and two worktrees naming the same compiler share one copy. +//! `compiler/`, `src/tools/`, `src/stage0` and `Cargo.lock`, the key of the +//! LLVM it links, which names `src/bootstrap`, and [`RECIPE`]. Nothing writes +//! that directory after its [`SOURCE`] file exists, and two worktrees naming +//! the same compiler share one copy. //! -//! **LLVM is built from `src/llvm-project`.** -//! [`source`] names that commit, so another LLVM is another compiler, and an -//! LLVM checkout holding what no commit does names none. +//! **LLVM is the host's, built from `src/llvm-project`** (`src/llvm.rs`), and +//! linked through its `llvm-config`. [`source`] names that LLVM's key, so +//! another LLVM is another compiler, the primary's among them, and an LLVM +//! checkout or a `src/bootstrap` holding what no commit does names none. A +//! worktree records the LLVM its compiler links for as long as it builds with +//! that compiler. //! //! **A compiler of a worktree's own never touches what the others build with**: //! not the primary's `stage2`, not its record, not the machine-global rustup //! `toyos` link — a sysroot is named by its directory, never by a toolchain //! name, so no link is made. The global lock is not taken either; nothing of -//! the primary's is read. +//! the primary's is read but the LLVM store, under its key's own lock. //! //! Locks, in the one order every acquirer takes them: the key's //! (`buildlock::keyed_*` with [`Keyed::Compiler`]), with this worktree's build //! lock put down, held shared for as long as a sysroot is being made from it; //! then, to build, this worktree's exclusively, because its fork build -//! directory is written. +//! directory is written; then the LLVM key's, held shared while it is linked. //! -//! A compiler no worktree names any more is removed by [`sweep`], which +//! A compiler no worktree names any more is removed by `keystore::sweep`, which //! `--worktree remove` and every placement run: each build records the key it used in its //! worktree's `target/`, and a key no registered worktree records, that nobody //! is making or using, goes. @@ -43,28 +47,28 @@ use std::fs; use std::path::{Path, PathBuf}; use crate::buildlock::{self, Guard, Held, Keyed}; +use crate::keystore; use crate::sysroot::{clone_tree, git_bytes, git_out, short, tree_identity}; use crate::toolchain::{self, host_triple}; /// What changes how a key's sources become a compiler and is none of them: the /// build below. Moving it moves every key. -const RECIPE: &str = "bootstrap stage 2 of compiler/rustc and library, profile compiler, host only, with rust-lld, host linker pinned, LLVM and clang from src/llvm-project; 4"; +const RECIPE: &str = "bootstrap stage 2 of compiler/rustc and library, profile compiler, host only, with rust-lld, host linker pinned, LLVM, clang and LLD from the host's LLVM; 5"; /// What a compiler's key is the identity of, in its fork checkout. -const KEYED: [&str; 5] = ["compiler", "src/bootstrap", "src/tools", "src/stage0", "Cargo.lock"]; +const KEYED: [&str; 4] = ["compiler", "src/tools", "src/stage0", "Cargo.lock"]; /// The submodule a compiler is built against by commit: its LLVM, which /// bootstrap builds from that commit, so its content is never read. -const LLVM: &str = "src/llvm-project"; +pub(crate) const LLVM: &str = "src/llvm-project"; + +/// Where a fork checkout builds a compiler of its own. +const BUILD_DIR: &str = "build/toyos-compiler"; /// The file a finished compiler carries last, naming what it was built from. A /// directory without it is a build that did not finish. const SOURCE: &str = "SOURCE"; -/// Where each build records the key of the compiler of its own it used, for -/// [`sweep`]. Absent while a worktree builds with the primary's. -const RECORD: &str = "target/toyos-compiler-key"; - /// A compiler, held in use for as long as this lives. pub struct Compiler { /// Its toolchain directory: `bin/rustc`, `lib/`. @@ -120,9 +124,14 @@ pub fn compilers_dir(rust_dir: &Path) -> PathBuf { rust_dir.join("build/compilers") } -/// [`compiler_source`] and the LLVM commit it links. +/// [`compiler_source`] and the key of the LLVM it links. pub fn source(checkout: &Path) -> String { - format!("{} llvm {}", compiler_source(checkout), llvm_commit(checkout)) + source_with(checkout, &crate::llvm::key(checkout)) +} + +/// [`source`], with the key of the LLVM `checkout` names. +fn source_with(checkout: &Path, llvm: &str) -> String { + format!("{} llvm {llvm}", compiler_source(checkout)) } /// What `checkout`'s `compiler/` is: its commit's tree, and whatever the working @@ -171,33 +180,37 @@ pub fn forget(rust_dir: &Path) { /// /// **The source's content, never its files' times**: a checkout that rewrites a /// file with the bytes it had is no new compiler. -fn primary_is(rust_dir: &Path, checkout: &Path) -> Result { - let names = source(checkout); +fn primary_is(rust_dir: &Path, checkout: &Path, llvm: &str) -> Result { + let names = source_with(checkout, llvm); Ok(fs::read_to_string(primary_record(rust_dir))?.trim() == names) } /// Whether the primary's `stage2` is built from what its own `rust/compiler/` /// holds: false until a bootstrap has finished and [`record`]ed it. pub fn primary_is_current(rust_dir: &Path) -> bool { - match primary_is(rust_dir, rust_dir) { + match primary_is(rust_dir, rust_dir, &crate::llvm::key(rust_dir)) { Ok(current) => current, Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, Err(e) => panic!("read {}: {e}", primary_record(rust_dir).display()), } } -/// The key of the compiler `fork`'s sources name: their content, so committing -/// what was built as local changes names the same compiler. +/// The key of the compiler `fork`'s sources name: their content. pub fn key(fork: &Path) -> String { - let parts = [RECIPE.to_string(), tree_identity(fork, &KEYED), llvm_commit(fork)]; + key_with(fork, &crate::llvm::key(fork)) +} + +/// [`key`], with the key of the LLVM `fork` names. +fn key_with(fork: &Path, llvm: &str) -> String { + let parts = [RECIPE, &tree_identity(fork, &KEYED), llvm]; short(parts.join("\n\0\n").as_bytes()) } -/// The LLVM commit `fork` builds against: the one its `HEAD` records, which is -/// the one bootstrap checks out and builds whatever the submodule holds. An +/// The LLVM commit `fork` builds against: the one its `HEAD` records, refused +/// when its index stages another, because bootstrap checks out the index's. An /// LLVM change is a commit there and a gitlink here, so a checkout holding /// anything no commit does is refused rather than named by its commit. -fn llvm_commit(fork: &Path) -> String { +pub(crate) fn llvm_commit(fork: &Path) -> String { let checkout = fork.join(LLVM); // Exactly what bootstrap's LLVM stamp hashes beyond the commit; the untracked // cache spares each call a walk of the whole tree. @@ -211,10 +224,22 @@ fn llvm_commit(fork: &Path) -> String { fork.display(), ); let recorded = git_out(fork, &["ls-tree", "HEAD", LLVM]); - match recorded.split_whitespace().collect::>().as_slice() { + let committed = match recorded.split_whitespace().collect::>().as_slice() { ["160000", "commit", sha, _] => sha.to_string(), _ => panic!("{} records no {LLVM} gitlink: `git ls-tree HEAD {LLVM}` said {recorded:?}", fork.display()), - } + }; + let indexed = git_out(fork, &["ls-files", "--stage", LLVM]); + let staged = match indexed.split_whitespace().collect::>().as_slice() { + ["160000", sha, "0", _] => sha.to_string(), + _ => panic!("{} indexes no {LLVM} gitlink: `git ls-files --stage {LLVM}` said {indexed:?}", fork.display()), + }; + assert!( + staged == committed, + "{} stages {LLVM} at {staged}, and its HEAD records {committed}: bootstrap builds the one \ + staged, and nothing is keyed on what no commit holds; commit the gitlink, or unstage it", + fork.display(), + ); + committed } /// The compiler `root`'s fork checkout at `fork` names: the primary's where its @@ -222,18 +247,18 @@ fn llvm_commit(fork: &Path) -> String { /// built if nobody has built it, held in use for as long as the returned value /// lives. pub fn resolve(root: &Path, rust_dir: &Path, fork: &Path, lock: &mut Held) -> Compiler { - lock.without_shared(|| choose(root, rust_dir, fork, build_in_fork)) + lock.without_shared(|| choose(root, rust_dir, fork, |fork| build_in_fork(root, rust_dir, fork))) } /// [`resolve`] with the build that makes a compiler's `stage2` passed in, so a /// test can stand in for bootstrap: `build` compiles the fork checkout it is /// given and returns the `stage2` it left there. fn choose(root: &Path, rust_dir: &Path, fork: &Path, build: impl Fn(&Path) -> PathBuf) -> Compiler { - let recorded = root.join(RECORD); if fork == rust_dir { return Compiler::primary(rust_dir); } - let names_primary = primary_is(rust_dir, fork).unwrap_or_else(|e| { + let llvm = crate::llvm::key(fork); + let names_primary = primary_is(rust_dir, fork, &llvm).unwrap_or_else(|e| { panic!( "{} cannot be read ({e}), so nothing says which compiler the primary's stage2 is, \ and no worktree can know whether it names that one.\n\ @@ -241,32 +266,29 @@ fn choose(root: &Path, rust_dir: &Path, fork: &Path, build: impl Fn(&Path) -> Pa primary_record(rust_dir).display(), ) }); + // The LLVM record follows the compiler's: a compiler links the LLVM its key + // names, and the primary's is recorded by the primary. if names_primary { - let _ = fs::remove_file(&recorded); + keystore::forget(root, Keyed::Compiler); + keystore::forget(root, Keyed::Llvm); + let build = fork.join(BUILD_DIR); + if !crate::llvm::in_tree(&build).is_empty() { + let _worktree = buildlock::worktree_exclusive(root, "removing the LLVM its compiler build built"); + crate::llvm::retire_in_tree(&build); + } return Compiler::primary(rust_dir); } - let key = key(fork); + let key = key_with(fork, &llvm); let dir = compilers_dir(rust_dir).join(&key); - fs::create_dir_all(recorded.parent().expect("a file under target/")).ok(); - fs::write(&recorded, &key).unwrap_or_else(|e| panic!("write {}: {e}", recorded.display())); - let mut placed = false; - let using = buildlock::keyed_made( + keystore::record(root, Keyed::Llvm, &llvm); + let using = keystore::made( root, Keyed::Compiler, + &compilers_dir(rust_dir), &key, || (!dir.join(SOURCE).is_file()).then(|| format!("{} carries no {SOURCE}", dir.display())), - || { - place(root, fork, &key, &dir, &build); - placed = true; - }, + || place(root, fork, &key, &dir, &build), ); - // A compiler edit loop places one per edit, and the one this replaced is - // named by nobody now; the one in use is held, so the sweep leaves it. - if placed { - for gone in sweep(root, rust_dir) { - eprintln!("Removed compiler {}: no worktree names it", gone.display()); - } - } Compiler { stage2: dir.join("stage2"), record: dir.join(SOURCE), primary: false, _using: Some(using) } } @@ -277,6 +299,7 @@ fn place(root: &Path, fork: &Path, key: &str, dir: &Path, build: &impl Fn(&Path) let _worktree = buildlock::worktree_exclusive(root, &what); eprintln!("Building compiler {key} in {}: its compiler/ is not the one the primary's was built from", fork.display()); let stage2 = build(fork); + crate::llvm::retire_in_tree(&fork.join(BUILD_DIR)); let partial = dir.with_extension("partial"); if partial.exists() { fs::remove_dir_all(&partial).unwrap_or_else(|e| panic!("remove {}: {e}", partial.display())); @@ -295,15 +318,16 @@ fn place(root: &Path, fork: &Path, key: &str, dir: &Path, build: &impl Fn(&Path) } /// Bootstrap's build of the compiler in `fork`, into its own build directory, -/// and the `stage2` it made, with the cargo and the clang every toolchain -/// directory carries. -fn build_in_fork(fork: &Path) -> PathBuf { +/// against the LLVM `fork` names (`src/llvm.rs`), and the `stage2` it made, +/// with the cargo and the clang every toolchain directory carries. +fn build_in_fork(root: &Path, rust_dir: &Path, fork: &Path) -> PathBuf { crate::ensure_submodule(fork, "library/backtrace"); + let llvm = crate::llvm::resolve(root, rust_dir, fork); let host = host_triple(); - let build_dir = fork.join("build/toyos-compiler"); + let build_dir = fork.join(BUILD_DIR); fs::create_dir_all(&build_dir).unwrap_or_else(|e| panic!("create {}: {e}", build_dir.display())); let config = build_dir.join("bootstrap.toml"); - fs::write(&config, config_text(&build_dir, &host)).unwrap_or_else(|e| panic!("write {}: {e}", config.display())); + fs::write(&config, config_text(&build_dir, &host, &llvm.dir)).unwrap_or_else(|e| panic!("write {}: {e}", config.display())); let config = config.to_str().unwrap_or_else(|| panic!("{} is not UTF-8", config.display())); let args = ["build", "--stage", "2", "--config", config, "--warnings", "warn", "compiler/rustc", "library"]; let (ok, log) = toolchain::x_build(fork, &args, "the compiler"); @@ -312,15 +336,15 @@ fn build_in_fork(fork: &Path) -> PathBuf { let stage2 = build_dir.join(&host).join("stage2"); assert!(stage2.join("bin/rustc").is_file(), "the compiler build left no {}", stage2.join("bin/rustc").display()); toolchain::provision_toolchain_cargo(&stage2); - crate::clang::provision(&stage2); + crate::clang::provision(&stage2, &llvm.dir); toolchain::assert_toolchain_is_honest(&stage2); stage2 } /// Bootstrap's configuration for a compiler of a worktree's own: the primary's /// `profile` and options, for the host alone, since every guest target's -/// libraries are the sysroot's to build. -fn config_text(build_dir: &Path, host: &str) -> String { +/// libraries are the sysroot's to build, linking the LLVM at `llvm`. +fn config_text(build_dir: &Path, host: &str, llvm: &Path) -> String { format!( r#"change-id = "ignore" profile = "compiler" @@ -339,48 +363,17 @@ lld = true [target.{host}] {pin} +{external} "#, build_dir = build_dir.display(), llvm = crate::clang::LLVM_CONFIG, pin = toolchain::HOST_LINKER_PIN, + external = crate::llvm::host_lines(llvm), ) } -/// The key of the compiler of its own `root`'s last build used, if it used one. -pub fn recorded_key(root: &Path) -> Option { - fs::read_to_string(root.join(RECORD)).ok().map(|k| k.trim().to_string()) -} - -/// Remove every compiler no registered worktree records and nobody is making -/// or using, and every half-built one nobody is making. Returns what went. -pub fn sweep(root: &Path, rust_dir: &Path) -> Vec { - let dir = compilers_dir(rust_dir); - let Ok(entries) = fs::read_dir(&dir) else { return Vec::new() }; - let named: std::collections::BTreeSet = git_out(root, &["worktree", "list", "--porcelain"]) - .lines() - .filter_map(|l| l.strip_prefix("worktree ")) - .filter_map(|w| recorded_key(Path::new(w))) - .collect(); - let mut removed = Vec::new(); - for entry in entries.flatten() { - let name = entry.file_name().to_string_lossy().into_owned(); - let (key, whole) = match name.split_once('.') { - Some((key, _)) => (key.to_string(), false), - None => (name.clone(), true), - }; - if whole && named.contains(&key) { - continue; - } - let Some(_idle) = buildlock::keyed_idle(root, Keyed::Compiler, &key) else { continue }; - let path = entry.path(); - fs::remove_dir_all(&path).unwrap_or_else(|e| panic!("remove {}: {e}", path.display())); - removed.push(path); - } - removed -} - #[cfg(test)] -mod tests { +pub(crate) mod tests { use std::cell::Cell; use toyos_tmpdir::TempDir; @@ -388,7 +381,7 @@ mod tests { use super::*; - fn git(dir: &Path, args: &[&str]) -> String { + pub(crate) fn git(dir: &Path, args: &[&str]) -> String { let out = Command::new("git") .args(["-c", "commit.gpgsign=false", "-c", "user.email=t@t", "-c", "user.name=t"]) .args(["-c", "protocol.file.allow=always", "-c", "init.defaultBranch=main"]) @@ -400,7 +393,7 @@ mod tests { String::from_utf8(out.stdout).unwrap().trim().to_string() } - fn write(path: &Path, text: &str) { + pub(crate) fn write(path: &Path, text: &str) { fs::create_dir_all(path.parent().unwrap()).unwrap(); fs::write(path, text).unwrap(); } @@ -424,13 +417,13 @@ mod tests { } /// The LLVM commits the fixtures' forks record. Nothing reads their content. - const LLVM_A: &str = "1111111111111111111111111111111111111111"; - const LLVM_B: &str = "2222222222222222222222222222222222222222"; + pub(crate) const LLVM_A: &str = "1111111111111111111111111111111111111111"; + pub(crate) const LLVM_B: &str = "2222222222222222222222222222222222222222"; /// A primary whose `rust` pins fork commit `C0` and has built a compiler /// from it, and three linked worktrees: `same` pins `C0`, `a` and `b` each /// pin a commit whose `compiler/` is its own. - fn estate(scratch: &Path) -> (PathBuf, PathBuf, [PathBuf; 3]) { + pub(crate) fn estate(scratch: &Path) -> (PathBuf, PathBuf, [PathBuf; 3]) { let base = fs::canonicalize(scratch).unwrap(); let fork = base.join("fork-src"); @@ -547,6 +540,7 @@ mod tests { let user = chosen_elsewhere(&a, &rust_dir); fs::remove_file(&spec).unwrap(); let kept = choose(&a, &rust_dir, &a.join("rust"), fake); + let sweep = |root: &Path, rust_dir: &Path| keystore::sweep(root, Keyed::Compiler, &compilers_dir(rust_dir)); assert_eq!(sweep(&primary, &rust_dir), Vec::::new(), "the sweep took a compiler still in use"); assert!(orphan.is_dir() && ca2.stage2.is_dir()); user.release(); @@ -567,6 +561,38 @@ mod tests { assert!(ca3.stage2.is_dir() && named.is_dir()); } + /// **A worktree names the LLVM of the compiler it builds with, and only + /// that one**: one it placed or found placed, never one it has gone back to + /// the primary's from; and its build directory keeps no LLVM of its own + /// either way. + #[test] + fn the_llvm_record_follows_the_compiler_in_use() { + let scratch = TempDir::new("compiler-llvm-record"); + let (primary, rust_dir, [_same, a, _b]) = estate(&scratch); + let fork = a.join("rust"); + let store = crate::llvm::store(&rust_dir); + let llvm = store.join(crate::llvm::key(&fork)); + let sweep = || keystore::sweep(&primary, Keyed::Llvm, &store); + + let own = fork.join(BUILD_DIR).join(host_triple()).join("llvm"); + write(&own.join("bin/llvm-config"), "the build directory's own"); + drop(choose(&a, &rust_dir, &fork, fake_build)); + assert!(!own.exists(), "a compiler built against the store left the LLVM its build directory built"); + fs::create_dir_all(&llvm).unwrap(); + assert_eq!(sweep(), Vec::::new(), "the LLVM of a worktree's own compiler was swept"); + + keystore::forget(&a, Keyed::Llvm); + let never = |_: &Path| -> PathBuf { panic!("a placed compiler was built again") }; + drop(choose(&a, &rust_dir, &fork, never)); + assert_eq!(keystore::recorded(&a, Keyed::Llvm), Some(crate::llvm::key(&fork)), "a placed compiler's LLVM went unrecorded"); + + git(&fork, &["checkout", "-q", &git(&rust_dir, &["rev-parse", "HEAD"])]); + write(&own.join("bin/llvm-config"), "the build directory's own, from before the store"); + assert!(choose(&a, &rust_dir, &fork, never).primary); + assert_eq!(sweep(), [llvm], "the LLVM of a compiler the worktree no longer builds with stayed"); + assert!(!own.exists(), "a worktree back on the primary's compiler kept the LLVM its build directory built"); + } + const WORKTREE: &str = "TOYOS_COMPILER_TEST_WORKTREE"; const RUST_DIR: &str = "TOYOS_COMPILER_TEST_RUST_DIR"; @@ -657,8 +683,14 @@ mod tests { assert!(!primary_is_current(&rust_dir), "a stage2 nothing recorded was taken for current"); } + /// Write the primary's record as it was written before its compiler linked + /// the host's LLVM: its `compiler/` and its LLVM commit. + pub(crate) fn record_before_the_store(rust_dir: &Path) { + fs::write(primary_record(rust_dir), format!("{} llvm {}", compiler_source(rust_dir), llvm_commit(rust_dir))).unwrap(); + } + /// `fork`'s LLVM checked out at a commit of its own. - fn llvm_checkout(fork: &Path) -> PathBuf { + pub(crate) fn llvm_checkout(fork: &Path) -> PathBuf { let llvm = fork.join(LLVM); git(&llvm, &["init", "-q"]); write(&llvm.join("llvm/lib/IR/Core.cpp"), "int core;\n"); @@ -729,7 +761,10 @@ mod tests { let tools = key(&fork); assert_ne!(tools, before, "a tool's source did not move the key"); git(&fork, &["update-index", "--add", "--cacheinfo", &format!("160000,{LLVM_B},{LLVM}")]); - assert_eq!(key(&fork), tools, "a gitlink staged and not committed moved the key; bootstrap builds HEAD's"); + let said = refusal("a gitlink staged and not committed named a compiler", || { + key(&fork); + }); + assert!(said.contains("stages"), "{said}"); git(&fork, &["commit", "-qm", "another LLVM"]); assert_ne!(key(&fork), tools, "another LLVM commit did not move the key"); } @@ -753,7 +788,7 @@ mod tests { let record = primary_record(&rust_dir); let recorded = fs::read_to_string(&record).unwrap(); let (compiler, llvm) = recorded.rsplit_once(" llvm ").expect("the record names its LLVM"); - assert_eq!(llvm, LLVM_A); + assert_eq!(llvm, crate::llvm::key(&rust_dir)); fs::write(&record, compiler).unwrap(); assert!(!choose(&same, &rust_dir, &fork, fake).primary, "a record naming no LLVM was taken for this one"); assert_eq!(builds.get(), 1); diff --git a/src/forkcheck.rs b/src/forkcheck.rs index 6b5aef196fb..e0e517a0eb9 100644 --- a/src/forkcheck.rs +++ b/src/forkcheck.rs @@ -33,10 +33,13 @@ //! never a repurposed cfg gate; prefer ToyOS-specific files (`sys/pal/toyos/`, //! `os/toyos/`, anything with `toyos` in the path); a cross-platform file is //! touched only to add a target arm at an existing platform-dispatch site, never -//! to change cross-platform semantics or API shape; `library/alloc` and -//! `library/core` have **zero** delta. Cherry-picking an already-merged upstream -//! commit is allowed. Copying an unmerged PR is not — the delta must stay -//! exactly the content of a future upstream PR. +//! to change cross-platform semantics or API shape, except `src/bootstrap`, which +//! takes a general bootstrap capability written to upstream quality; a +//! `change_tracker` entry is carried only with the upstream pull request number +//! it records, and until one exists the change is recorded in `issues/`; +//! `library/alloc` and `library/core` have **zero** delta. Cherry-picking an already-merged upstream commit is allowed. Copying +//! an unmerged PR is not — the delta must stay exactly the content of a future +//! upstream PR. //! //! **Coverage, and this is the one that bites:** the fork sources live *outside* //! this repository, so a repository-wide search or gate does not reach them. An diff --git a/src/keystore.rs b/src/keystore.rs new file mode 100644 index 00000000000..8cfd9d6d41c --- /dev/null +++ b/src/keystore.rs @@ -0,0 +1,266 @@ +//! What every content-addressed product of the host ([`Keyed`]) shares: the +//! record each worktree keeps of the key it uses, and the sweep that removes a +//! key no registered worktree records and nobody is making or using. +//! +//! A product lives at `//`, whole once its maker renamed it there; +//! any other name beginning `.` is one half-made or half-removed. A whole +//! one is renamed out of the way before anything in it is removed ([`retire`]), +//! so a sweep that is stopped leaves nothing at `/` but what was made. A +//! product may be read-only, directories and all: removing one gives its +//! directories back their write permission first ([`remove`]). + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::ErrorKind; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; + +use crate::buildlock::{self, Guard, Keyed}; +use crate::sysroot::git_out; + +/// Where `root`'s builds record the key of `kind` they use. +fn record_path(root: &Path, kind: Keyed) -> PathBuf { + root.join(match kind { + Keyed::Sysroot => "target/toyos-sysroot-key", + Keyed::Compiler => "target/toyos-compiler-key", + Keyed::Llvm => "target/toyos-llvm-key", + }) +} + +/// Record that `root` uses `kind`'s `key`: whole or not at all, so a sweep +/// never reads a record half-written. +pub fn record(root: &Path, kind: Keyed, key: &str) { + record_by(root, kind, key, |path, key| fs::write(path, key).unwrap_or_else(|e| panic!("write {}: {e}", path.display()))); +} + +/// [`record`], writing with `write`, so a test can stop it. +fn record_by(root: &Path, kind: Keyed, key: &str, write: impl FnOnce(&Path, &str)) { + let path = record_path(root, kind); + let dir = path.parent().expect("a file under target/"); + fs::create_dir_all(dir).unwrap_or_else(|e| panic!("create {}: {e}", dir.display())); + let written = path.with_extension("new"); + write(&written, key); + fs::rename(&written, &path).unwrap_or_else(|e| panic!("rename {} -> {}: {e}", written.display(), path.display())); +} + +/// Record that `root` uses `kind`'s `key`, whose product is `store/`, and +/// hold it in use: made by `make` first when `defect` says it is not whole +/// (`buildlock::keyed_made`), and then `store` swept, so the product one +/// replaced goes once nobody names it. +pub fn made( + root: &Path, + kind: Keyed, + store: &Path, + key: &str, + defect: impl Fn() -> Option, + mut make: impl FnMut(), +) -> Guard { + record(root, kind, key); + let mut placed = false; + let using = buildlock::keyed_made(root, kind, key, defect, || { + make(); + placed = true; + }); + if placed { + for gone in sweep(root, kind, store) { + eprintln!("Removed {} {}: no worktree names it", kind.name(), gone.display()); + } + } + using +} + +/// Record that `root` uses no `kind` of its own. +pub fn forget(root: &Path, kind: Keyed) { + let path = record_path(root, kind); + match fs::remove_file(&path) { + Err(e) if e.kind() != ErrorKind::NotFound => panic!("remove {}: {e}", path.display()), + _ => {} + } +} + +/// The key of `kind` `root` records, if it records one. +pub fn recorded(root: &Path, kind: Keyed) -> Option { + let path = record_path(root, kind); + match fs::read_to_string(&path) { + Ok(key) => Some(key.trim().to_string()), + Err(e) if e.kind() == ErrorKind::NotFound => None, + Err(e) => panic!("read {}: {e}", path.display()), + } +} + +/// Remove from `store` every `kind` no registered worktree of `root` records +/// and nobody is making or using, and every half-made or half-removed one +/// nobody is making. Returns what went. +pub fn sweep(root: &Path, kind: Keyed, store: &Path) -> Vec { + sweep_by(root, kind, store, remove) +} + +/// [`sweep`], removing with `remove`, so a test can stop it. +pub(crate) fn sweep_by(root: &Path, kind: Keyed, store: &Path, remove: impl Fn(&Path) + Copy) -> Vec { + let entries = match fs::read_dir(store) { + Ok(entries) => entries, + Err(e) if e.kind() == ErrorKind::NotFound => return Vec::new(), + Err(e) => panic!("read {}: {e}", store.display()), + }; + let named: BTreeSet = git_out(root, &["worktree", "list", "--porcelain"]) + .lines() + .filter_map(|l| l.strip_prefix("worktree ")) + .filter_map(|w| recorded(Path::new(w), kind)) + .collect(); + let mut by_key: BTreeMap> = BTreeMap::new(); + for entry in entries { + let entry = entry.unwrap_or_else(|e| panic!("read {}: {e}", store.display())); + let name = entry + .file_name() + .into_string() + .unwrap_or_else(|n| panic!("{} holds {n:?}, which names no key", store.display())); + let key = name.split_once('.').map_or(name.as_str(), |(key, _)| key).to_string(); + by_key.entry(key).or_default().push(name); + } + let mut removed = Vec::new(); + for (key, mut names) in by_key { + names.retain(|name| *name != key || !named.contains(&key)); + if names.is_empty() { + continue; + } + let Some(_idle) = buildlock::keyed_idle(root, kind, &key) else { continue }; + // The whole one last: its `retire` goes where a stopped one's remains were. + names.sort_by_key(|name| *name == key); + for name in names { + let path = store.join(&name); + if name == key { + retire_by(&path, remove); + } else { + remove(&path); + } + removed.push(path); + } + } + removed +} + +/// Remove the directory `path` if it is there, renamed to `.swept` before +/// anything in it is removed; what a stopped one left there goes first. +pub fn retire(path: &Path) { + retire_by(path, remove); +} + +/// [`retire`], removing with `remove`, so a test can stop it. +fn retire_by(path: &Path, remove: impl Fn(&Path)) { + let away = path.with_extension("swept"); + if away.exists() { + remove(&away); + } + if path.exists() { + fs::rename(path, &away).unwrap_or_else(|e| panic!("rename {} -> {}: {e}", path.display(), away.display())); + remove(&away); + } +} + +/// Remove the directory `path` and all it holds, read-only or not. +pub fn remove(path: &Path) { + writable(path); + fs::remove_dir_all(path).unwrap_or_else(|e| panic!("remove {}: {e}", path.display())); +} + +/// Give `dir` and every directory under it back its owner's write permission. +pub(crate) fn writable(dir: &Path) { + let meta = fs::metadata(dir).unwrap_or_else(|e| panic!("stat {}: {e}", dir.display())); + let mut permissions = meta.permissions(); + permissions.set_mode(permissions.mode() | 0o700); + fs::set_permissions(dir, permissions).unwrap_or_else(|e| panic!("chmod {}: {e}", dir.display())); + for entry in fs::read_dir(dir).unwrap_or_else(|e| panic!("read {}: {e}", dir.display())) { + let entry = entry.unwrap_or_else(|e| panic!("read {}: {e}", dir.display())); + if entry.file_type().unwrap_or_else(|e| panic!("stat {}: {e}", entry.path().display())).is_dir() { + writable(&entry.path()); + } + } +} + +#[cfg(test)] +mod tests { + use toyos_tmpdir::TempDir; + + use super::*; + use crate::compiler::tests::{git, write}; + + /// A key no registered worktree records goes, read-only or not, and so does + /// a half-made or half-removed one; a key a worktree records stays, and so + /// does one somebody is using; an unreadable record is refused, never read + /// as "names nothing". + #[test] + fn a_sweep_removes_what_no_worktree_names_and_nobody_uses() { + let root = TempDir::new("sweep"); + git(&root, &["init", "-q"]); + write(&root.join("f"), "x\n"); + git(&root, &["add", "f"]); + git(&root, &["commit", "-qm", "init"]); + let linked = root.join("linked"); + git(&root, &["worktree", "add", "-q", "-b", "wt", linked.to_str().unwrap()]); + + let dir = root.join("store"); + for name in ["named", "linked-named", "in-use", "orphan", "named.partial", "gone.swept", "orphan.swept"] { + fs::create_dir_all(dir.join(name).join("sub")).unwrap(); + } + for read_only in ["orphan/sub", "orphan", "named.partial"] { + fs::set_permissions(dir.join(read_only), fs::Permissions::from_mode(0o555)).unwrap(); + } + record(&root, Keyed::Sysroot, "named"); + record(&linked, Keyed::Sysroot, "linked-named"); + let user = buildlock::tests::sysroot_used_elsewhere(&root, "in-use"); + + let mut removed = sweep(&root, Keyed::Sysroot, &dir); + removed.sort(); + let want = ["gone.swept", "named.partial", "orphan", "orphan.swept"].map(|n| dir.join(n)); + assert_eq!(removed, want); + for stays in ["named", "linked-named", "in-use"] { + assert!(dir.join(stays).is_dir(), "{stays} was swept"); + } + for gone in want { + assert!(!gone.exists(), "{} was reported and kept", gone.display()); + } + user.release(); + assert_eq!(sweep(&root, Keyed::Sysroot, &dir), [dir.join("in-use")]); + + fs::remove_file(record_path(&linked, Keyed::Sysroot)).unwrap(); + fs::create_dir(record_path(&linked, Keyed::Sysroot)).unwrap(); + let unreadable = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| sweep(&root, Keyed::Sysroot, &dir))); + assert!(unreadable.is_err(), "an unreadable record was read as naming nothing"); + assert!(dir.join("linked-named").is_dir(), "an unreadable record's key was swept"); + } + + /// **A record stopped mid-write leaves the one before it readable**: the + /// new key is written beside it and renamed over it whole. + #[test] + fn a_stopped_record_leaves_the_one_before_it() { + let root = TempDir::new("record"); + record(&root, Keyed::Llvm, "0123456789abcdef"); + let stopped = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + record_by(&root, Keyed::Llvm, "fedcba9876543210", |path, key| { + fs::write(path, &key[..8]).unwrap(); + panic!("stopped"); + }) + })); + assert!(stopped.is_err(), "the stand-in write was never asked"); + assert_eq!(recorded(&root, Keyed::Llvm).as_deref(), Some("0123456789abcdef"), "a record stopped mid-write was read"); + record(&root, Keyed::Llvm, "fedcba9876543210"); + assert_eq!(recorded(&root, Keyed::Llvm).as_deref(), Some("fedcba9876543210")); + } + + /// **A retire stopped halfway leaves nothing at the name it removes**: what + /// it removes is out of the way before anything in it goes, so a stopped + /// sweep leaves nothing that passes for whole, and the next retire takes + /// what the stopped one left. + #[test] + fn a_stopped_retire_leaves_nothing_at_its_name() { + let store = TempDir::new("retire"); + let whole = store.join("key"); + write(&whole.join("SOURCE"), "key\n"); + write(&whole.join("lib/libLLVMCore.a"), "core"); + let stopped = std::panic::catch_unwind(|| retire_by(&whole, |_| panic!("stopped"))); + assert!(stopped.is_err(), "the stand-in removal was never asked"); + assert!(!whole.exists(), "a stopped retire left {}", whole.display()); + retire(&whole); + assert!(!whole.with_extension("swept").exists(), "the next retire kept what the stopped one left"); + } +} diff --git a/src/lib.rs b/src/lib.rs index a143a0e9c4d..87c57e13c2d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,8 +25,10 @@ pub mod signing; /// but its own tests. #[cfg(test)] pub mod kernelkeys; +pub mod keystore; pub mod lan; pub mod libc; +pub mod llvm; pub mod licence; pub mod metal; pub mod metaldevices; diff --git a/src/llvm.rs b/src/llvm.rs new file mode 100644 index 00000000000..4c66bbfce78 --- /dev/null +++ b/src/llvm.rs @@ -0,0 +1,962 @@ +//! The LLVM every host compiler links, with its clang and LLD, content-addressed: +//! one per key on this host, shared by every compiler build that names it. +//! +//! **An LLVM is a function of its key** ([`key`]): the `src/llvm-project` commit +//! the fork checkout's gitlink names (`compiler::llvm_commit`, which refuses a +//! checkout holding what no commit does and a gitlink staged and not committed), +//! the committed tree of its `src/bootstrap` (one holding what no commit does is +//! refused), the bootstrap configuration below, [`RECIPE`], and the tools the +//! host builds it with ([`host_tools`]). `rust/build/llvm//` in the primary +//! is bootstrap's install of that LLVM and its clang, with its LLD in `bin/` +//! beside `llvm-config`, made by whichever build first needs it ([`resolve`]), +//! and stored only when it was built from what the key names. Once its +//! [`SOURCE`] file exists it is read-only, its directories as well as its files. +//! Every compiler build, the primary's and a worktree's own, names it as the +//! host's `llvm-config` with `llvm-has-rust-patches`, so bootstrap builds no +//! LLVM and takes LLD from beside it as `rust-lld`; `clang::provision` copies its +//! clang. Once a build directory's compiler is built against it, the LLVM that +//! directory built itself goes ([`retire_in_tree`]). +//! +//! **Nothing of the environment it is asked from reaches it but +//! [`ENVIRONMENT`]**: the build and every tool its key asks run with the rest +//! cleared ([`clear`]), the configuration names the C and C++ compilers by path, +//! and every host library LLVM would otherwise find and link is turned off +//! ([`NO_HOST_LIBRARIES`]). +//! +//! Its lock (`buildlock::keyed_*` with [`Keyed::Llvm`]) is taken inside the +//! worktree or global lock that covers the fork build directory its maker +//! writes, `build/toyos-llvm/`, which is removed once the LLVM is placed. +//! +//! An LLVM no worktree names any more is removed by `keystore::sweep`, which +//! `--worktree remove` and every placement run: each worktree records the key of +//! the LLVM its compiler links, and a key no registered worktree records, that +//! nobody is making or using, goes. + +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::OnceLock; + +use crate::buildlock::{Guard, Keyed}; +use crate::compiler::{llvm_commit, LLVM}; +use crate::keystore; +use crate::sysroot::{clone_tree, git_bytes, git_out, short}; +use crate::toolchain::{self, host_triple}; + +/// What changes how a key's sources become an LLVM and is none of the other +/// parts: the build's targets and what is kept of it. Moving it moves every key. +const RECIPE: &str = "bootstrap build of src/llvm-project/llvm and src/llvm-project/lld; the install's bin, \ + include and lib, and lld in bin, read-only; 2"; + +/// What of the caller's environment the LLVM build, and every tool its key +/// asks, sees: +/// - `PATH` finds what runs the build: the Python behind `./x`, git, curl, +/// Ninja, and CMake, which the key names by path and version. The C and C++ +/// compilers it finds, the configuration names by path. +/// - `TMPDIR` is where those tools write what they discard; the sandbox a build +/// runs in may allow no other place. +/// +/// Everything else is dropped on purpose, the proxy, CA-bundle and Nix +/// variables a fetch may need among them: such a host's fetch fails loudly. +const ENVIRONMENT: [&str; 2] = ["PATH", "TMPDIR"]; + +/// Every host library the pinned LLVM and clang look for, and link when they +/// find one, turned off by the CMake option that asks for it +/// (`llvm/CMakeLists.txt`, `llvm/cmake/config-ix.cmake`, `clang/CMakeLists.txt`): +/// what a host has installed never decides what is built. Bootstrap's LLD step +/// takes none of these: LLD looks for no library unless asked, and links what this +/// LLVM chose. +const NO_HOST_LIBRARIES: [&str; 12] = [ + "LLVM_ENABLE_ZLIB", + "LLVM_ENABLE_ZSTD", + "LLVM_ENABLE_LIBXML2", + "CLANG_ENABLE_LIBXML2", + "LLVM_ENABLE_LIBEDIT", + "LLVM_ENABLE_LIBPFM", + "LLVM_ENABLE_FFI", + "LLVM_ENABLE_CURL", + "LLVM_ENABLE_HTTPLIB", + "LLVM_ENABLE_ICU", + "LLVM_ENABLE_ICONV", + "LLVM_ENABLE_Z3_SOLVER", +]; + +/// What of bootstrap's install an LLVM keeps: `build/` beside them is CMake's +/// tree, which nothing reads once the install is made. +const KEPT: [&str; 3] = ["bin", "include", "lib"]; + +/// What a compiler build and `clang::provision` read of an LLVM. +const TOOLS: [&str; 4] = ["bin/llvm-config", "bin/lld", "bin/clang", "bin/llvm-ar"]; + +/// The file a finished LLVM carries last, naming its key. A directory without +/// it is a build that did not finish. +const SOURCE: &str = "SOURCE"; + +/// The fork's bootstrap, whose `Llvm` step and `compiler` profile decide how +/// LLVM is configured. +const BOOTSTRAP: &str = "src/bootstrap"; + +/// The build directory the key's configuration names. +const KEYED_BUILD_DIR: &str = ""; + +/// An LLVM, held in use for as long as this lives. +pub struct Llvm { + /// Its install: `bin/`, `include/`, `lib/`. + pub dir: PathBuf, + _using: Guard, +} + +/// The `[target.]` lines that make a `bootstrap.toml` link the LLVM at +/// `dir` and take its LLD. +pub fn host_lines(dir: &Path) -> String { + format!("llvm-config = \"{}\"\nllvm-has-rust-patches = true", dir.join("bin/llvm-config").display()) +} + +/// Every LLVM on this host. +pub fn store(rust_dir: &Path) -> PathBuf { + rust_dir.join("build/llvm") +} + +/// The key of the LLVM `fork` names; refused while its `src/bootstrap` holds +/// changes no commit does. +pub fn key(fork: &Path) -> String { + let tools = host_tools(); + key_of(fork, RECIPE, &config_text(Path::new(KEYED_BUILD_DIR), &host_triple(), tools), &tools.identity) +} + +fn key_of(fork: &Path, recipe: &str, config: &str, tools: &str) -> String { + refuse_uncommitted_bootstrap(fork); + let bootstrap = git_out(fork, &["rev-parse", &format!("HEAD:{BOOTSTRAP}")]); + short([recipe, config, &llvm_commit(fork), bootstrap.trim(), tools].join("\n\0\n").as_bytes()) +} + +/// Give `command` nothing of this process's environment but [`ENVIRONMENT`]. +fn clear(command: &mut Command) { + command.env_clear(); + for name in ENVIRONMENT { + if let Some(value) = std::env::var_os(name) { + command.env(name, value); + } + } +} + +/// The tools this host builds an LLVM with, as the cleared environment finds +/// them. +struct HostTools { + cc: PathBuf, + cxx: PathBuf, + /// The C and C++ compilers and CMake, each by its resolved path and all its + /// `--version` says, and on macOS the SDK path and version `xcrun` resolves. + identity: String, +} + +fn host_tools() -> &'static HostTools { + static TOOLS: OnceLock = OnceLock::new(); + TOOLS.get_or_init(|| tools_with(|question| asked(Command::new("xcrun").arg(question)))) +} + +/// [`host_tools`], with `xcrun`'s answer to each question it is asked, so a +/// test can stand in for it. +fn tools_with(xcrun: impl Fn(&str) -> String) -> HostTools { + let [cc, cxx, cmake] = ["cc", "c++", "cmake"].map(on_path); + let mut identity = String::new(); + for tool in [&cc, &cxx, &cmake] { + let mut version = Command::new(tool); + identity += &format!("{}\n{}", tool.display(), asked(version.arg("--version"))); + } + if host_triple().ends_with("apple-darwin") { + for question in ["--show-sdk-path", "--show-sdk-version"] { + identity += &xcrun(question); + } + } + HostTools { cc, cxx, identity } +} + +/// Everything `command` prints, run with the environment [`clear`]ed; a +/// failure is refused. +fn asked(command: &mut Command) -> String { + clear(command); + let out = command.output().unwrap_or_else(|e| panic!("run {command:?}: {e}")); + assert!(out.status.success(), "{command:?} failed: {}", String::from_utf8_lossy(&out.stderr)); + format!("{}{}\n", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)) +} + +/// `name` as the `PATH` of [`ENVIRONMENT`] finds it, resolved. +fn on_path(name: &str) -> PathBuf { + let path = std::env::var_os("PATH").unwrap_or_default(); + let found = std::env::split_paths(&path) + .map(|dir| dir.join(name)) + .find(|candidate| candidate.is_file()) + .unwrap_or_else(|| panic!("no `{name}` on PATH, and the LLVM build runs it")); + fs::canonicalize(&found).unwrap_or_else(|e| panic!("resolve {}: {e}", found.display())) +} + +/// The LLVM `fork` names, made if nobody on this host has made it, and held in +/// use for as long as the returned value lives. `root` records its key. +pub fn resolve(root: &Path, rust_dir: &Path, fork: &Path) -> Llvm { + choose(root, rust_dir, fork, build_in_fork) +} + +/// [`resolve`] with the build that makes an LLVM passed in, so a test can stand +/// in for bootstrap: `build` builds in the fork checkout it is given and returns +/// the build directory, holding `/llvm` and `/lld`. +fn choose(root: &Path, rust_dir: &Path, fork: &Path, build: impl Fn(&Path) -> PathBuf) -> Llvm { + let key = key(fork); + let dir = store(rust_dir).join(&key); + let using = keystore::made(root, Keyed::Llvm, &store(rust_dir), &key, || defect(&dir), || place(fork, &key, &dir, &build)); + Llvm { dir, _using: using } +} + +/// Why `dir` is not a finished LLVM, if it is not. +fn defect(dir: &Path) -> Option { + if !dir.join(SOURCE).is_file() { + return Some(format!("{} carries no {SOURCE}", dir.display())); + } + let kept = KEPT.iter().map(|k| dir.join(k)).filter(|p| !p.is_dir()); + let tools = TOOLS.iter().map(|t| dir.join(t)).filter(|p| !p.is_file()); + let gone: Vec = kept.chain(tools).map(|p| p.display().to_string()).collect(); + (!gone.is_empty()).then(|| format!("{} carries no {}", dir.display(), gone.join(", "))) +} + +/// Refuse what `fork`'s `src/bootstrap` holds that no commit does: an LLVM is +/// keyed on the tree its commit records. +fn refuse_uncommitted_bootstrap(fork: &Path) { + let status = git_bytes(fork, &["status", "--porcelain", "--untracked-files=normal", "--", BOOTSTRAP]); + assert!( + status.is_empty(), + "{} holds changes no commit does, and an LLVM is keyed on the tree its commit records: \ + commit them, and the build makes the LLVM they name\n{}", + fork.join(BOOTSTRAP).display(), + String::from_utf8_lossy(&status), + ); +} + +/// Build the LLVM `key` names from `fork` and put it at `dir`. The caller holds +/// the key's lock. +fn place(fork: &Path, key: &str, dir: &Path, build: &impl Fn(&Path) -> PathBuf) { + eprintln!("Building LLVM {key} in {}: nobody on this host has", fork.display()); + let built = build(fork); + let host = host_triple(); + let partial = dir.with_extension("partial"); + if partial.exists() { + keystore::remove(&partial); + } + for part in KEPT { + clone_tree(&built.join(&host).join("llvm").join(part), &partial.join(part)); + } + let lld = built.join(&host).join("lld/bin/lld"); + fs::copy(&lld, partial.join("bin/lld")) + .unwrap_or_else(|e| panic!("copy {} -> {}: {e}", lld.display(), partial.join("bin/lld").display())); + // What was built is what the key names, or it is not that key's: the key + // refuses a bootstrap the build left holding what no commit does. + let again = self::key(fork); + assert!( + again == key, + "the fork's LLVM sources moved while LLVM {key} was being built (they now name {again}); \ + nothing was kept, and the next build makes the one they name" + ); + let checkout = fork.join(LLVM); + assert!(checkout.join(".git").exists(), "the LLVM build left no checkout at {}", checkout.display()); + let (built_from, commit) = (git_out(&checkout, &["rev-parse", "HEAD"]), llvm_commit(fork)); + // Bootstrap's `Llvm` step checks the gitlink's commit out before it builds, + // so a checkout behind it, the key never reads, is moved first. + assert!( + built_from.trim() == commit, + "{} is checked out at {}, and its gitlink names {commit}: bootstrap built the commit checked \ + out, which is not LLVM {key}'s; nothing was kept. `git -C {} submodule update {LLVM}` checks \ + the gitlink's commit out", + checkout.display(), + built_from.trim(), + fork.display(), + ); + fs::write(partial.join(SOURCE), format!("{key}\n")) + .unwrap_or_else(|e| panic!("write {}: {e}", partial.join(SOURCE).display())); + read_only(&partial); + keystore::retire(dir); + fs::rename(&partial, dir).unwrap_or_else(|e| panic!("rename {} -> {}: {e}", partial.display(), dir.display())); + fs::remove_dir_all(&built).unwrap_or_else(|e| panic!("remove {}: {e}", built.display())); +} + +/// Take write permission from every file and directory under `dir`, and from +/// `dir`. +fn read_only(dir: &Path) { + for entry in fs::read_dir(dir).unwrap_or_else(|e| panic!("read {}: {e}", dir.display())) { + let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", dir.display())).path(); + let meta = fs::symlink_metadata(&path).unwrap_or_else(|e| panic!("stat {}: {e}", path.display())); + if meta.is_dir() { + read_only(&path); + } else if meta.is_file() { + let mut permissions = meta.permissions(); + permissions.set_readonly(true); + fs::set_permissions(&path, permissions).unwrap_or_else(|e| panic!("chmod {}: {e}", path.display())); + } + } + let mut permissions = fs::metadata(dir).unwrap_or_else(|e| panic!("stat {}: {e}", dir.display())).permissions(); + permissions.set_readonly(true); + fs::set_permissions(dir, permissions).unwrap_or_else(|e| panic!("chmod {}: {e}", dir.display())); +} + +/// What the bootstrap build directory `build` holds of an LLVM of its own: +/// bootstrap's LLVM and LLD, and `download-ci-llvm`'s with its downloads, +/// whole or as a stopped removal left them. +pub fn in_tree(build: &Path) -> Vec { + let host = build.join(host_triple()); + let mut own: Vec = ["llvm", "lld", "ci-llvm"].iter().map(|d| host.join(d)).collect(); + let cache = build.join("cache"); + if cache.is_dir() { + let downloads: BTreeSet = fs::read_dir(&cache) + .unwrap_or_else(|e| panic!("read {}: {e}", cache.display())) + .map(|e| e.unwrap_or_else(|e| panic!("read {}: {e}", cache.display())).file_name()) + .map(|name| name.to_string_lossy().trim_end_matches(".swept").to_string()) + .filter(|name| name.starts_with("llvm-")) + .collect(); + own.extend(downloads.iter().map(|name| cache.join(name))); + } + own.retain(|dir| dir.exists() || dir.with_extension("swept").exists()); + own +} + +/// Remove [`in_tree`]. The caller holds the lock covering `build`, where +/// nothing builds an LLVM or downloads one any more. +pub fn retire_in_tree(build: &Path) { + for dir in in_tree(build) { + eprintln!("Removing {}: builds here link the host's LLVM or none", dir.display()); + keystore::retire(&dir); + } +} + +/// Bootstrap's build of LLVM, clang and LLD in `fork`, into its own build +/// directory, which it returns. +fn build_in_fork(fork: &Path) -> PathBuf { + let host = host_triple(); + let build_dir = fork.join("build/toyos-llvm"); + fs::create_dir_all(&build_dir).unwrap_or_else(|e| panic!("create {}: {e}", build_dir.display())); + let config = build_dir.join("bootstrap.toml"); + fs::write(&config, config_text(&build_dir, &host, host_tools())) + .unwrap_or_else(|e| panic!("write {}: {e}", config.display())); + let config = config.to_str().unwrap_or_else(|| panic!("{} is not UTF-8", config.display())); + let args = ["build", "--config", config, "src/llvm-project/llvm", "src/llvm-project/lld"]; + let (ok, _) = toolchain::x_build_with(fork, &args, "LLVM", clear); + assert!(ok, "the LLVM build in {} failed: its output above says why", fork.display()); + build_dir +} + +/// Bootstrap's configuration for an LLVM: the options every compiler build's +/// `[llvm]` names, no host library, and `tools`' compilers, for the host alone. +fn config_text(build_dir: &Path, host: &str, tools: &HostTools) -> String { + let off: Vec = NO_HOST_LIBRARIES.iter().map(|option| format!("{option} = \"OFF\"")).collect(); + format!( + r#"change-id = "ignore" +profile = "compiler" + +[build] +build-dir = "{build_dir}" +host = ["{host}"] +target = ["{host}"] + +[llvm] +{llvm} +build-config = {{ {off} }} + +[target.{host}] +cc = "{cc}" +cxx = "{cxx}" +"#, + build_dir = build_dir.display(), + llvm = crate::clang::LLVM_CONFIG, + off = off.join(", "), + cc = tools.cc.display(), + cxx = tools.cxx.display(), + ) +} + +#[cfg(test)] +mod tests { + use std::cell::Cell; + + use toyos_tmpdir::TempDir; + + use super::*; + use crate::buildlock; + use crate::compiler::tests::{estate, git, write, LLVM_B}; + + /// A scratch directory whose read-only LLVMs are made writable again before + /// it goes. + struct Scratch(TempDir); + + impl Scratch { + fn new(name: &str) -> Self { + Self(TempDir::new(name)) + } + } + + impl std::ops::Deref for Scratch { + type Target = Path; + fn deref(&self) -> &Path { + &self.0 + } + } + + impl Drop for Scratch { + fn drop(&mut self) { + keystore::writable(&self.0); + } + } + + /// Bootstrap's stand-in: what its LLVM and LLD builds leave in the build + /// directory, CMake's tree among them. + fn fake_build(fork: &Path) -> PathBuf { + let built = fork.join("build/toyos-llvm"); + let _ = fs::remove_dir_all(&built); + let install = built.join(host_triple()).join("llvm"); + for tool in ["llvm-config", "clang-22", "llvm-ar"] { + write(&install.join("bin").join(tool), &format!("the {tool}")); + } + std::os::unix::fs::symlink("clang-22", install.join("bin/clang")).unwrap(); + write(&install.join("include/llvm/Config/llvm-config.h"), "#define LLVM_VERSION_MAJOR 22"); + write(&install.join("lib/libLLVMCore.a"), "core"); + write(&install.join("lib/clang/22/include/stddef.h"), "typedef long ptrdiff_t;"); + write(&install.join("build/CMakeCache.txt"), "the build tree"); + write(&built.join(host_triple()).join("lld/bin/lld"), "the lld"); + built + } + + /// Check `fork`'s LLVM out at the commit `content` names, one commit in every + /// checkout whatever came before it: what bootstrap leaves once it has built + /// one. Returns it. + fn check_out_llvm(fork: &Path, content: &str) -> String { + let checkout = fork.join(LLVM); + if !checkout.join(".git").exists() { + git(&checkout, &["init", "-q"]); + } + write(&checkout.join("llvm/CMakeLists.txt"), content); + git(&checkout, &["add", "-A"]); + let tree = git(&checkout, &["write-tree"]); + let out = Command::new("git") + .args(["-c", "user.email=t@t", "-c", "user.name=t", "commit-tree", &tree, "-m", content]) + .envs([("GIT_AUTHOR_DATE", "2000-01-01T00:00:00Z"), ("GIT_COMMITTER_DATE", "2000-01-01T00:00:00Z")]) + .current_dir(&checkout) + .output() + .unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let commit = String::from_utf8(out.stdout).unwrap().trim().to_string(); + git(&checkout, &["reset", "-q", "--hard", &commit]); + commit + } + + /// [`check_out_llvm`], and that commit recorded as `fork`'s gitlink. + fn pin_llvm(fork: &Path, content: &str) { + let commit = check_out_llvm(fork, content); + git(fork, &["update-index", "--add", "--cacheinfo", &format!("160000,{commit},{LLVM}")]); + git(fork, &["commit", "-qm", &format!("LLVM {content}")]); + } + + /// `estate`, every fork's LLVM checked out at the one commit its gitlink + /// records. + fn estate_built(scratch: &Path) -> (PathBuf, PathBuf, [PathBuf; 3]) { + let (primary, rust_dir, forks) = estate(scratch); + for worktree in &forks { + pin_llvm(&worktree.join("rust"), "A"); + } + pin_llvm(&rust_dir, "A"); + (primary, rust_dir, forks) + } + + /// What `f` panicked with; `expect` if it returned. + fn refusal(expect: &str, f: impl FnOnce()) -> String { + let refused = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).expect_err(expect); + refused.downcast_ref::().cloned().unwrap_or_default() + } + + /// **One LLVM per key, made once and found again**: two worktrees whose + /// compilers differ and whose LLVM commit is one share one LLVM, and the + /// second build makes nothing and writes nothing; what is kept is the + /// install and its LLD, never CMake's tree, and the build directory goes. + #[test] + fn two_compilers_of_one_llvm_make_it_once() { + let scratch = Scratch::new("llvm"); + let (primary, rust_dir, [same, a, b]) = estate_built(&scratch); + let makes = Cell::new(0); + let once = |fork: &Path| { + makes.set(makes.get() + 1); + assert_eq!(makes.get(), 1, "an LLVM whose key was made was made again"); + fake_build(fork) + }; + + let la = choose(&a, &rust_dir, &a.join("rust"), once); + assert_eq!(makes.get(), 1); + assert_eq!(defect(&la.dir), None); + assert_eq!(fs::read_to_string(la.dir.join("bin/lld")).unwrap(), "the lld"); + assert_eq!(fs::read_link(la.dir.join("bin/clang")).unwrap(), Path::new("clang-22")); + assert!(la.dir.join("lib/clang/22/include/stddef.h").is_file()); + assert!(!la.dir.join("build").exists(), "CMake's tree was kept"); + assert!(!a.join("rust/build/toyos-llvm").exists(), "the build directory outlived the placement"); + + let before = snapshot(&store(&rust_dir)); + let lb = choose(&b, &rust_dir, &b.join("rust"), once); + let primary_s = choose(&primary, &rust_dir, &rust_dir, once); + let same_s = choose(&same, &rust_dir, &same.join("rust"), once); + assert_eq!(makes.get(), 1); + assert!(lb.dir == la.dir && primary_s.dir == la.dir && same_s.dir == la.dir, "one LLVM commit named two LLVMs"); + assert_eq!(snapshot(&store(&rust_dir)), before, "an LLVM was written after it was whole"); + } + + /// **A placed LLVM cannot be written**, through its own path or through a + /// link bootstrap makes to one of its files, and nothing in it can be + /// removed, replaced or added. + #[test] + fn a_placed_llvm_is_never_written() { + let scratch = Scratch::new("llvm-read-only"); + let (_primary, rust_dir, [_same, a, _b]) = estate_built(&scratch); + let dir = choose(&a, &rust_dir, &a.join("rust"), fake_build).dir; + let stage = scratch.join("stage1-rust-lld"); + fs::hard_link(dir.join("bin/lld"), &stage).unwrap(); + let denied = |what: &str, done: std::io::Result<()>| { + assert_eq!(done.map_err(|e| e.kind()).err(), Some(std::io::ErrorKind::PermissionDenied), "{what}"); + }; + for file in [dir.join("bin/lld"), stage, dir.join("lib/libLLVMCore.a"), dir.join(SOURCE)] { + denied(&format!("{} could be written", file.display()), fs::OpenOptions::new().write(true).open(&file).map(drop)); + } + denied("a placed tool could be removed", fs::remove_file(dir.join("bin/lld"))); + denied("a file could be added to a placed LLVM", fs::write(dir.join("bin/new"), "x")); + denied("a placed LLVM could be emptied", fs::remove_dir_all(dir.join("include"))); + } + + /// **An LLVM that is not whole is made again, all of it**: one whose + /// `SOURCE` says it finished and that lost a tool, or a directory it keeps, + /// is replaced. + #[test] + fn an_llvm_that_is_not_whole_is_made_again() { + let scratch = Scratch::new("llvm-whole"); + let (_primary, rust_dir, [_same, a, _b]) = estate_built(&scratch); + let makes = Cell::new(0); + let counted = |fork: &Path| { + makes.set(makes.get() + 1); + fake_build(fork) + }; + let dir = choose(&a, &rust_dir, &a.join("rust"), counted).dir; + for (lost, made) in [("bin/lld", 2), ("lib", 3)] { + keystore::writable(&dir); + let lost = dir.join(lost); + if lost.is_dir() { + fs::remove_dir_all(&lost).unwrap(); + } else { + fs::remove_file(&lost).unwrap(); + } + assert!(defect(&dir).is_some_and(|d| d.contains(&lost.display().to_string())), "{:?}", defect(&dir)); + let again = choose(&a, &rust_dir, &a.join("rust"), counted); + assert_eq!((makes.get(), defect(&again.dir)), (made, None)); + } + } + + /// **The key is the LLVM and nothing else**: the same inputs give the same + /// key; each of the recipe, the configuration (its `[llvm]` and its host), + /// the host's tools, the committed LLVM gitlink and the committed + /// `src/bootstrap` moves it; a compiler edit does not. + #[test] + fn the_key_moves_with_the_llvm_and_only_with_it() { + let scratch = Scratch::new("llvm-key"); + let (_primary, rust_dir, [same, a, _b]) = estate(&scratch); + let fork = same.join("rust"); + let tools = host_tools(); + let config = config_text(Path::new(KEYED_BUILD_DIR), &host_triple(), tools); + let base = key(&fork); + assert_eq!(key_of(&fork, RECIPE, &config, &tools.identity), base); + let linux = config_text(Path::new(KEYED_BUILD_DIR), "x86_64-unknown-linux-gnu", tools); + for (what, other) in [ + ("the recipe", key_of(&fork, "another recipe", &config, &tools.identity)), + ("the [llvm]", key_of(&fork, RECIPE, &config.replace("X86", "RISCV;X86"), &tools.identity)), + ("the host", key_of(&fork, RECIPE, &linux, &tools.identity)), + ("the host's tools", key_of(&fork, RECIPE, &config, "/usr/bin/gcc\ngcc 14\n")), + ] { + assert_ne!(other, base, "{what} did not move the key"); + } + + assert_eq!(key(&fork), key(&rust_dir), "one LLVM commit named two keys"); + assert_eq!(key(&fork), key(&a.join("rust")), "a compiler/ edit moved the LLVM key"); + write(&fork.join("compiler/rustc_target/src/new_target.rs"), "pub fn t() {}\n"); + git(&fork, &["add", "-A"]); + git(&fork, &["commit", "-qm", "a target"]); + assert_eq!(key(&fork), base, "a compiler/ commit moved the LLVM key"); + + git(&fork, &["update-index", "--add", "--cacheinfo", &format!("160000,{LLVM_B},{LLVM}")]); + git(&fork, &["commit", "-qm", "another LLVM"]); + let moved = key(&fork); + assert_ne!(moved, base, "another LLVM commit kept the key"); + + write(&fork.join("src/bootstrap/src/core/build_steps/llvm.rs"), "cfg.define(\"LLVM_ENABLE_ZLIB\", \"OFF\");\n"); + git(&fork, &["add", "-A"]); + git(&fork, &["commit", "-qm", "another LLVM step"]); + assert_ne!(key(&fork), moved, "another LLVM step in bootstrap kept the key"); + } + + /// **The tools the key names are the host's**: the C and C++ compilers the + /// configuration names by path, and CMake, each by the file it resolves to + /// and what its `--version` says; on macOS, the SDK. + #[test] + fn the_key_names_the_host_s_tools() { + let tools = host_tools(); + let lines: Vec<&str> = tools.identity.lines().collect(); + for tool in [&tools.cc, &tools.cxx] { + assert!(tool.is_absolute() && tool.is_file(), "{} is no compiler", tool.display()); + let at = lines.iter().position(|l| Path::new(l) == tool.as_path()).unwrap_or_else(|| panic!("{}", tools.identity)); + assert!(lines[at + 1].contains("version"), "{} said no version: {}", tool.display(), tools.identity); + } + assert!(lines.iter().any(|l| l.starts_with("cmake version")), "{}", tools.identity); + if host_triple().ends_with("apple-darwin") { + assert!(lines.iter().any(|l| l.ends_with(".sdk") && Path::new(l).is_dir()), "no SDK: {}", tools.identity); + } + let config = config_text(Path::new(KEYED_BUILD_DIR), "h", tools); + let named = format!("[target.h]\ncc = \"{}\"\ncxx = \"{}\"\n", tools.cc.display(), tools.cxx.display()); + assert!(config.contains(&named), "{config}"); + } + + /// **Two SDK versions are two LLVMs**, though both answer one SDK path, as + /// the unversioned `MacOSX.sdk` a Command Line Tools update moves does. + #[test] + #[cfg(target_os = "macos")] + fn two_sdk_versions_are_two_keys() { + let scratch = Scratch::new("llvm-sdk"); + let (_primary, _rust_dir, [same, _a, _b]) = estate(&scratch); + let fork = same.join("rust"); + let [older, newer] = ["26.0", "27.0"].map(|version| { + let tools = tools_with(|question| match question { + "--show-sdk-path" => "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk\n".to_string(), + "--show-sdk-version" => format!("{version}\n"), + other => panic!("xcrun was asked {other}"), + }); + key_of(&fork, RECIPE, &config_text(Path::new(KEYED_BUILD_DIR), &host_triple(), &tools), &tools.identity) + }); + assert_ne!(older, newer, "two SDK versions at one SDK path named one LLVM"); + } + + /// **Every host library LLVM looks for is turned off by name**, in the + /// `build-config` bootstrap passes to CMake after its own options. + #[test] + fn the_llvm_config_turns_every_host_library_off() { + let config = config_text(Path::new(KEYED_BUILD_DIR), "h", host_tools()); + let off = "build-config = { LLVM_ENABLE_ZLIB = \"OFF\", LLVM_ENABLE_ZSTD = \"OFF\", \ + LLVM_ENABLE_LIBXML2 = \"OFF\", CLANG_ENABLE_LIBXML2 = \"OFF\", LLVM_ENABLE_LIBEDIT = \"OFF\", \ + LLVM_ENABLE_LIBPFM = \"OFF\", LLVM_ENABLE_FFI = \"OFF\", LLVM_ENABLE_CURL = \"OFF\", \ + LLVM_ENABLE_HTTPLIB = \"OFF\", LLVM_ENABLE_ICU = \"OFF\", LLVM_ENABLE_ICONV = \"OFF\", \ + LLVM_ENABLE_Z3_SOLVER = \"OFF\" }\n"; + assert!(config.contains(&format!("\n{off}")), "{config}"); + } + + /// **A checkout behind its gitlink names the gitlink's LLVM**: the key + /// reads the gitlink and never the checkout, and the build, which checks the + /// gitlink's commit out first as bootstrap's `Llvm` step does, stores it. + #[test] + fn a_checkout_behind_its_gitlink_is_built_at_the_gitlink() { + let scratch = Scratch::new("llvm-behind"); + let (_primary, rust_dir, [same, a, _b]) = estate_built(&scratch); + let (fork, at_gitlink) = (a.join("rust"), same.join("rust")); + for fork in [&fork, &at_gitlink] { + pin_llvm(fork, "B"); + } + check_out_llvm(&fork, "A"); + assert_eq!(key(&fork), key(&at_gitlink), "a checkout behind its gitlink moved the key"); + let updating = |fork: &Path| { + check_out_llvm(fork, "B"); + fake_build(fork) + }; + let llvm = choose(&a, &rust_dir, &fork, updating); + assert_eq!((llvm.dir.clone(), defect(&llvm.dir)), (store(&rust_dir).join(key(&at_gitlink)), None)); + } + + const FORK: &str = "TOYOS_LLVM_TEST_FORK"; + + /// What a caller's environment may hold that would reach an LLVM build: + /// flags, compilers, tools, and the SDK and deployment target. + const AMBIENT: [(&str, &str); 11] = [ + ("CFLAGS", "-O0"), + ("CXXFLAGS", "-O0"), + ("LDFLAGS", "-Wl,-no-such-flag"), + ("CC", "/no/such/cc"), + ("CXX", "/no/such/c++"), + ("AR", "/no/such/ar"), + ("RANLIB", "/no/such/ranlib"), + ("SDKROOT", "/no/such.sdk"), + ("MACOSX_DEPLOYMENT_TARGET", "10.9"), + ("CMAKE", "/no/such/cmake"), + ("CMAKE_TOOLCHAIN_FILE", "/no/such/toolchain.cmake"), + ]; + + /// **The caller's environment reaches neither the LLVM build nor its key**: + /// a process holding every [`AMBIENT`] name keys the LLVM as this one does, + /// and the bootstrap it runs, a script that writes down its environment, + /// sees nothing but `PATH`, `TMPDIR` and what a shell sets itself, named + /// here and not read from [`ENVIRONMENT`]. + #[test] + fn the_caller_s_environment_reaches_neither_the_build_nor_the_key() { + use std::os::unix::fs::PermissionsExt; + let scratch = Scratch::new("llvm-environment"); + let (_primary, _rust_dir, [same, _a, _b]) = estate(&scratch); + let fork = same.join("rust"); + write(&fork.join("library/Cargo.lock"), "# lock\n"); + write(&fork.join("x"), "#!/bin/sh\nenv > build/toyos-llvm/environment\n"); + fs::set_permissions(fork.join("x"), fs::Permissions::from_mode(0o755)).unwrap(); + + let out = buildlock::tests::rerun("llvm::tests::keyed_and_built").env(FORK, &fork).envs(AMBIENT).output().unwrap(); + assert!(out.status.success(), "{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)); + let built = fork.join("build/toyos-llvm"); + assert_eq!(fs::read_to_string(built.join("key")).unwrap(), key(&fork), "the caller's environment moved the key"); + let seen = fs::read_to_string(built.join("environment")).unwrap(); + let allowed = ["PATH", "TMPDIR", "BOOTSTRAP_SKIP_TARGET_SANITY", "PWD", "OLDPWD", "SHLVL", "_"]; + for name in seen.lines().filter_map(|l| l.split_once('=')).map(|(name, _)| name) { + assert!(allowed.contains(&name), "the build saw {name}: {seen}"); + } + assert!(seen.lines().any(|l| l.starts_with("PATH=")), "the build saw no PATH: {seen}"); + } + + /// The process [`the_caller_s_environment_reaches_neither_the_build_nor_the_key`] + /// runs: the key of the fork in [`FORK`] and its build, the key written + /// beside what the build wrote. + #[test] + #[ignore = "the process the environment test runs; never runs on its own"] + fn keyed_and_built() { + let fork = PathBuf::from(std::env::var(FORK).unwrap_or_else(|_| panic!("keyed_and_built ran without {FORK}; it is not a test"))); + let key = key(&fork); + let built = build_in_fork(&fork); + fs::write(built.join("key"), key).unwrap(); + } + + /// **A gitlink staged and not committed names no LLVM**: bootstrap checks + /// out the index's, and the key would name HEAD's. + #[test] + fn a_staged_llvm_gitlink_is_refused() { + let scratch = Scratch::new("llvm-staged"); + let (_primary, _rust_dir, [same, _a, _b]) = estate(&scratch); + let fork = same.join("rust"); + git(&fork, &["update-index", "--add", "--cacheinfo", &format!("160000,{LLVM_B},{LLVM}")]); + let said = refusal("a staged gitlink named an LLVM", || { + key(&fork); + }); + assert!(said.contains("stages") && said.contains(LLVM_B), "{said}"); + } + + /// **An uncommitted `src/bootstrap` names no LLVM**, not even one already + /// stored: the key refuses it, and nothing is built or resolved. + #[test] + fn an_uncommitted_bootstrap_names_no_llvm() { + let scratch = Scratch::new("llvm-dirty-key"); + let (_primary, rust_dir, [_same, a, _b]) = estate_built(&scratch); + let fork = a.join("rust"); + drop(choose(&a, &rust_dir, &fork, fake_build)); + write(&fork.join("src/bootstrap/src/core/build_steps/llvm.rs"), "cfg.define(\"LLVM_ENABLE_ZLIB\", \"ON\");\n"); + let never = |_: &Path| -> PathBuf { panic!("an LLVM was built from a bootstrap no commit holds") }; + let said = refusal("an uncommitted bootstrap edit resolved to the stored LLVM", || { + choose(&a, &rust_dir, &fork, never); + }); + assert!(said.contains("src/bootstrap holds changes no commit does"), "{said}"); + let said = refusal("an uncommitted bootstrap edit named an LLVM", || { + key(&fork); + }); + assert!(said.contains("src/bootstrap holds changes no commit does"), "{said}"); + } + + /// **Only an LLVM built from what its key names is stored**: not from a + /// `src/bootstrap` holding what no commit does, which is refused before + /// anything is built, and after, when the build left it so; not from an + /// LLVM checkout other than the gitlink's; not when the sources moved while + /// it was being built. + #[test] + fn what_the_key_does_not_name_is_never_stored() { + let scratch = Scratch::new("llvm-dirt"); + let (_primary, rust_dir, [_same, a, _b]) = estate_built(&scratch); + let fork = a.join("rust"); + let step = fork.join("src/bootstrap/src/core/build_steps/llvm.rs"); + let never = |_: &Path| -> PathBuf { panic!("an LLVM was built from a bootstrap no commit holds") }; + + write(&step, "cfg.define(\"LLVM_ENABLE_ZLIB\", \"OFF\");\n"); + let said = refusal("an uncommitted bootstrap edit was built", || { + choose(&a, &rust_dir, &fork, never); + }); + assert!(said.contains("src/bootstrap holds changes no commit does"), "{said}"); + git(&fork, &["add", "-A"]); + git(&fork, &["commit", "-qm", "the step"]); + + let dirtying = |fork: &Path| { + write(&fork.join("src/bootstrap/src/core/build_steps/llvm.rs"), "cfg.define(\"LLVM_ENABLE_ZSTD\", \"ON\");\n"); + fake_build(fork) + }; + let said = refusal("an LLVM built while its bootstrap was edited was stored", || { + choose(&a, &rust_dir, &fork, dirtying); + }); + assert!(said.contains("src/bootstrap holds changes no commit does"), "{said}"); + git(&fork, &["checkout", "-q", "--", "src/bootstrap"]); + + let lagging = |fork: &Path| { + check_out_llvm(fork, "A"); + fake_build(fork) + }; + pin_llvm(&fork, "B"); + let said = refusal("an LLVM checkout other than the gitlink's was stored", || { + choose(&a, &rust_dir, &fork, lagging); + }); + assert!(said.contains("is checked out at") && said.contains("submodule update src/llvm-project"), "{said}"); + check_out_llvm(&fork, "B"); + + let moving = |fork: &Path| { + write(&step, "cfg.define(\"LLVM_ENABLE_ZSTD\", \"OFF\");\n"); + git(fork, &["commit", "-qam", "moved while built"]); + fake_build(fork) + }; + let said = refusal("an LLVM whose sources moved while it was built was stored", || { + choose(&a, &rust_dir, &fork, moving); + }); + assert!(said.contains("moved while LLVM"), "{said}"); + + let stored: Vec<_> = fs::read_dir(store(&rust_dir)).unwrap().flatten().map(|e| e.file_name()).collect(); + assert!(stored.iter().all(|n| n.to_string_lossy().ends_with(".partial")), "stored: {stored:?}"); + } + + const WORKTREE: &str = "TOYOS_LLVM_TEST_WORKTREE"; + const RUST_DIR: &str = "TOYOS_LLVM_TEST_RUST_DIR"; + const ROLE: &str = "TOYOS_LLVM_TEST_ROLE"; + + /// The competing process for the tests below: the LLVM the worktree in + /// [`WORKTREE`] names, held in use until released — or, as `make`, held + /// while it is being made. + #[test] + #[ignore = "the competing process for the tests below; never runs on its own"] + fn child_role() { + let worktree = PathBuf::from(std::env::var(WORKTREE).unwrap_or_else(|_| panic!("child_role ran without {WORKTREE}; it is not a test"))); + let rust_dir = PathBuf::from(std::env::var(RUST_DIR).unwrap()); + match std::env::var(ROLE).unwrap().as_str() { + "use" => { + let _held = choose(&worktree, &rust_dir, &worktree.join("rust"), fake_build); + buildlock::tests::hold_until_released(); + } + "make" => { + let held = |fork: &Path| { + buildlock::tests::hold_until_released(); + fake_build(fork) + }; + choose(&worktree, &rust_dir, &worktree.join("rust"), held); + } + other => panic!("unknown child role {other}"), + } + } + + fn elsewhere(role: &str, worktree: &Path, rust_dir: &Path) -> buildlock::tests::Elsewhere { + let env = [(ROLE, std::ffi::OsStr::new(role)), (WORKTREE, worktree.as_os_str()), (RUST_DIR, rust_dir.as_os_str())]; + buildlock::tests::Elsewhere::hold("llvm::tests::child_role", &env) + } + + /// **An LLVM another process is making is waited for, not made again.** + #[test] + fn an_llvm_being_made_elsewhere_is_not_made_again() { + let scratch = Scratch::new("llvm-made-elsewhere"); + let (primary, rust_dir, [_same, _a, b]) = estate_built(&scratch); + let maker = elsewhere("make", &b, &rust_dir); + let made = key(&b.join("rust")); + assert!(buildlock::keyed_idle(&primary, Keyed::Llvm, &made).is_none(), "a sweep could take an LLVM being made"); + maker.release(); + let never = |_: &Path| -> PathBuf { panic!("an LLVM another process made was made here too") }; + assert_eq!(defect(&choose(&b, &rust_dir, &b.join("rust"), never).dir), None); + } + + /// **A sweep takes an LLVM only once no worktree names it and nobody uses + /// it**: `a` moves to another LLVM while a process of its own still uses + /// the first, and then `b` names the first until it moves too. + #[test] + fn an_llvm_is_swept_once_nobody_names_or_uses_it() { + let scratch = Scratch::new("llvm-sweep"); + let (primary, rust_dir, [_same, a, b]) = estate_built(&scratch); + let user = elsewhere("use", &a, &rust_dir); + let first = store(&rust_dir).join(key(&a.join("rust"))); + + let fork = a.join("rust"); + pin_llvm(&fork, "B"); + let second = choose(&a, &rust_dir, &fork, fake_build); + assert_ne!(second.dir, first); + assert!(first.is_dir(), "placing an LLVM swept one still in use"); + + keystore::record(&b, Keyed::Llvm, &key(&b.join("rust"))); + user.release(); + let swept = |root: &Path| keystore::sweep(root, Keyed::Llvm, &store(&rust_dir)); + assert_eq!(swept(&primary), Vec::::new(), "the sweep took an LLVM a worktree names"); + keystore::record(&b, Keyed::Llvm, &key(&fork)); + assert_eq!(swept(&primary), [first], "the sweep kept an LLVM nobody names, or took a named one"); + assert!(second.dir.is_dir()); + } + + /// **A sweep stopped halfway leaves no LLVM that passes for whole**: the + /// entry is renamed away from its key before anything in it is removed, and + /// the next sweep takes what the stopped one left. + #[test] + fn a_stopped_sweep_leaves_no_llvm_that_passes_for_whole() { + let scratch = Scratch::new("llvm-sweep-stopped"); + let (primary, rust_dir, [_same, a, _b]) = estate_built(&scratch); + let dir = choose(&a, &rust_dir, &a.join("rust"), fake_build).dir; + keystore::forget(&a, Keyed::Llvm); + let halfway = |path: &Path| { + keystore::writable(path); + fs::remove_file(path.join("lib/libLLVMCore.a")).unwrap(); + panic!("stopped"); + }; + let stopped = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + keystore::sweep_by(&primary, Keyed::Llvm, &store(&rust_dir), halfway) + })); + assert!(stopped.is_err(), "the stand-in removal was never asked"); + assert!(defect(&dir).is_some(), "a stopped sweep left {} passing for whole", dir.display()); + assert_eq!(keystore::sweep(&primary, Keyed::Llvm, &store(&rust_dir)), [dir.with_extension("swept")]); + } + + /// **An LLVM a worktree resolved stays once nothing uses it**: its record, + /// not its use, is what names it. + #[test] + fn a_resolved_llvm_is_named_by_its_worktree() { + let scratch = Scratch::new("llvm-named"); + let (primary, rust_dir, [_same, a, _b]) = estate_built(&scratch); + elsewhere("use", &a, &rust_dir).release(); + let dir = store(&rust_dir).join(key(&a.join("rust"))); + assert_eq!(keystore::sweep(&primary, Keyed::Llvm, &store(&rust_dir)), Vec::::new()); + assert_eq!(defect(&dir), None, "the sweep took an LLVM the worktree that resolved it names"); + } + + /// **A build directory whose compiler links the host's LLVM keeps none of + /// its own**: bootstrap's LLVM and LLD, `download-ci-llvm`'s and its + /// downloads go, and what a stopped removal left; the rest stays. + #[test] + fn a_build_directory_keeps_no_llvm_of_its_own() { + let build = TempDir::new("llvm-in-tree"); + let host = build.join(host_triple()); + for file in ["llvm/bin/llvm-config", "lld/bin/lld", "ci-llvm/lib/libLLVM.dylib", "llvm.swept/bin/clang", "stage2/bin/rustc"] { + write(&host.join(file), "x"); + } + for file in ["cache/llvm-1111-false/rust-dev.tar.xz", "cache/llvm-2222-false.swept/rust-dev.tar.xz", "cache/2026-07-13/rustc.tar.xz"] { + write(&build.join(file), "x"); + } + retire_in_tree(&build); + for gone in ["llvm", "lld", "ci-llvm", "llvm.swept"] { + assert!(!host.join(gone).exists(), "{gone} stayed"); + } + let cache: Vec<_> = fs::read_dir(build.join("cache")).unwrap().flatten().map(|e| e.file_name()).collect(); + assert_eq!(cache, ["2026-07-13"]); + assert!(host.join("stage2/bin/rustc").is_file()); + } + + /// Every file under `dir` with its bytes, and every link with its target. + fn snapshot(dir: &Path) -> Vec<(PathBuf, Vec)> { + let mut out = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(at) = stack.pop() { + for entry in fs::read_dir(&at).unwrap().flatten() { + let path = entry.path(); + let meta = fs::symlink_metadata(&path).unwrap(); + if meta.file_type().is_symlink() { + out.push((path.clone(), fs::read_link(&path).unwrap().into_os_string().into_encoded_bytes())); + } else if meta.is_dir() { + stack.push(path); + } else { + out.push((path.clone(), fs::read(&path).unwrap())); + } + } + } + out.sort(); + out + } +} diff --git a/src/main.rs b/src/main.rs index 4aa5b32c73b..2b95eaeea75 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,28 +21,28 @@ const REQUIRED: &[Tool] = &[ Tool { any: &["git"], why: "every build; the image ships what git says is tracked" }, Tool { any: &["rustup"], why: "the toolchain — install from https://rustup.rs" }, Tool { any: &["cc"], why: "rustc links every host binary through it; no guest binary" }, + Tool { + any: &["cmake"], + why: "every build keys the host's LLVM on its `--version`, and rustc's bootstrap \ + configures LLVM and clang with it; `brew install cmake` on macOS", + }, ]; /// Named, because a list that stops at what is fatal reads as the whole list. /// Each of these costs one thing when absent rather than the build, so none of /// them exits. /// -/// CMake and Ninja are what rustc's bootstrap builds LLVM and clang from -/// `rust/src/llvm-project` with, which it does only when that commit has not -/// been built here. Host tools like `cc`, and never in a guest: on macOS from -/// Homebrew, on CI's toolchain runner at the versions `.github/workflows` -/// pins. They go when the build no longer needs a host. +/// Ninja is what rustc's bootstrap builds LLVM and clang from +/// `rust/src/llvm-project` with, under CMake, which it does only when this +/// host has not built that LLVM. Both are host tools like `cc`, and never in a +/// guest: on macOS from Homebrew, on CI's toolchain runner at the versions +/// `.github/workflows` pins. They go when the build no longer needs a host. const ALSO_USED: &[Tool] = &[ Tool { any: &["python3", "python", "py", "python2", "uv"], why: "rust/x runs rustc's bootstrap, which is Python — a clean clone and \ every toolchain change need one", }, - Tool { - any: &["cmake"], - why: "rustc's bootstrap configures LLVM and clang with it — every toolchain build \ - whose LLVM commit this host has not built; `brew install cmake` on macOS", - }, Tool { any: &["ninja"], why: "rustc's bootstrap builds LLVM and clang with it, under CMake; `brew install \ diff --git a/src/release.rs b/src/release.rs index de7c014234c..c1f998bef55 100644 --- a/src/release.rs +++ b/src/release.rs @@ -239,7 +239,7 @@ fn build(root: &Path, tag: &str, tmp: &Path) -> Result<(), String> { // the compiler with the guest libraries and `libtoyos_c.a` this tree's // sources name, recorded beside the witness an installer checks it by. let build = root.join("rust/build"); - let key = crate::sysroot::recorded_key(root).ok_or("the build recorded no sysroot key")?; + let key = crate::keystore::recorded(root, crate::buildlock::Keyed::Sysroot).ok_or("the build recorded no sysroot key")?; let sysroot = format!("sysroots/{key}"); let stage2 = build.join(&sysroot); fs::write(build.join("toyos-sysroot-witness"), crate::sysroot::witness(root)) diff --git a/src/sourcegate.rs b/src/sourcegate.rs index e3e02dd9bcf..61ad06f95f4 100644 --- a/src/sourcegate.rs +++ b/src/sourcegate.rs @@ -621,6 +621,20 @@ const HOST_SPAWNS: &[Spawn] = &[ whose whole job is to find a Python — the standing failure \ issues/build/python-and-cc-are-declared.md is about", }, + Spawn { + arg: "tool", + sites: &[("src/llvm.rs", 1)], + why: "the host's `cc`, `c++` and `cmake`, asked their `--version` for the key of the \ + LLVM they build: the `cc` and CMake standing failure \ + issues/build/python-and-cc-are-declared.md is about", + }, + Spawn { + arg: "\"xcrun\"", + sites: &[], + why: "a macOS binary, asked which SDK the host's `cc` compiles against, for the key of \ + the LLVM it builds: the host C toolchain's identity, under the same `cc` standing \ + failure issues/build/python-and-cc-are-declared.md is about", + }, Spawn { arg: "\"./x.py\"", sites: &[], @@ -712,7 +726,7 @@ const CI_PACKAGES: &[Package] = &[ Package { name: "cmake", why: "CMake, which rustc's bootstrap configures LLVM and clang with — a declared host \ - tool (`ALSO_USED` in src/main.rs, issues/build/python-and-cc-are-declared.md), \ + tool (`REQUIRED` in src/main.rs, issues/build/python-and-cc-are-declared.md), \ unpinned where a portability job installs its platform's own", }, Package { diff --git a/src/sysroot.rs b/src/sysroot.rs index e6220f123fa..ba247abb204 100644 --- a/src/sysroot.rs +++ b/src/sysroot.rs @@ -29,7 +29,7 @@ //! exclusively (its fork build directory is written); then, if the compiler is //! the primary's, the global one shared, because it is read. //! -//! A sysroot no worktree names any more is removed by [`sweep`], which +//! A sysroot no worktree names any more is removed by `keystore::sweep`, which //! `--worktree remove` runs: each build records the key it used in its //! worktree's `target/`, and a key no registered worktree records, that nobody //! is making or using, goes. @@ -62,12 +62,9 @@ const SOURCES: &str = "SOURCES"; /// What changes how a key's sources become a sysroot and is none of them: the /// std build's recipe below. Moving it moves every key. -const RECIPE: &str = "bootstrap stage-0 local rebuild, profile compiler, \ +const RECIPE: &str = "bootstrap stage-0 local rebuild, profile compiler, no LLVM, \ libtoyos_c merged, libraries from the stamp, linked by rust-lld, \ - a C sysroot of libc's staticlib and headers per target; 4"; - -/// Where each build records the key it compiled against, for [`sweep`]. -const RECORD: &str = "target/toyos-sysroot-key"; + a C sysroot of libc's staticlib and headers per target; 5"; /// Every sysroot on this host. pub fn sysroots_dir(rust_dir: &Path) -> PathBuf { @@ -294,11 +291,6 @@ pub fn fork_checkout(root: &Path) -> PathBuf { fork } -/// The key `root`'s last build compiled against. -pub fn recorded_key(root: &Path) -> Option { - fs::read_to_string(root.join(RECORD)).ok().map(|k| k.trim().to_string()) -} - /// Why `dir` is not a finished sysroot, if it is not: no [`SOURCES`], or not a /// whole toolchain (`toolchain::toolchain_defect`). One found with the first and /// not the second is made again rather than trusted — all of it even when only @@ -324,9 +316,7 @@ pub fn ensure(root: &Path, rust_dir: &Path, lock: &mut Held) -> Sysroot { let compiler = compiler::resolve(root, rust_dir, &fork, lock); let key = key(root, &compiler, &fork); let dir = sysroots_dir(rust_dir).join(&key); - let record = root.join(RECORD); - fs::create_dir_all(record.parent().expect("a file under target/")).ok(); - fs::write(&record, &key).unwrap_or_else(|e| panic!("write {}: {e}", record.display())); + crate::keystore::record(root, Keyed::Sysroot, &key); let using = lock.without_shared(|| held(root, &key, &dir, || build(root, &compiler, &fork, &key, &dir))); Sysroot { dir, primary_compiler: compiler.primary, _using: Some(using) } @@ -400,15 +390,9 @@ fn build_std(root: &Path, compiler: &Compiler, fork: &Path) -> PathBuf { crate::ensure_submodule(fork, "library/backtrace"); let host = host_triple(); let build_dir = fork.join("build/toyos-std"); - fs::create_dir_all(&build_dir).unwrap_or_else(|e| panic!("create {}: {e}", build_dir.display())); - forget_another_compiler(&build_dir, &host, &compiler.identity()); - // Bootstrap reuses what it built before and does not see a path dependency - // outside the fork move, so each target's std starts from nothing. - for target in GUEST_TARGETS { - remove(&build_dir.join(&host).join("stage0-std").join(target)); - } + prepare_std_build(&build_dir, &host, &compiler.identity()); let config = build_dir.join("bootstrap.toml"); - fs::write(&config, std_config(&compiler.stage2, &build_dir, &host)) + fs::write(&config, std_config(&compiler.stage2, &bootstrap_cargo(), &build_dir, &host)) .unwrap_or_else(|e| panic!("write {}: {e}", config.display())); let targets = GUEST_TARGETS.join(","); @@ -423,6 +407,20 @@ fn build_std(root: &Path, compiler: &Compiler, fork: &Path) -> PathBuf { build_dir.join(&host).join("stage0-std") } +/// Ready the std build directory `build_dir` for a build by the compiler +/// `identity` names: nothing another compiler built, no LLVM, and no guest +/// target's std. +fn prepare_std_build(build_dir: &Path, host: &str, identity: &str) { + fs::create_dir_all(build_dir).unwrap_or_else(|e| panic!("create {}: {e}", build_dir.display())); + forget_another_compiler(build_dir, host, identity); + crate::llvm::retire_in_tree(build_dir); + // Bootstrap reuses what it built before and does not see a path dependency + // outside the fork move, so each target's std starts from nothing. + for target in GUEST_TARGETS { + remove(&build_dir.join(host).join("stage0-std").join(target)); + } +} + /// Empty the std build directory `build_dir` of all but what bootstrap /// downloaded unless `identity` ([`Compiler::identity`]) is the compiler its /// `compiled-by` records as having compiled the rest, then record `identity` @@ -438,7 +436,7 @@ fn forget_another_compiler(build_dir: &Path, host: &str, identity: &str) { return; } let kept = [(build_dir.to_path_buf(), &["cache", host, "compiled-by"][..]), - (build_dir.join(host), &["ci-llvm", "rustfmt"][..])]; + (build_dir.join(host), &["rustfmt"][..])]; for (dir, kept) in kept { let entries = match fs::read_dir(&dir) { Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, @@ -520,7 +518,8 @@ fn place_std(stamp: &Path, lib: &Path) { /// The linker is the compiler's own `rust-lld`, named by path so that which sysroot /// a stage-0 build searches for tools decides nothing; and no rpath, which bootstrap /// spells as a C driver's `-Wl,` arguments that a linker run directly refuses. -fn std_config(compiler: &Path, build_dir: &Path, host: &str) -> String { +/// No LLVM: std builds none, and the profile's `download-ci-llvm` fetches one. +fn std_config(compiler: &Path, cargo: &Path, build_dir: &Path, host: &str) -> String { let targets = GUEST_TARGETS.iter().map(|t| format!("\"{t}\"")).collect::>().join(", "); let linker = toolchain::rust_lld(compiler); let userland: String = Arch::ALL @@ -539,11 +538,14 @@ build-dir = "{build_dir}" host = ["{host}"] target = [{targets}] +[llvm] +download-ci-llvm = false + [rust] lld = false {userland}"#, rustc = compiler.join("bin/rustc").display(), - cargo = bootstrap_cargo().display(), + cargo = cargo.display(), build_dir = build_dir.display(), ) } @@ -595,36 +597,6 @@ pub(crate) fn clone_tree(from: &Path, to: &Path) { } } -/// Remove every sysroot no registered worktree records and nobody is making -/// or using, and every half-built one nobody is making. Returns what went. -pub fn sweep(root: &Path) -> Vec { - let rust_dir = toolchain::rust_dir(root); - let dir = sysroots_dir(&rust_dir); - let Ok(entries) = fs::read_dir(&dir) else { return Vec::new() }; - let named: BTreeSet = git_out(root, &["worktree", "list", "--porcelain"]) - .lines() - .filter_map(|l| l.strip_prefix("worktree ")) - .filter_map(|w| fs::read_to_string(Path::new(w).join(RECORD)).ok()) - .map(|k| k.trim().to_string()) - .collect(); - let mut removed = Vec::new(); - for entry in entries.flatten() { - let name = entry.file_name().to_string_lossy().into_owned(); - let (key, whole) = match name.split_once('.') { - Some((key, _)) => (key.to_string(), false), - None => (name.clone(), true), - }; - if whole && named.contains(&key) { - continue; - } - let Some(_idle) = buildlock::keyed_idle(root, Keyed::Sysroot, &key) else { continue }; - let path = entry.path(); - fs::remove_dir_all(&path).unwrap_or_else(|e| panic!("remove {}: {e}", path.display())); - removed.push(path); - } - removed -} - fn path_str(path: &Path) -> &str { path.to_str().unwrap_or_else(|| panic!("{} is not UTF-8", path.display())) } @@ -786,10 +758,10 @@ mod tests { build.join("host/a-directory-bootstrap-adds/lib.rlib"), build.join("tmp/cc-rs-out-dir/out.o"), build.join("host/a-stamp-bootstrap-writes"), + build.join("host/ci-llvm/lib/libLLVM.dylib"), ]; let downloaded = [ - build.join("cache/llvm-1/llvm.tar.xz"), - build.join("host/ci-llvm/lib/libLLVM.dylib"), + build.join("cache/2026-07-13/rustc.tar.xz"), build.join("host/rustfmt/bin/rustfmt"), ]; let lay = || { @@ -818,6 +790,41 @@ mod tests { assert!(downloaded.iter().all(|f| f.is_file()), "a download went"); } + /// **A std build directory keeps no LLVM, even under the compiler that + /// built the rest**: bootstrap's and `download-ci-llvm`'s go with their + /// download, and what that compiler built and the other downloads stay. + #[test] + fn a_std_build_under_the_same_compiler_keeps_no_llvm() { + let base = TempDir::new("std-llvm"); + let (_root, rust_dir, _fork) = keyed(&base); + let build = base.join("toyos-std"); + let host = host_triple(); + let identity = Compiler::primary(&rust_dir).identity(); + prepare_std_build(&build, &host, &identity); + let llvm = [ + build.join(&host).join("ci-llvm/lib/libLLVM.dylib"), + build.join(&host).join("llvm/bin/llvm-config"), + build.join("cache/llvm-ad3d0bc-false/rust-dev.tar.xz"), + ]; + let kept = [build.join("bootstrap/debug/deps/libserde-1.rlib"), build.join("cache/2026-07-13/rustc.tar.xz")]; + for file in llvm.iter().chain(&kept) { + write(file, "built"); + } + prepare_std_build(&build, &host, &identity); + for file in &llvm { + assert!(!file.exists(), "{} outlived a std build's preparation", file.display()); + } + assert!(kept.iter().all(|f| f.is_file()), "the same compiler's build went"); + } + + /// **A std build fetches no LLVM**: it builds none, and the `compiler` + /// profile would download one. + #[test] + fn a_std_build_downloads_no_llvm() { + let config = std_config(Path::new("/c"), Path::new("/cargo"), Path::new("/b"), "h"); + assert!(config.contains("\n[llvm]\ndownload-ci-llvm = false\n"), "{config}"); + } + /// **A switch that cannot remove the other compiler's build fails and does /// not record the new compiler**, so the next call removes it. #[test] @@ -1041,36 +1048,6 @@ mod tests { assert_eq!(made.get(), 1); } - /// A key no registered worktree records goes, and so does a half-built one; - /// a key a worktree records stays, and so does one somebody is using. - #[test] - fn a_sweep_removes_what_no_worktree_names_and_nobody_uses() { - let root = TempDir::new("sweep"); - git(&root, &["init", "-q"]); - write(&root.join("f"), "x\n"); - git(&root, &["add", "f"]); - git(&root, &["commit", "-qm", "init"]); - let linked = root.join("linked"); - git(&root, &["worktree", "add", "-q", "-b", "wt", linked.to_str().unwrap()]); - - let dir = sysroots_dir(&root.join("rust")); - for name in ["named", "linked-named", "in-use", "orphan", "named.partial"] { - fs::create_dir_all(dir.join(name)).unwrap(); - } - write(&root.join(RECORD), "named"); - write(&linked.join(RECORD), "linked-named"); - let user = buildlock::tests::sysroot_used_elsewhere(&root, "in-use"); - - let mut removed = sweep(&root); - removed.sort(); - assert_eq!(removed, [dir.join("named.partial"), dir.join("orphan")]); - for stays in ["named", "linked-named", "in-use"] { - assert!(dir.join(stays).is_dir(), "{stays} was swept"); - } - user.release(); - assert_eq!(sweep(&root), [dir.join("in-use")]); - } - /// **What a stage-0 std build made is what its stamp names**: its /// `stage0-sysroot` holds the compiler's own libraries, and taking those was /// a sysroot built from sources it never compiled. Each kind goes where diff --git a/src/toolchain.rs b/src/toolchain.rs index d274c54787f..7c9d2ca0b0a 100644 --- a/src/toolchain.rs +++ b/src/toolchain.rs @@ -350,37 +350,41 @@ fn incomplete(rust_dir: &Path) -> bool { cargo_link_stale(&stage2) || crate::clang::defect(&stage2).is_some() || host_target_missing(rust_dir) } -/// Give the primary's toolchain what [`incomplete`] finds missing. -fn complete(rust_dir: &Path) { +/// Give the primary's toolchain what [`incomplete`] finds missing, its clang by +/// `provision_clang`. +fn complete(rust_dir: &Path, provision_clang: impl FnOnce(&Path)) { let stage2 = stage2(rust_dir); if cargo_link_stale(&stage2) { provision_toolchain_cargo(&stage2); } if crate::clang::defect(&stage2).is_some() { - crate::clang::provision(&stage2); + provision_clang(&stage2); } if host_target_missing(rust_dir) { link_host_target(rust_dir); } } -/// Run `bootstrap` in the primary's `rust/`, then [`complete`] what it -/// reassembled. Called inside the act that holds the global lock exclusively. +/// Run `bootstrap` in the primary's `rust/` against the LLVM at `llvm`, then +/// remove the LLVM its build directory built itself (`llvm::retire_in_tree`) +/// and [`complete`] what it reassembled. Called inside the act that holds the +/// global lock exclusively. /// /// **In the same hold, because bootstrap recreates `stage2` without its cargo /// and clang**: a completion under a hold of its own queues behind every sysroot /// build that takes the lock shared in between, and those last minutes. -fn reassemble(rust_dir: &Path, bootstrap: impl FnOnce()) { +fn reassemble(rust_dir: &Path, llvm: &Path, bootstrap: impl FnOnce()) { bootstrap(); - complete(rust_dir); + crate::llvm::retire_in_tree(&rust_dir.join("build")); + complete(rust_dir, |stage2| crate::clang::provision(stage2, llvm)); } /// [`reassemble`] the primary's compiler with `bootstrap`, with nothing recording /// which compiler `stage2` is until it is whole: a bootstrap that is stopped is /// run again by the primary, and refused by name in every linked worktree. -fn rebuild_compiler(rust_dir: &Path, bootstrap: impl FnOnce()) { +fn rebuild_compiler(rust_dir: &Path, llvm: &Path, bootstrap: impl FnOnce()) { crate::compiler::forget(rust_dir); - reassemble(rust_dir, bootstrap); + reassemble(rust_dir, llvm, bootstrap); crate::compiler::record(rust_dir); } @@ -467,7 +471,8 @@ pub fn ensure(root: &Path, force_rebuild: bool, lock: &mut buildlock::Held) -> S }, |kind| { eprintln!("Building full toolchain (this takes a while on first run)..."); - rebuild_compiler(&rust_dir, || full_bootstrap(root, &rust_dir)); + let llvm = crate::llvm::resolve(root, &rust_dir, &rust_dir); + rebuild_compiler(&rust_dir, &llvm.dir, || full_bootstrap(root, &rust_dir, &llvm.dir)); if kind.invalidate_hosted { let _ = fs::remove_file(&hosted_stamp); } @@ -480,7 +485,8 @@ pub fn ensure(root: &Path, force_rebuild: bool, lock: &mut buildlock::Held) -> S "build the ToyOS-hosted rustc", || (!hosted_stamp.exists() || !hosted_rustc.exists()).then_some(()), |()| { - reassemble(&rust_dir, || build_hosted_rustc(&rust_dir)); + let llvm = crate::llvm::resolve(root, &rust_dir, &rust_dir); + reassemble(&rust_dir, &llvm.dir, || build_hosted_rustc(&rust_dir, &llvm.dir)); assert!(hosted_rustc.exists(), "Failed to build hosted rustc"); fs::write(&hosted_stamp, "").unwrap(); }, @@ -504,7 +510,7 @@ pub fn ensure(root: &Path, force_rebuild: bool, lock: &mut buildlock::Held) -> S Scope::Global, "complete the toyos toolchain", || incomplete(&rust_dir).then_some(()), - |()| complete(&rust_dir), + |()| complete(&rust_dir, |stage2| crate::clang::provision(stage2, &crate::llvm::resolve(root, &rust_dir, &rust_dir).dir)), ); assert_toolchain_is_honest(&stage2); @@ -601,6 +607,17 @@ fn link_stale(stage2: &Path) -> bool { /// enough for the question [`refuse_on_compile_error`] asks, which is what the /// failure *was*. pub(crate) fn x_build(rust_dir: &Path, args: &[&str], what: &str) -> (bool, Vec) { + x_build_with(rust_dir, args, what, |_| {}) +} + +/// [`x_build`], with bootstrap's environment what `environment` makes of this +/// process's. +pub(crate) fn x_build_with( + rust_dir: &Path, + args: &[&str], + what: &str, + environment: impl FnOnce(&mut Command), +) -> (bool, Vec) { use std::io::{BufRead, BufReader, Read, Write}; use std::sync::{Arc, Mutex}; @@ -615,6 +632,7 @@ pub(crate) fn x_build(rust_dir: &Path, args: &[&str], what: &str) -> (bool, Vec< } else { ("./x.py", Command::new("./x.py")) }; + environment(&mut command); let mut child = command .args(args) .env("BOOTSTRAP_SKIP_TARGET_SANITY", "1") @@ -715,14 +733,14 @@ fn tolerated_failure(log: &[String], what: &str) { ); } -fn full_bootstrap(root: &Path, rust_dir: &Path) { +fn full_bootstrap(root: &Path, rust_dir: &Path, llvm: &Path) { // Ensure library/backtrace is checked out — std depends on it. // Other rust submodules (llvm, docs, cargo) are handled by bootstrap on demand. crate::ensure_submodule(rust_dir, "library/backtrace"); // Write bootstrap.toml — ToyOS as target only, not host (fast rebuilds) let host = host_triple(); - write_config(rust_dir, &host, false); + write_config(rust_dir, &host, false, llvm); // Clean cached std for all ToyOS targets so bootstrap picks up compiler changes // (e.g. target spec changes like default_uwtable that affect codegen). @@ -757,10 +775,10 @@ fn full_bootstrap(root: &Path, rust_dir: &Path) { } } -fn build_hosted_rustc(rust_dir: &Path) { +fn build_hosted_rustc(rust_dir: &Path, llvm: &Path) { eprintln!("Building ToyOS-hosted rustc..."); let host = host_triple(); - write_config(rust_dir, &host, true); + write_config(rust_dir, &host, true, llvm); let (ok, log) = x_build(rust_dir, &["build", "--stage", "2", "--warnings", "warn"], "the hosted rustc"); @@ -789,7 +807,7 @@ fn build_hosted_rustc(rust_dir: &Path) { // That build reassembled the host's `stage2` without `rust-lld` // (`write_config` says why), so the host-only build runs once more to put // it back: everything it would compile is already built. - write_config(rust_dir, &host, false); + write_config(rust_dir, &host, false, llvm); let (ok, log) = x_build( rust_dir, &["build", "--stage", "2", "--warnings", "warn"], @@ -815,13 +833,14 @@ fn build_hosted_rustc(rust_dir: &Path) { /// `stage2` first, so [`build_hosted_rustc`] reassembles it under the host-only /// config after. /// -/// `clang::LLVM_CONFIG` is the `[llvm]` both builds share. +/// `clang::LLVM_CONFIG` is the `[llvm]` both builds share, and both link the LLVM +/// at `llvm` (`src/llvm.rs`), whose LLD every guest target names by path. /// /// The host's `default-linker-linux-override` is pinned off because bootstrap /// otherwise ties it to `lld` for `x86_64-unknown-linux-gnu`, and a host rustc /// whose build environment flips with the config is rebuilt by each of those /// two builds. -fn write_config(rust_dir: &Path, host: &str, with_hosted_rustc: bool) { +fn write_config(rust_dir: &Path, host: &str, with_hosted_rustc: bool, llvm: &Path) { let host_line = if with_hosted_rustc { format!("host = [\"{host}\", \"{}\"]", HOSTED_ARCH.userland()) } else { @@ -832,11 +851,10 @@ fn write_config(rust_dir: &Path, host: &str, with_hosted_rustc: bool) { .map(|t| format!("\"{t}\"")) .collect::>() .join(", "); - let build = rust_dir.join(format!("build/{host}")); let userland: String = Arch::ALL .iter() .map(|arch| { - let linker = format!("linker = \"{}\"", build.join("lld/bin/lld").display()); + let linker = format!("linker = \"{}\"", llvm.join("bin/lld").display()); let hosted = if with_hosted_rustc && *arch == HOSTED_ARCH { // Cranelift because no LLVM is built for a ToyOS host yet, and // only for that reason: the hosted rustc carries LLVM once clang @@ -849,7 +867,7 @@ fn write_config(rust_dir: &Path, host: &str, with_hosted_rustc: bool) { // leaves those ELF members out of the index lld pulls from. format!( "\nar = \"{}\"\ncodegen-backends = [\"cranelift\"]", - build.join("llvm/bin/llvm-ar").display(), + llvm.join("bin/llvm-ar").display(), ) } else { String::new() @@ -874,9 +892,11 @@ lld = {lld} [target.{host}] {HOST_LINKER_PIN} +{external} {userland}"#, llvm = crate::clang::LLVM_CONFIG, + external = crate::llvm::host_lines(llvm), lld = !with_hosted_rustc, ); fs::write(rust_dir.join("bootstrap.toml"), config).unwrap(); @@ -1063,15 +1083,22 @@ mod tests { assert!(said.contains("clang") && !said.contains("rust-lld,"), "the refusal names clang alone: {said}"); } - /// Every guest links through the host build's LLD, named by path. + /// Every build links the host's LLVM, and every guest links through its + /// LLD, named by path. #[test] - fn every_build_names_its_lld_by_path() { + fn every_build_links_the_host_s_llvm_and_names_its_lld_by_path() { let rust_dir = TempDir::new("lld-config"); - let lld = format!("linker = \"{}\"", rust_dir.join("build/h/lld/bin/lld").display()); + let llvm = rust_dir.join("build/llvm/k"); + let lld = format!("linker = \"{}\"", llvm.join("bin/lld").display()); for (hosted, lld_flag) in [(true, "lld = false"), (false, "lld = true")] { - write_config(&rust_dir, "h", hosted); + write_config(&rust_dir, "h", hosted, &llvm); let config = fs::read_to_string(rust_dir.join("bootstrap.toml")).unwrap(); assert!(config.contains(lld_flag) && config.contains(&lld) && !config.contains("\"rust-lld\""), "{config}"); + let host = format!( + "[target.h]\ndefault-linker-linux-override = \"off\"\nllvm-config = \"{}/bin/llvm-config\"\nllvm-has-rust-patches = true\n", + llvm.display() + ); + assert!(config.contains(&host), "{config}"); } } @@ -1084,26 +1111,8 @@ mod tests { fn a_bootstrap_leaves_nothing_to_wait_on_a_sysroot_build_for() { let rust_dir = TempDir::new("no-wait"); let stage2 = stage2(&rust_dir); - // The LLVM bootstrap builds beside `stage2`, which `clang::provision` reads. - let llvm = stage2.with_file_name("llvm"); - for (file, text) in [("bin/clang", "clang"), ("lib/clang/22/include/stddef.h", "stddef")] { - fs::create_dir_all(llvm.join(file).parent().unwrap()).unwrap(); - fs::write(llvm.join(file), text).unwrap(); - } - - reassemble(&rust_dir, || { - // What bootstrap leaves: `stage2` made again, with `rustc` and the - // LLVM tools it assembles, and neither cargo nor clang; and the - // hosted rustc's sysroot without the host target. - let _ = fs::remove_dir_all(&stage2); - let lld = rust_lld(&stage2); - for file in [stage2.join("bin/rustc"), lld.clone(), lld.with_file_name("llvm-ar")] { - fs::create_dir_all(file.parent().unwrap()).unwrap(); - fs::write(file, b"").unwrap(); - } - let hosted = rust_dir.join(format!("build/{}/stage2/lib/rustlib", HOSTED_ARCH.userland())); - fs::create_dir_all(&hosted).unwrap(); - }); + let llvm = store_llvm(&rust_dir); + reassemble(&rust_dir, &llvm, || bootstrapped(&rust_dir)); assert!( !incomplete(&rust_dir), "a bootstrap let its exclusive hold go with a global step left, which the primary's \ @@ -1112,18 +1121,93 @@ mod tests { assert_eq!(toolchain_defect(&stage2), None, "a bootstrap let its exclusive hold go with stage2 not whole"); } + /// The LLVM `clang::provision` reads, in `rust_dir`'s store. + fn store_llvm(rust_dir: &Path) -> PathBuf { + let llvm = rust_dir.join("build/llvm/k"); + for (file, text) in [("bin/clang", "clang"), ("lib/clang/22/include/stddef.h", "stddef")] { + fs::create_dir_all(llvm.join(file).parent().unwrap()).unwrap(); + fs::write(llvm.join(file), text).unwrap(); + } + llvm + } + + /// What bootstrap leaves in `rust_dir`: `stage2` made again, with `rustc` + /// and the LLVM tools it assembles, and neither cargo nor clang; and the + /// hosted rustc's sysroot without the host target. + fn bootstrapped(rust_dir: &Path) { + let stage2 = stage2(rust_dir); + let _ = fs::remove_dir_all(&stage2); + let lld = rust_lld(&stage2); + for file in [stage2.join("bin/rustc"), lld.clone(), lld.with_file_name("llvm-ar")] { + fs::create_dir_all(file.parent().unwrap()).unwrap(); + fs::write(file, b"").unwrap(); + } + let hosted = rust_dir.join(format!("build/{}/stage2/lib/rustlib", HOSTED_ARCH.userland())); + fs::create_dir_all(&hosted).unwrap(); + } + + /// What a build directory holds of an LLVM of its own: bootstrap's LLVM and + /// LLD. + fn in_tree_llvm(rust_dir: &Path) -> [PathBuf; 2] { + let host = rust_dir.join("build").join(host_triple()); + let own = [host.join("llvm"), host.join("lld")]; + for dir in &own { + fs::create_dir_all(dir.join("bin")).unwrap(); + fs::write(dir.join("bin/tool"), "a tool").unwrap(); + } + own + } + + /// **A primary whose compiler is built against the store keeps no LLVM of + /// its own**: the rebuild that links the store removes the one its build + /// directory built. + #[test] + fn a_rebuilt_compiler_leaves_no_llvm_of_its_own() { + let scratch = TempDir::new("in-tree-llvm"); + let (_primary, rust_dir, _) = crate::compiler::tests::estate(&scratch); + let own = in_tree_llvm(&rust_dir); + let llvm = store_llvm(&rust_dir); + rebuild_compiler(&rust_dir, &llvm, || bootstrapped(&rust_dir)); + for dir in own { + assert!(!dir.exists() && !dir.with_extension("swept").exists(), "{} outlived the rebuild", dir.display()); + } + assert_eq!(toolchain_defect(&stage2(&rust_dir)), None); + } + + /// **Landing the store moves an existing primary onto it**: a primary whose + /// record was written before its compiler linked the host's LLVM is not + /// current, so its next build bootstraps, and that rebuild removes the LLVM + /// and LLD its build directory built. Its LLVM checkout sitting at a commit + /// its gitlink does not name changes neither answer. + #[test] + fn a_primary_recorded_before_the_store_is_rebuilt_onto_it() { + let scratch = TempDir::new("store-migration"); + let (_primary, rust_dir, _) = crate::compiler::tests::estate(&scratch); + crate::compiler::tests::llvm_checkout(&rust_dir); + assert!(crate::compiler::primary_is_current(&rust_dir)); + crate::compiler::tests::record_before_the_store(&rust_dir); + let own = in_tree_llvm(&rust_dir); + let kind = bootstrap(false, crate::compiler::primary_is_current(&rust_dir), true); + assert!(kind.is_some_and(|k| k.invalidate_hosted), "a primary recorded before the store was taken for current"); + rebuild_compiler(&rust_dir, &store_llvm(&rust_dir), || bootstrapped(&rust_dir)); + assert!(own.iter().all(|dir| !dir.exists()), "the rebuild kept the LLVM its build directory built"); + assert!(crate::compiler::primary_is_current(&rust_dir), "the rebuild recorded a compiler that is not current"); + } + /// **A stopped bootstrap is run again**: nothing records which compiler /// `stage2` is while one runs, so the primary's next build is not told the - /// old one is current. + /// old one is current; and the LLVM the old one linked stays. #[test] fn a_stopped_bootstrap_leaves_no_record() { let rust_dir = TempDir::new("stopped"); let record = crate::compiler::primary_record(&rust_dir); fs::create_dir_all(record.parent().unwrap()).unwrap(); fs::write(&record, "the compiler before").unwrap(); - let stopped = std::panic::catch_unwind(|| rebuild_compiler(&rust_dir, || panic!("stopped"))); + let own = in_tree_llvm(&rust_dir); + let stopped = std::panic::catch_unwind(|| rebuild_compiler(&rust_dir, Path::new("no-llvm"), || panic!("stopped"))); assert!(stopped.is_err()); assert!(!record.exists(), "a stopped bootstrap left the record of the compiler before it"); + assert!(own.iter().all(|dir| dir.join("bin/tool").is_file()), "a stopped bootstrap took the LLVM its compiler linked"); } /// **The primary bootstraps a new compiler exactly when asked to or when its diff --git a/src/worktree.rs b/src/worktree.rs index 720c61c025a..cbc4ec98262 100644 --- a/src/worktree.rs +++ b/src/worktree.rs @@ -20,6 +20,7 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; +use crate::buildlock::Keyed; use crate::flags; use crate::toolchain; @@ -376,8 +377,8 @@ fn gib(bytes: u64) -> String { /// a worktree of anything. Once git has let go of it nothing in it is work, /// so the whole directory goes. /// -/// Then every sysroot and every compiler no remaining worktree names goes too -/// (`src/sysroot.rs`, `src/compiler.rs`). +/// Then every sysroot, compiler and LLVM no remaining worktree names goes too +/// (`src/sysroot.rs`, `src/compiler.rs`, `src/llvm.rs`). pub(crate) fn remove(root: &Path, path: &str) { let at = root.join(path); remove_fork_checkout(root, &at); @@ -391,13 +392,16 @@ pub(crate) fn remove(root: &Path, path: &str) { eprintln!("git unregistered {path} and left its ignored files; deleted them"); } eprintln!("removed {path}; its branch is still there, and `git branch -d` will say if it is unmerged"); - let swept = crate::sysroot::sweep(root); - if !swept.is_empty() { - eprintln!("removed {} sysroot(s) no worktree names any more", swept.len()); - } - let swept = crate::compiler::sweep(root, &crate::toolchain::rust_dir(root)); - if !swept.is_empty() { - eprintln!("removed {} compiler(s) no worktree names any more", swept.len()); + let rust_dir = crate::toolchain::rust_dir(root); + for (kind, store, what) in [ + (Keyed::Sysroot, crate::sysroot::sysroots_dir(&rust_dir), "sysroot"), + (Keyed::Compiler, crate::compiler::compilers_dir(&rust_dir), "compiler"), + (Keyed::Llvm, crate::llvm::store(&rust_dir), "LLVM"), + ] { + let swept = crate::keystore::sweep(root, kind, &store); + if !swept.is_empty() { + eprintln!("removed {} {what}(s) no worktree names any more", swept.len()); + } } }