From 341192deb3b2de1a4145424bbc3d68e0af7b2ce1 Mon Sep 17 00:00:00 2001 From: Quentin Date: Wed, 28 Oct 2026 10:26:03 +0100 Subject: [PATCH 1/2] Keep weft's build folders to what is still used --- CHANGELOG.md | 14 + crates/weft-cli/src/commands/runs.rs | 22 +- crates/weft-cli/src/main.rs | 19 +- crates/weft-compiler/src/build.rs | 105 ++-- crates/weft-compiler/src/build_plan.rs | 6 + crates/weft-compiler/src/weft-cache-gc.sh | 134 ++--- crates/weft-compiler/src/worker_image.rs | 158 +----- crates/weft-core/src/builds.rs | 14 +- crates/weft-core/src/file_lock.rs | 9 + crates/weft-dispatcher/src/build/mod.rs | 10 +- crates/weft-e2e/README.md | 2 +- crates/weft-platform-local/src/cargo_cache.rs | 477 ++++++++++++++++++ crates/weft-platform-local/src/host_build.rs | 146 +++++- crates/weft-platform-local/src/images.rs | 86 +++- crates/weft-platform-local/src/lib.rs | 1 + docs/src/running/architecture.md | 10 +- docs/src/running/cli.md | 2 +- setup.sh | 122 +---- 18 files changed, 857 insertions(+), 480 deletions(-) create mode 100644 crates/weft-platform-local/src/cargo_cache.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aa3fced3..bebd3912a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ What changed between versions of weft that you need to know about when you update. To update, run `weft maintenance on`, pull and run `./setup.sh` again, then run `weft maintenance off`. +## Unreleased + +**The folders weft compiles in stop growing with every update.** Each +program weft compiles for a project now notes which compiled pieces and +source crates it used, and after every build weft keeps in its compile +folders (`~/.local/share/weft/worker-target/` and +`~/.local/share/weft/worker-crates/`) only what a program still on the +machine was built from. Before, every update left a full copy of weft's +libraries behind in each of them. The first project build after you update +clears out what older versions left, which can free tens of gigabytes. If +you build weft from source, `./setup.sh` now also removes from the +checkout's `target/` every compiled piece a newer build of it replaced, and +`WEFT_TARGET_CAP_GB` is gone. + ## 2026-10-28 (0.2.277) **If a project has its own nodes, or its own copy of the standard nodes, it diff --git a/crates/weft-cli/src/commands/runs.rs b/crates/weft-cli/src/commands/runs.rs index 148658c1e..7aab638d7 100644 --- a/crates/weft-cli/src/commands/runs.rs +++ b/crates/weft-cli/src/commands/runs.rs @@ -634,9 +634,9 @@ pub async fn clean_build_plane() -> anyhow::Result<()> { /// The files host builds leave on the machine that nothing uses any more: /// the worker programs no install holds or names as its standard worker /// (the standard worker of a weft an update replaced, a removed install's -/// programs), and the shared crates no build staged for -/// `WORKER_CACHE_RETENTION_DAYS` (the sources of a package since edited, -/// the workspace of a weft since updated). Needs no running install. +/// programs), and the shared crates no program left on the machine was +/// built from (the sources of a package since edited, the workspace of a +/// weft since updated). Needs no running install. fn unused_worker_files(machine: &weft_platform_local::MachineLayout) -> anyhow::Result<()> { let installs: Vec = super::daemon::Install::every_on(machine.root())?.iter().map(|i| i.id.label_value().to_string()).collect(); let workers = weft_platform_local::images::reclaim_workers(&machine.workers(), &installs)?; @@ -645,12 +645,10 @@ fn unused_worker_files(machine: &weft_platform_local::MachineLayout) -> anyhow:: } else { println!("removed {} unused worker program(s)", workers.len()); } - let days = weft_compiler::worker_image::WORKER_CACHE_RETENTION_DAYS; - let crates = weft_compiler::build::reclaim_shared_crates(&machine.shared_crates(), std::time::Duration::from_secs(u64::from(days) * 24 * 3600))?; - if crates.is_empty() { - println!("no shared build crates unused for {days} days"); - } else { - println!("removed {} shared build crate(s) unused for {days} days", crates.len()); + match weft_platform_local::host_build::reclaim_shared_crates(machine)? { + None => println!("a build is staging its shared crates; they are left for the next clean"), + Some(crates) if crates.is_empty() => println!("no shared build crates unused"), + Some(crates) => println!("removed {} shared build crate(s) no program on this machine uses", crates.len()), } Ok(()) } @@ -763,14 +761,12 @@ async fn report_compile_cache_size() -> anyhow::Result<()> { let lanes = dir_size(&machine.compile_caches())?; let crates = dir_size(&machine.shared_crates())?; println!( - "worker compile caches: {:.1}GB in {} (every lane and the seed), and {:.1}GB of shared crates in {}; a build drops the \ - per-project crates no build on this machine has linked for {} days, `weft clean --images --all` (the last step of \ - `setup.sh`) the shared crates no build staged for as long, `weft clean --build-cache` drops everything now", + "worker compile caches: {:.1}GB in {} (every lane and the seed), and {:.1}GB of shared crates in {}; a build keeps \ + in both only what a program on this machine was built from, `weft clean --build-cache` drops everything now", gb(lanes), machine.compile_caches().display(), gb(crates), machine.shared_crates().display(), - weft_compiler::worker_image::WORKER_CACHE_RETENTION_DAYS ); match super::test_node::cache_size_bytes()? { None => println!("node-test cache: nothing built on this host"), diff --git a/crates/weft-cli/src/main.rs b/crates/weft-cli/src/main.rs index 85eaa63b0..91877d693 100644 --- a/crates/weft-cli/src/main.rs +++ b/crates/weft-cli/src/main.rs @@ -108,6 +108,12 @@ enum Cmd { #[arg(long)] out: std::path::PathBuf, }, + /// For setup.sh: remove from a cargo build folder (a checkout's + /// `target/`) every compiled piece a newer build of the same piece + /// replaced, keeping one copy of each way it is built (tests, clippy, + /// release), so the next build of any of them compiles nothing more. + #[command(name = "trim-cargo-target", hide = true)] + TrimCargoTarget { dir: std::path::PathBuf }, /// Make every image the release publishes exist locally: the runtime /// image (every role weft has, in one), and for the cloud the worker /// builder base and the full-library worker image, each under its @@ -953,9 +959,9 @@ enum Cmd { /// behind) /// --images --build-plane /// the worker programs no install on this machine - /// uses, the shared build crates no build staged - /// for 30 days, and the builder bases and node-test - /// images an older weft left + /// uses, the shared build crates no program on it + /// was built from, and the builder bases and + /// node-test images an older weft left /// --build-cache drop the docker buildkit cache, the node-test cache /// and the worker compile caches /// --all with the journal subject: nuke every run @@ -1006,8 +1012,8 @@ enum Cmd { #[arg(long, default_value_t = false)] images: bool, /// With --images: only the build plane, the worker programs no - /// install on this machine uses, the shared build crates no build - /// staged for 30 days, and the builder bases and node-test images + /// install on this machine uses, the shared build crates no program + /// on it was built from, and the builder bases and node-test images /// an older weft left that no install uses (kept whole while an /// install has not recorded its own). Needs no running install; /// setup.sh runs it after every start. @@ -2089,6 +2095,9 @@ async fn run(cli: Cli, wanted: Result) -> anyh Cmd::Ci { action: CiCmd::Add { cloud } } => commands::ci::add(ctx, cloud).await, Cmd::Build { referenced } => commands::build::run(ctx, node_set(referenced)).await, Cmd::BuildStandardWorker { out } => host_programs::pack_standard_program(&out).await, + Cmd::TrimCargoTarget { dir } => weft_platform_local::cargo_cache::drop_superseded(&dir).map(|swept| { + println!("removed {} superseded compiled pieces ({:.1}GB)", swept.units, swept.bytes as f64 / 1e9); + }), Cmd::BuildImages { push, push_suffix, print, runtime_binary } => { commands::build::run_build_images(push, push_suffix, print, runtime_binary.as_deref()).await } diff --git a/crates/weft-compiler/src/build.rs b/crates/weft-compiler/src/build.rs index e3ee9ebb6..f907ce012 100644 --- a/crates/weft-compiler/src/build.rs +++ b/crates/weft-compiler/src/build.rs @@ -25,6 +25,9 @@ use weft_catalog::FsCatalog; /// hash that NAMES what it builds, so the caller can build it or /// recognize it as already present. pub struct StagedImageBuild { + /// A host build's claim on the shared crates it staged, until it is + /// compiled: dropped with the staged build ([`SharedCratesHold`]). + pub hold: Option>, /// Absolute path to the build context: the host recipe and the worker /// crate for a host build; for an image, the `Dockerfile`, `build/` /// (the generated cargo crate), `project-nodes/` and, when the @@ -196,7 +199,7 @@ pub fn build_project( .map_err(|e| BuildError::Build(format!("compute binary hash: {e}")))?; let builder = match packaging { weft_core::builds::WorkerPackaging::Host { shared_crates } => { - let build_context = stage_host_build( + let (build_context, hold) = stage_host_build( &project_root.join(".weft").join("target").join("worker-host"), shared_crates, definition, @@ -204,6 +207,7 @@ pub fn build_project( node_set, )?; return Ok(StagedImageBuild { + hold: Some(std::sync::Arc::new(hold)), build_context, content_hash, recipe: weft_core::builds::BuildRecipe::Host, @@ -257,6 +261,7 @@ pub fn build_project( )?; Ok(StagedImageBuild { + hold: None, build_context, content_hash, recipe: weft_core::builds::BuildRecipe::Dockerfile { @@ -303,7 +308,7 @@ pub fn stage_host_build( definition: &weft_core::project::ProjectDefinition, catalog: &FsCatalog, node_set: weft_core::builds::NodeSet, -) -> BuildResult { +) -> BuildResult<(PathBuf, SharedCratesHold)> { stage_host_crate(ctx, shared, catalog, &|crate_root, paths| { Ok(HostCrate { package_dirs: codegen::emit(definition, crate_root, catalog, WORKER_CRATE_NAME, node_set, paths)?, @@ -318,7 +323,7 @@ pub fn stage_host_build( /// replaced by the package's test crate (`codegen::emit_test_crate`), /// whose package crate is the one a worker holding that package compiles /// (same folder under `shared`, compiled with its `node-tests` feature). -pub fn stage_host_test_build(ctx: &Path, shared: &Path, catalog: &FsCatalog, package_name: &str) -> BuildResult { +pub fn stage_host_test_build(ctx: &Path, shared: &Path, catalog: &FsCatalog, package_name: &str) -> BuildResult<(PathBuf, SharedCratesHold)> { stage_host_crate(ctx, shared, catalog, &|crate_root, paths| { let test_crate = codegen::emit_test_crate(crate_root, catalog, package_name, paths)?; Ok(HostCrate { @@ -347,7 +352,8 @@ fn stage_host_crate( shared: &Path, catalog: &FsCatalog, emit: &dyn Fn(&Path, &codegen::EmitPaths) -> BuildResult, -) -> BuildResult { +) -> BuildResult<(PathBuf, SharedCratesHold)> { + let hold = SharedCratesHold::of(shared)?; if ctx.exists() { std::fs::remove_dir_all(ctx).map_err(BuildError::Io)?; } @@ -393,26 +399,35 @@ fn stage_host_crate( // image build without a base seeds from (`/weft/Cargo.lock`). seed_lock: slice.join("Cargo.lock"), binary_name: emitted.binary_name, - cache_gc_script: crate_root.join(worker_image::CACHE_GC_SCRIPT_NAME), - cache_retention_days: worker_image::WORKER_CACHE_RETENTION_DAYS, - cache_versions_kept: worker_image::WORKER_CACHE_VERSIONS_KEPT, crate_dir: crate_root, }; let json = serde_json::to_vec_pretty(&recipe).map_err(|e| BuildError::Build(format!("the host recipe: {e}")))?; std::fs::write(ctx.join(weft_core::builds::HOST_RECIPE_FILE), json).map_err(BuildError::Io)?; - Ok(ctx.to_path_buf()) + Ok((ctx.to_path_buf(), hold)) +} + +/// A build's claim on the crates it staged in the folder every host build +/// on the machine shares, from before it stages them until it is compiled +/// (`weft_core::builds::shared_crates_lock`). +#[derive(Debug)] +pub struct SharedCratesHold { + _lock: std::fs::File, +} + +impl SharedCratesHold { + fn of(shared: &Path) -> BuildResult { + Ok(Self { _lock: weft_core::file_lock::hold_shared(&weft_core::builds::shared_crates_lock(shared)).map_err(BuildError::Io)? }) + } } /// Make `dest` by `write` once and never again: a folder named by its /// content is whole as soon as it is there, and another build may be /// reading it. `write` fills a fresh folder beside it, which is then /// renamed into place, so `dest` is either whole or absent; when another -/// writer got there first, its copy (the same content) stays. A folder -/// already there has its time moved to now, which is how -/// [`reclaim_shared_crates`] tells it is still used. +/// writer got there first, its copy (the same content) stays. pub(crate) fn write_once(dest: &Path, write: &dyn Fn(&Path) -> BuildResult<()>) -> BuildResult<()> { if dest.is_dir() { - return std::fs::File::open(dest).and_then(|dir| dir.set_modified(std::time::SystemTime::now())).map_err(BuildError::Io); + return Ok(()); } let parent = dest.parent().ok_or_else(|| BuildError::Build(format!("{} has no parent folder", dest.display())))?; std::fs::create_dir_all(parent).map_err(BuildError::Io)?; @@ -429,37 +444,6 @@ pub(crate) fn write_once(dest: &Path, write: &dyn Fn(&Path) -> BuildResult<()>) written } -/// Remove what no build has used for `unused_for` from `shared` (the -/// folder of crates every host build on the machine shares), -/// answering the names it removed: a package crate of sources since -/// edited, the workspace slice of a weft since updated, a half-written -/// folder a stopped build left. Each is written once and touched by every -/// build that uses it ([`write_once`]), so what goes is what nothing staged -/// in that time; a build that wants one again writes it again. -pub fn reclaim_shared_crates(shared: &Path, unused_for: std::time::Duration) -> BuildResult> { - let entries = match std::fs::read_dir(shared) { - Ok(entries) => entries, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), - Err(e) => return Err(BuildError::Io(e)), - }; - let now = std::time::SystemTime::now(); - let mut removed = Vec::new(); - for entry in entries { - let entry = entry.map_err(BuildError::Io)?; - let used = entry.metadata().and_then(|m| m.modified()).map_err(BuildError::Io)?; - if now.duration_since(used).is_ok_and(|idle| idle > unused_for) { - if entry.file_type().map_err(BuildError::Io)?.is_dir() { - std::fs::remove_dir_all(entry.path()).map_err(BuildError::Io)?; - } else { - std::fs::remove_file(entry.path()).map_err(BuildError::Io)?; - } - removed.push(entry.file_name().to_string_lossy().into_owned()); - } - } - removed.sort(); - Ok(removed) -} - /// Copy a package's sources to `dest` as a build compiles them: what /// `weft_catalog::is_left_out_of_package_build` leaves out stays out (the /// package hash skips the same files), and each file carries the date @@ -616,13 +600,14 @@ pub fn build_test_artifact(project: &Project, catalog: &FsCatalog, package_name: // servers side by side, so a shared path would have one package's // staging clobber another's mid-build. let stem = sanitize_crate_name(package_name); - let build_context = stage_host_test_build( + let (build_context, hold) = stage_host_test_build( &project.root.join(".weft").join("target").join("test-host").join(&stem), shared_crates, catalog, package_name, )?; Ok(StagedImageBuild { + hold: Some(std::sync::Arc::new(hold)), build_context, content_hash: node_test_content_hash(catalog, package_name)?, recipe: weft_core::builds::BuildRecipe::Host, @@ -1454,25 +1439,6 @@ pub fn build_project_catalog(project_root: &Path) -> BuildResult { mod tests { use super::sanitize_crate_name; - /// A shared crate a build stages again is touched, so a reclaim takes - /// only what no build staged for longer than it keeps them. - #[test] - fn a_reclaim_takes_the_shared_crates_no_build_staged_lately() { - let shared = tempfile::tempdir().unwrap(); - let write = |dir: &std::path::Path| std::fs::create_dir_all(dir).map_err(super::BuildError::Io); - for name in ["pkg_a-old", "pkg_a-new", "weft-k"] { - super::write_once(&shared.path().join(name), &write).unwrap(); - } - let long_ago = std::time::SystemTime::now() - std::time::Duration::from_secs(3600); - for name in ["pkg_a-old", "weft-k"] { - std::fs::File::open(shared.path().join(name)).unwrap().set_modified(long_ago).unwrap(); - } - super::write_once(&shared.path().join("weft-k"), &write).unwrap(); - let removed = super::reclaim_shared_crates(shared.path(), std::time::Duration::from_secs(60)).unwrap(); - assert_eq!(removed, ["pkg_a-old"]); - assert!(shared.path().join("pkg_a-new").is_dir() && shared.path().join("weft-k").is_dir()); - } - /// Two installs on the same toolchain never share a compile cache, and /// each key says whose it is; an install keeps its cache across source /// edits and dependency bumps, and moves to a fresh one when its @@ -1556,17 +1522,16 @@ mod tests { "id": uuid::Uuid::nil(), "nodes": [{"id": "greet", "nodeType": "Run", "config": {}, "position": {"x": 0, "y": 0}}], "edges": [] })) .unwrap(); - let ctx = super::stage_host_build(&project.join("ctx"), &shared, &definition, &catalog, weft_core::builds::NodeSet::Referenced).expect("stage"); + let (ctx, _hold) = super::stage_host_build(&project.join("ctx"), &shared, &definition, &catalog, weft_core::builds::NodeSet::Referenced).expect("stage"); let recipe: weft_core::builds::HostRecipe = serde_json::from_slice(&std::fs::read(ctx.join(weft_core::builds::HOST_RECIPE_FILE)).unwrap()).unwrap(); assert_eq!(recipe.crate_dir, ctx.join(super::HOST_CRATE_DIR)); - for file in [&recipe.isolate_candidates, &recipe.left_out_record, &recipe.seed_lock, &recipe.cache_gc_script] { + for file in [&recipe.isolate_candidates, &recipe.left_out_record, &recipe.seed_lock] { assert!(file.is_file(), "{}", file.display()); } assert!(recipe.isolate_dir.join("Cargo.toml").is_file()); assert!(recipe.seed_lock.starts_with(&shared)); assert_eq!(recipe.binary_name, super::sanitize_crate_name(super::WORKER_CRATE_NAME)); - assert_eq!(recipe.cache_retention_days, crate::worker_image::WORKER_CACHE_RETENTION_DAYS); manifests.push(std::fs::read_to_string(recipe.crate_dir.join("Cargo.toml")).unwrap()); } let package_dep = |manifest: &str| manifest.lines().find(|l| l.starts_with("pkg_run") && l.contains("path =")).expect("the package dep").to_string(); @@ -1598,9 +1563,9 @@ mod tests { "id": uuid::Uuid::nil(), "nodes": [{"id": "greet", "nodeType": "Run", "config": {}, "position": {"x": 0, "y": 0}}], "edges": [] })) .unwrap(); - let worker = super::stage_host_build(&project.join("worker"), &shared, &definition, &catalog, weft_core::builds::NodeSet::Referenced).expect("stage the worker"); + let (worker, _worker_hold) = super::stage_host_build(&project.join("worker"), &shared, &definition, &catalog, weft_core::builds::NodeSet::Referenced).expect("stage the worker"); let package = catalog.package_of("Run").unwrap().name.clone(); - let tests = super::stage_host_test_build(&project.join("tests"), &shared, &catalog, &package).expect("stage the test server"); + let (tests, _tests_hold) = super::stage_host_test_build(&project.join("tests"), &shared, &catalog, &package).expect("stage the test server"); let recipe_of = |ctx: &std::path::Path| -> weft_core::builds::HostRecipe { serde_json::from_slice(&std::fs::read(ctx.join(weft_core::builds::HOST_RECIPE_FILE)).unwrap()).unwrap() }; @@ -1608,7 +1573,7 @@ mod tests { assert_eq!(recipe.crate_dir, tests.join(super::HOST_CRATE_DIR)); assert_eq!(recipe.binary_name, format!("{}_tests", super::sanitize_crate_name(&package))); assert_eq!(recipe.seed_lock, recipe_of(&worker).seed_lock, "one workspace slice for both"); - for file in [&recipe.isolate_candidates, &recipe.left_out_record, &recipe.cache_gc_script] { + for file in [&recipe.isolate_candidates, &recipe.left_out_record] { assert!(file.is_file(), "{}", file.display()); } let candidates: serde_json::Value = serde_json::from_slice(&std::fs::read(&recipe.isolate_candidates).unwrap()).unwrap(); @@ -1642,7 +1607,7 @@ mod tests { "id": uuid::Uuid::nil(), "nodes": [{"id": "thumb", "nodeType": "Vips", "config": {}, "position": {"x": 0, "y": 0}}], "edges": [] })) .unwrap(); - let ctx = super::stage_host_build(&project.join("ctx"), &temp.path().join("shared"), &definition, &catalog, weft_core::builds::NodeSet::Referenced).expect("stage"); + let (ctx, _hold) = super::stage_host_build(&project.join("ctx"), &temp.path().join("shared"), &definition, &catalog, weft_core::builds::NodeSet::Referenced).expect("stage"); let config: toml::Table = std::fs::read_to_string(ctx.join(super::HOST_CRATE_DIR).join(".cargo/config.toml")).unwrap().parse().unwrap(); let entry = config["env"]["PKG_CONFIG_PATH"].as_table().expect("a table entry"); assert_eq!(entry["force"].as_bool(), Some(true)); diff --git a/crates/weft-compiler/src/build_plan.rs b/crates/weft-compiler/src/build_plan.rs index e25d9237c..7b19bc8b9 100644 --- a/crates/weft-compiler/src/build_plan.rs +++ b/crates/weft-compiler/src/build_plan.rs @@ -84,6 +84,10 @@ pub struct BuildPlan { /// on (`crate::build::StagedImageBuild::node_needs`), kept with the /// worker. pub node_needs: weft_core::needs::NodeNeeds, + /// A host build's claim on the shared crates it staged + /// (`crate::build::StagedImageBuild::hold`), kept until the worker is + /// compiled. + pub hold: Option>, } /// How image refs are NAMED (the platform's image builder answers it); every @@ -143,6 +147,7 @@ pub fn plan_build_from( let staged = crate::build::build_project(project, definition, catalog, fs, packaging, node_set, fingerprints)?; let binary_hash = staged.content_hash.clone(); + let hold = staged.hold.clone(); let mut images = vec![PlannedImage { kind: ImageKind::Worker, @@ -178,5 +183,6 @@ pub fn plan_build_from( infra_hash, infra_places, images, + hold, }) } diff --git a/crates/weft-compiler/src/weft-cache-gc.sh b/crates/weft-compiler/src/weft-cache-gc.sh index c06652589..4e5664e2e 100644 --- a/crates/weft-compiler/src/weft-cache-gc.sh +++ b/crates/weft-compiler/src/weft-cache-gc.sh @@ -1,24 +1,13 @@ #!/bin/sh -# Bounds a cargo compile cache by what builds use. Run after a `cargo -# build` whose JSON messages (`--message-format=json...`) went to a file: -# cargo reports there every unit the build used, the ones it reused -# untouched included, and that report is how this knows what was used. -# -# usage: weft-cache-gc.sh this-build -# weft-cache-gc.sh aged -# -# `this-build`, for a cache one build owns (the runtime image's, the -# builder base's: one install, one top binary): everything the build did -# not use goes, so the cache is exactly one build's worth (superseded +# Bounds a cargo compile cache one build owns (the runtime image's, the +# builder base's, a worker image's) to what that build used. Run after a +# `cargo build` whose JSON messages (`--message-format=json...`) went to +# a file: cargo reports there every unit the build used, the ones it +# reused untouched included, and everything else goes (superseded # dependency versions, engine artifacts of an older lock, package slots # the stdlib moved past). # -# `aged`, for the cache every worker build on a host shares: what one -# project's build did not use may be another project's, so only the -# per-project units (the `pkg_` package crates, the worker crate, the -# `weft-isolate` helper) are bounded, by when a build last used them and -# by how many versions of each stay. The engine and the shared -# dependencies are never touched. +# usage: weft-cache-gc.sh this-build # # A unit is known by the hash cargo names all its files with: # `deps/lib-.rlib`, `deps/-.d`, @@ -71,93 +60,38 @@ unused_in() { }' } -if [ "$mode" = this-build ]; then - if [ -n "$untraced" ]; then - echo "weft: cannot tell which compiled unit is$untraced (cargo copied it instead of linking it); the compile cache is left as it is" >&2 - exit 0 - fi - for dir in "$deps" "$fingerprints" "$scripts"; do - [ -d "$dir" ] || continue - for name in $(unused_in "$dir"); do - rm -rf "${dir:?}/$name" - done - done - # Incremental state is kept per crate in a directory cargo's hash does - # not name (`incremental/-`), so it is bounded by - # count: a crate keeps as many directories as this build used units of - # it, the most recently written first (rustc writes into a crate's - # directory each time it compiles it). A unit's dep-info names its - # crate: in `deps/`, or for a build script in its own `build/` dir. - incremental="$release/incremental" - if [ -d "$incremental" ]; then - crates="$(mktemp)" - { ls -A "$deps"; for dir in "$scripts"/*/; do [ -d "$dir" ] && ls -A "$dir"; done; } | awk -v used="$used" ' - BEGIN { while ((getline h < used) > 0) keep[h] = 1 } - /\.d$/ { - unit = $0; sub(/\.d$/, "", unit) - n = split(unit, part, "-") - if (part[n] in keep) { sub(/-[^-]*$/, "", unit); print unit } - }' > "$crates" - for name in $(ls -t "$incremental" | awk -v crates="$crates" ' - BEGIN { while ((getline c < crates) > 0) wanted[c]++ } - { crate = $0; sub(/-[^-]*$/, "", crate); if (++seen[crate] > wanted[crate]) print }'); do - rm -rf "${incremental:?}/$name" - done - rm -f "$crates" - fi +[ "$mode" = this-build ] || { echo "weft: unknown sweep '$mode' (this-build)" >&2; exit 1; } +if [ -n "$untraced" ]; then + echo "weft: cannot tell which compiled unit is$untraced (cargo copied it instead of linking it); the compile cache is left as it is" >&2 exit 0 fi - -[ "$mode" = aged ] || { echo "weft: unknown sweep '$mode' (this-build or aged)" >&2; exit 1; } -worker="$4"; days="$5"; versions="$6" -# Mark: every per-project unit this build used is touched, so its date is -# when a build last used it rather than when one compiled it. Only those: -# a shared unit dated after its dependents would make cargo rebuild them. -for dir in "$deps" "$fingerprints"; do +for dir in "$deps" "$fingerprints" "$scripts"; do [ -d "$dir" ] || continue - for name in $(ls -A "$dir" | awk -v used="$used" -v worker="$worker" ' - BEGIN { while ((getline h < used) > 0) keep[h] = 1 } - { - unit = $0; sub(/\..*/, "", unit); sub(/^lib/, "", unit) - n = split(unit, part, "-") - if (!(part[n] in keep)) next - if (unit ~ /^pkg_/ || unit ~ /^weft[-_]isolate-/ || index(unit, worker "-") == 1) print $0 - }'); do - touch -c "$dir/$name" + for name in $(unused_in "$dir"); do + rm -rf "${dir:?}/$name" done done -# Bound: of each package's compiled versions (every stem of one crate -# name), of the worker crate's and of the weft-isolate helper's, only the -# newest few stay. -drop_stem() { - rm -f "$deps/$1" "$deps/$1".* "$deps/lib$1".* - rm -rf "$fingerprints"/*-"${1##*-}" -} -for name in $(for d in "$deps"/pkg_*.d "$deps"/"$worker"-*.d "$deps"/weft_isolate-*.d; do [ -e "$d" ] && basename "$d" .d; done | sed 's/-[^-]*$//' | sort -u); do - n=0 - for d in $(ls -t "$deps"/"$name"-*.d 2>/dev/null); do - n=$((n + 1)) - [ "$n" -le "$versions" ] && continue - drop_stem "$(basename "$d" .d)" +# Incremental state is kept per crate in a directory cargo's hash does +# not name (`incremental/-`), so it is bounded by +# count: a crate keeps as many directories as this build used units of +# it, the most recently written first (rustc writes into a crate's +# directory each time it compiles it). A unit's dep-info names its +# crate: in `deps/`, or for a build script in its own `build/` dir. +incremental="$release/incremental" +if [ -d "$incremental" ]; then + crates="$(mktemp)" + { ls -A "$deps"; for dir in "$scripts"/*/; do [ -d "$dir" ] && ls -A "$dir"; done; } | awk -v used="$used" ' + BEGIN { while ((getline h < used) > 0) keep[h] = 1 } + /\.d$/ { + unit = $0; sub(/\.d$/, "", unit) + n = split(unit, part, "-") + if (part[n] in keep) { sub(/-[^-]*$/, "", unit); print unit } + }' > "$crates" + for name in $(ls -t "$incremental" | awk -v crates="$crates" ' + BEGIN { while ((getline c < crates) > 0) wanted[c]++ } + { crate = $0; sub(/-[^-]*$/, "", crate); if (++seen[crate] > wanted[crate]) print }'); do + rm -rf "${incremental:?}/$name" done -done -# Expire: a per-project crate any of whose files is older than the -# retention period (a used crate had every file touched above; an orphan -# missing its dep-info still ages by its rlib or fingerprint). -keep=$((days - 1)) -for file in "$deps"/pkg_*.d "$deps"/libpkg_*.rlib "$deps"/libpkg_*.rmeta "$deps"/"$worker"-*.d \ - "$fingerprints"/pkg_* "$fingerprints"/"$worker"-*; do - [ -e "$file" ] || continue - [ -n "$(find "$file" -maxdepth 0 -mtime +"$keep")" ] || continue - stem="$(basename "$file")" - case "$stem" in *.d|*.rlib|*.rmeta) stem="${stem%.*}";; esac - stem="${stem#lib}" - drop_stem "$stem" -done -# The worker crate's build-script units carry their own hashes; they age -# on their own and are cheap to run again. -for dir in "$scripts"/"$worker"-*/; do - [ -d "$dir" ] || continue - [ -n "$(find "$dir" -maxdepth 0 -mtime +"$keep")" ] && rm -rf "$dir" -done + rm -f "$crates" +fi exit 0 diff --git a/crates/weft-compiler/src/worker_image.rs b/crates/weft-compiler/src/worker_image.rs index b4b156e32..d4f32ca4e 100644 --- a/crates/weft-compiler/src/worker_image.rs +++ b/crates/weft-compiler/src/worker_image.rs @@ -256,8 +256,6 @@ pub fn emit( .replace("{{left_out_record_path}}", weft_core::builds::LEFT_OUT_RECORD_PATH) .replace("{{cache_gc_script}}", CACHE_GC_SCRIPT_NAME) .replace("{{cargo_messages}}", CARGO_MESSAGES_PATH) - .replace("{{cache_retention_days}}", &WORKER_CACHE_RETENTION_DAYS.to_string()) - .replace("{{cache_versions_kept}}", &WORKER_CACHE_VERSIONS_KEPT.to_string()) .replace( "{{install_builder_base}}", &render_builder_base(base.manager), @@ -693,11 +691,9 @@ fn rhel_family_version(tag: &str) -> Option<&'static str> { /// a slot of its own and can never be mistaken for the stock one) and /// on every staged package file carrying the date its content gives it /// (`build::package_source_mtime`), so a package compiled from another -/// checkout still counts. Every distinct content compiled keeps its -/// slot's artifacts until no build has used them for -/// [`WORKER_CACHE_RETENTION_DAYS`] ([`CACHE_GC_SCRIPT`] runs after every -/// such build); the mount is `sharing=locked`, so a second build waits -/// for the first rather than race it; +/// checkout still counts. After every such build, [`CACHE_GC_SCRIPT`] +/// drops what that build did not use; the mount is `sharing=locked`, so +/// a second build waits for the first rather than race it; /// - seeds the worker crate's `Cargo.lock` from `{{seed_lock}}`: on the /// builder base, [`BASE_WORKER_LOCK`], the lock the stock worker /// resolved to, so every crate the base compiled resolves to the SAME @@ -747,7 +743,7 @@ const CARGO_BUILD_RUN_FRAGMENT: &str = concat!( " && \"$CARGO_TARGET_DIR\"/release/weft-isolate /work/{{isolate_candidates}} /work/{{left_out_record}} {{cargo_messages}} \\\n", " && cp \"$CARGO_TARGET_DIR\"/release/{{binary_name}} /worker \\\n", " && ( [ \"$CARGO_TARGET_DIR\" != /cache/target ] \\\n", - " || sh /work/{{cache_gc_script}} aged /cache/target/release {{cargo_messages}} {{binary_name}} {{cache_retention_days}} {{cache_versions_kept}} \\\n", + " || sh /work/{{cache_gc_script}} this-build /cache/target/release {{cargo_messages}} \\\n", " || echo 'weft: the compile cache sweep failed; the build is unaffected' >&2 )\n", ); @@ -772,19 +768,6 @@ const WORKSPACE_LOCK: &str = "/weft/Cargo.lock"; /// look for. pub const WORKER_CACHE_MOUNT_ID_PREFIX: &str = "weft-worker-target-"; -/// How long a compiled node package stays in the shared cache after the -/// last build that linked it. Every distinct content of a package keeps -/// its own compiled copy (`codegen::write_package_crates`), so without -/// this a host that edits nodes would keep every version it ever built. -pub const WORKER_CACHE_RETENTION_DAYS: u32 = 30; - -/// How many compiled versions of one node package (and of the worker -/// crate on top) stay in the cache, newest by last link: the current one -/// and a couple of edits back, so going back an edit relinks instead of -/// recompiling, while a node edited all day keeps three copies, not one -/// per save. -pub const WORKER_CACHE_VERSIONS_KEPT: u32 = 3; - /// File name of [`CACHE_GC_SCRIPT`] inside the generated crate (`/work` /// in the builder stage). pub const CACHE_GC_SCRIPT_NAME: &str = "weft-cache-gc.sh"; @@ -795,33 +778,17 @@ pub const CACHE_GC_SCRIPT_NAME: &str = "weft-cache-gc.sh"; /// change to it for a build script that watches the whole package. pub const CARGO_MESSAGES_PATH: &str = "/tmp/weft-cargo-messages.json"; -/// The compile-cache sweep, the one that bounds every cache weft compiles -/// in: it reads what the build just used off cargo's JSON messages (every -/// unit, the ones cargo reused untouched included), and either drops -/// everything else (`this-build`, for a cache one build owns: the runtime -/// image's, the builder base's) or marks the per-project crates used and -/// expires the ones no build used for a while (`aged`, for a worker -/// cache every project on a machine shares). Its own header says how. -/// -/// Every worker compiled on a machine runs it in its lane's cache -/// (`weft_platform_local::host_build`), and every built-in-template worker -/// image build that keeps its cache runs it inside the mount, after the -/// binary is copied out (a custom `dockerfile_template` writes its own RUN -/// and gets none of this), with [`WORKER_CACHE_RETENTION_DAYS`] and -/// [`WORKER_CACHE_VERSIONS_KEPT`]. It can never fail a build. -/// -/// "Used" is stored, for `aged`, on the crate's own files: cargo rewrites -/// them when it compiles the crate and leaves them alone when it reuses -/// it, so touching them on every build makes their date "last used" -/// rather than "last compiled". Two families are per project and expire: -/// the `pkg_` package crates (every distinct content of a package keeps -/// its own compiled copy, `codegen::write_package_crates`), and the top -/// worker crate (its build-script units included), whose metadata hash -/// folds in its package deps so every distinct project leaves its own -/// copy. The `weft-isolate` helper is compiled again by each weft that -/// changes its source, as a package named by that source -/// (`codegen::isolate_dir_name`), so its compiled versions are bounded -/// like a package's. +/// The sweep of an image build's compile cache: it reads what the build +/// just used off cargo's JSON messages (every unit, the ones cargo reused +/// untouched included) and drops everything else, so a cache one build +/// owns (the runtime image's, the builder base's, a worker image's kept by +/// `weft build-images`) holds exactly one build's worth. Every +/// built-in-template worker image build that keeps its cache runs it +/// inside the mount, after the binary is copied out (a custom +/// `dockerfile_template` writes its own RUN and gets none of this). It can +/// never fail a build. A worker compiled on the machine is swept by +/// `weft_platform_local::host_build` instead, by what the programs on the +/// machine were built from. // SYNC: the script's path <-> deploy/docker/runtime.Dockerfile (the sweep after `cargo build`), hash::CACHE_GC_SCRIPT_PATH pub const CACHE_GC_SCRIPT: &str = include_str!("weft-cache-gc.sh"); @@ -1367,10 +1334,8 @@ mod tests { ); assert!(out.writes_left_out_record); assert!(!out.body.contains("{{"), "every token is substituted: {}", out.body); - let sweep = format!( - "sh /work/{CACHE_GC_SCRIPT_NAME} aged /cache/target/release {CARGO_MESSAGES_PATH} worker_test {WORKER_CACHE_RETENTION_DAYS} {WORKER_CACHE_VERSIONS_KEPT}" - ); - assert!(out.body.contains(&sweep), "every build expires unused per-project crates from the shared cache: {}", out.body); + let sweep = format!("sh /work/{CACHE_GC_SCRIPT_NAME} this-build /cache/target/release {CARGO_MESSAGES_PATH}"); + assert!(out.body.contains(&sweep), "every build keeps only what it used in the cache: {}", out.body); assert!( out.body.find("/worker \\").unwrap() < out.body.find(&sweep).unwrap() && out.body.contains("|| echo 'weft: the compile cache sweep failed"), "the sweep runs after the binary is out and can never fail the build: {}", @@ -1607,93 +1572,4 @@ mod tests { assert!(cache.deps().join(format!("libserde-{}.rlib", H[1])).exists(), "an untraced binary: nothing goes"); assert!(cache.deps().join(format!("weft_runtime-{}", H[4])).exists()); } - - /// The aged sweep marks the per-project units this build used (even - /// when cargo reused them untouched, and never the shared ones), - /// expires per-project units no build used within the retention period - /// (an orphan without its dep-info included, and the top worker - /// crate's per-project copies), and never touches the shared crates. - #[test] - fn the_aged_sweep_expires_only_per_project_crates_no_build_used() { - let cache = FakeCache::new(); - let (deps, fingerprints) = (cache.deps(), cache.fingerprints()); - let age_unit = |krate: &str, hash: &str| { - for file in [format!("lib{krate}-{hash}.rlib"), format!("lib{krate}-{hash}.rmeta"), format!("{krate}-{hash}.d")] { - cache.age(&deps.join(file), 40); - } - }; - cache.lib("pkg_basic", H[0]); // used by this build: kept and marked - age_unit("pkg_basic", H[0]); - cache.lib("pkg_basic", H[1]); // an earlier content, unused: expired - age_unit("pkg_basic", H[1]); - std::fs::write(deps.join(format!("libpkg_ai-{}.rlib", H[2])), b"orphan").unwrap(); // no dep-info left: expired - cache.age(&deps.join(format!("libpkg_ai-{}.rlib", H[2])), 40); - cache.lib("weft-engine", H[3]); // shared: never touched - age_unit("weft_engine", H[3]); - cache.bin("worker_test", H[4], Some("worker_test")); // this build's top crate - cache.bin("worker_test", H[5], None); // another project's, long unused - cache.age(&deps.join(format!("worker_test-{}", H[5])), 40); - cache.age(&deps.join(format!("worker_test-{}.d", H[5])), 40); - let scripts = cache.release.join("build"); - for (dir, days) in [(format!("worker_test-{}", H[6]), 0), (format!("worker_test-{}", H[7]), 40)] { - std::fs::create_dir_all(scripts.join(&dir).join("out")).unwrap(); - cache.age(&scripts.join(&dir), days); - } - cache.used(&[("pkg_basic", H[0]), ("weft_engine", H[3])], &[], "worker_test"); - - cache.sweep(&["aged", "worker_test", &WORKER_CACHE_RETENTION_DAYS.to_string(), &WORKER_CACHE_VERSIONS_KEPT.to_string()]); - - for file in [format!("pkg_basic-{}.d", H[0]), format!("libpkg_basic-{}.rlib", H[0]), format!("libpkg_basic-{}.rmeta", H[0])] { - let marked = std::fs::metadata(deps.join(&file)).unwrap().modified().unwrap(); - assert!(marked.elapsed().unwrap().as_secs() < 3600, "{file} is marked used now"); - } - for gone in [format!("libpkg_basic-{}.rlib", H[1]), format!("pkg_basic-{}.d", H[1]), format!("libpkg_ai-{}.rlib", H[2]), format!("worker_test-{}", H[5])] { - assert!(!deps.join(&gone).exists(), "{gone} expired"); - } - assert!(!fingerprints.join(format!("pkg_basic-{}", H[1])).exists() && !fingerprints.join(format!("worker_test-{}", H[5])).exists()); - assert!(deps.join(format!("worker_test-{}", H[4])).exists(), "this build's worker crate stays"); - let engine = std::fs::metadata(deps.join(format!("libweft_engine-{}.rlib", H[3]))).unwrap().modified().unwrap(); - assert!(engine.elapsed().unwrap().as_secs() > 3600, "a shared unit is never marked, used or not"); - assert!(scripts.join(format!("worker_test-{}", H[6])).exists(), "a fresh build-script unit stays"); - assert!(!scripts.join(format!("worker_test-{}", H[7])).exists(), "an old build-script unit goes"); - } - - /// Of one package's compiled versions, only the newest few by last - /// use stay, however recent the others are; another package counts - /// on its own, and the `weft-isolate` helper's versions are bounded - /// the same way, its fingerprint (named by its package) included. - #[test] - fn the_aged_sweep_keeps_only_the_newest_versions_of_a_package() { - let cache = FakeCache::new(); - let deps = cache.deps(); - let version = |krate: &str, hash: &str, minutes_ago: i64| { - cache.lib(krate, hash); - let when = filetime::FileTime::from_unix_time(filetime::FileTime::now().unix_seconds() - minutes_ago * 60, 0); - for file in [format!("{krate}-{hash}.d"), format!("lib{krate}-{hash}.rlib")] { - filetime::set_file_mtime(deps.join(file), when).unwrap(); - } - }; - let mine = ["00000000000000b1", "00000000000000b2", "00000000000000b3", "00000000000000b4", "00000000000000b5"]; - for (i, hash) in mine.iter().enumerate() { - version("pkg_mine", hash, i as i64 + 1); // b1 is the newest - } - version("pkg_other", "00000000000000c1", 10); - let helper = ["00000000000000d1", "00000000000000d2", "00000000000000d3", "00000000000000d4"]; - for (i, hash) in helper.iter().enumerate() { - version("weft_isolate", hash, i as i64 + 1); - std::fs::create_dir_all(cache.fingerprints().join(format!("weft-isolate-0123-{hash}"))).unwrap(); - } - cache.used(&[], &[], "worker_test"); - cache.sweep(&["aged", "worker_test", &WORKER_CACHE_RETENTION_DAYS.to_string(), &WORKER_CACHE_VERSIONS_KEPT.to_string()]); - for kept in [mine[0], mine[1], mine[2]] { - assert!(deps.join(format!("libpkg_mine-{kept}.rlib")).exists(), "{kept} stays"); - } - assert!(deps.join("libpkg_other-00000000000000c1.rlib").exists(), "another package counts on its own"); - assert!(deps.join(format!("libweft_isolate-{}.rlib", helper[2])).exists(), "the newest helpers stay"); - for gone in [mine[3], mine[4]] { - assert!(!deps.join(format!("libpkg_mine-{gone}.rlib")).exists() && !cache.fingerprints().join(format!("pkg_mine-{gone}")).exists(), "{gone} goes"); - } - assert!(!deps.join(format!("libweft_isolate-{}.rlib", helper[3])).exists()); - assert!(!cache.fingerprints().join(format!("weft-isolate-0123-{}", helper[3])).exists(), "the helper's fingerprint goes with it"); - } } diff --git a/crates/weft-core/src/builds.rs b/crates/weft-core/src/builds.rs index fe87adb47..d5939f127 100644 --- a/crates/weft-core/src/builds.rs +++ b/crates/weft-core/src/builds.rs @@ -124,6 +124,14 @@ pub enum BuildRecipe { /// The file in a host build's context that says how to build it. pub const HOST_RECIPE_FILE: &str = "host-recipe.json"; +/// The lock beside `shared`, the folder of crates every host build on a +/// machine shares: held shared by every build from before it stages into +/// the folder until it is compiled, and alone by whatever removes from it, +/// so nothing a build is about to compile is removed. +pub fn shared_crates_lock(shared: &std::path::Path) -> std::path::PathBuf { + shared.with_extension("lock") +} + /// How a worker is compiled on the machine that runs it, written by the /// compiler (`weft_compiler::build`) into the build's context and run by /// the machine's image builder (`weft_platform_local::images`): the same @@ -148,12 +156,6 @@ pub struct HostRecipe { pub seed_lock: std::path::PathBuf, /// The worker binary's name under `/release/`. pub binary_name: String, - /// The compile cache sweep (`weft-cache-gc.sh`). - pub cache_gc_script: std::path::PathBuf, - /// How many days an unused compiled piece stays in the cache. - pub cache_retention_days: u32, - /// How many versions of one piece the cache keeps. - pub cache_versions_kept: u32, } /// Where a worker image (a cloud's) keeps the record of the node packages diff --git a/crates/weft-core/src/file_lock.rs b/crates/weft-core/src/file_lock.rs index da6ed2434..26f69a267 100644 --- a/crates/weft-core/src/file_lock.rs +++ b/crates/weft-core/src/file_lock.rs @@ -27,6 +27,15 @@ pub fn try_hold(path: &Path) -> io::Result> { } } +/// Hold `path` together with every other shared holder, waiting while an +/// exclusive one ([`hold`], [`try_hold`]) has it: many readers, one +/// writer. +pub fn hold_shared(path: &Path) -> io::Result { + let file = open(path)?; + file.lock_shared()?; + Ok(file) +} + /// Hold the lock on `path` like [`hold`], but only if the file is there: /// `None` when it is not, and nothing is made. For a reader that must /// never bring back a folder another process just removed. diff --git a/crates/weft-dispatcher/src/build/mod.rs b/crates/weft-dispatcher/src/build/mod.rs index 25880b4df..1301267c4 100644 --- a/crates/weft-dispatcher/src/build/mod.rs +++ b/crates/weft-dispatcher/src/build/mod.rs @@ -248,10 +248,6 @@ impl VersionBuilder { .tempdir() .context("create the build's working directory")?; let root = workdir.path().to_path_buf(); - // The image contexts are staged under it, and a build started from - // one may read it after this request answered: every start keeps a - // clone, and the directory goes with the last (`Staging`). - let staging = Staging::new(workdir); source::materialize(storage.as_ref(), &self.blobs, tenant, &source.manifest, &root).await?; let files = Arc::new(source::VersionFiles::new(root.clone(), source.manifest.clone(), storage.clone(), tenant.to_string())); @@ -319,6 +315,12 @@ impl VersionBuilder { images, ask, }; + // The image contexts are staged under the working directory, and a + // build started from one may read it after this request answered: + // every start keeps a clone, and the directory goes with the last + // (`Staging`), and with it the claim on the shared crates the + // worker staged. + let staging = Staging::new((workdir, plan.hold.clone())); self.start_images(version, &plan.images, &staging, project_id, tenant, gate, hold).await } diff --git a/crates/weft-e2e/README.md b/crates/weft-e2e/README.md index 9cc9729f2..345a57c5f 100644 --- a/crates/weft-e2e/README.md +++ b/crates/weft-e2e/README.md @@ -57,7 +57,7 @@ connections by the ledger every test writes them into while they live what a run on another kept. The `weft clean` step reaches past the suite: it removes every worker and infra image no project of the install references (yours too, once nothing runs them), the worker programs no install on the machine holds, -and the shared build crates no build used for 30 days, and it drops every +and the shared build crates no program on the machine was built from, and it drops every runtime image, and every builder base, node-test image, standard worker image and image compile cache an older weft left, that no install on the machine records using. While some install has no record yet, it removes nothing of diff --git a/crates/weft-platform-local/src/cargo_cache.rs b/crates/weft-platform-local/src/cargo_cache.rs new file mode 100644 index 000000000..32abd81b4 --- /dev/null +++ b/crates/weft-platform-local/src/cargo_cache.rs @@ -0,0 +1,477 @@ +//! Keeps a cargo build folder to what is still used. +//! +//! cargo never deletes anything from its build folder: every time a +//! dependency moves or weft is updated, it compiles a new copy of each +//! piece under a new name and leaves the old one beside it. Each piece (a +//! "unit") is named by a 16-hex hash that every one of its files carries: +//! `deps/lib-.rlib`, `deps/-.d`, +//! `examples/-`, `.fingerprint/-/`, +//! `build/-/`. Removing all of a unit's files together leaves cargo nothing stale: a build that +//! wants the unit again compiles it again. +//! +//! Two ways to know what is used, one per kind of folder: +//! - [`keep_used`]: a folder whose builds say what they used. cargo's JSON +//! messages list every unit a build used, the ones it reused untouched +//! included ([`used_units`]); a compile lane keeps the units some program +//! still on the machine was built from (`host_build`). +//! - [`drop_superseded`]: a folder anybody runs cargo in (a checkout's +//! `target/`, where tests, clippy and setup all build). There, each +//! unit's fingerprint says how it was built (its features, profile, +//! compiler, and the same for every unit it was built against). Of the +//! top units (the ones no other unit was built against: test and +//! program binaries) built the same way, only the newest is what +//! today's sources and lock build; what those were built against is +//! used, and every other unit goes. + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::path::{Path, PathBuf}; + +/// What a sweep removed. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct Swept { + pub units: usize, + pub bytes: u64, +} + +impl std::ops::AddAssign for Swept { + fn add_assign(&mut self, other: Self) { + self.units += other.units; + self.bytes += other.bytes; + } +} + +/// The unit hash a file name in `deps/`, `.fingerprint/` or `build/` +/// carries: the 16 hex after its last `-`, before any extension. +fn unit_of(name: &str) -> Option<&str> { + let stem = name.split('.').next()?; + let (_, hash) = stem.rsplit_once('-')?; + (hash.len() == 16 && hash.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase())).then_some(hash) +} + +/// Every unit cargo's JSON messages (`--message-format=json...`, one +/// object per line) say the build used, as hashes. A binary is reported +/// where cargo linked it, at the top of `profile_dir`; that file is a hard +/// link of its unit's file in `deps/`, which is how it is traced. +pub fn used_units(messages: &[u8], profile_dir: &Path) -> anyhow::Result> { + let mut used = BTreeSet::new(); + let mut top = Vec::new(); + for line in messages.split(|b| *b == b'\n') { + let Ok(message) = serde_json::from_slice::(line) else { continue }; + let paths = message["filenames"].as_array().into_iter().flatten().chain(std::iter::once(&message["out_dir"])); + for path in paths.filter_map(|p| p.as_str()).map(Path::new) { + let Ok(rest) = path.strip_prefix(profile_dir) else { continue }; + let mut parts = rest.components().map(|c| c.as_os_str().to_string_lossy()); + match (parts.next(), parts.next()) { + (Some(dir), Some(name)) if dir == "deps" || dir == "build" => { + if let Some(hash) = unit_of(&name) { + used.insert(hash.to_string()); + } + } + (Some(_), None) => top.push(path.to_path_buf()), + _ => {} + } + } + } + if top.is_empty() { + return Ok(used); + } + let deps = profile_dir.join("deps"); + let ids: Vec<_> = top.iter().filter_map(|path| file_id(path)).collect(); + for entry in std::fs::read_dir(&deps).map_err(|e| anyhow::anyhow!("read {}: {e}", deps.display()))? { + let entry = entry.map_err(|e| anyhow::anyhow!("read {}: {e}", deps.display()))?; + if file_id(&entry.path()).is_some_and(|id| ids.contains(&id)) { + if let Some(hash) = unit_of(&entry.file_name().to_string_lossy()) { + used.insert(hash.to_string()); + } + } + } + Ok(used) +} + +/// The folders right under `under` that the units cargo's messages report +/// were compiled from (each unit's `manifest_path`). +pub fn source_folders(messages: &[u8], under: &Path) -> BTreeSet { + messages + .split(|b| *b == b'\n') + .filter_map(|line| serde_json::from_slice::(line).ok()) + .filter_map(|message| { + let path = Path::new(message["manifest_path"].as_str()?).strip_prefix(under).ok()?.components().next()?; + Some(path.as_os_str().to_string_lossy().into_owned()) + }) + .collect() +} + +/// The device and inode of a file: equal for two hard links of one file. +fn file_id(path: &Path) -> Option<(u64, u64)> { + use std::os::unix::fs::MetadataExt; + std::fs::metadata(path).ok().map(|m| (m.dev(), m.ino())) +} + +/// Remove from `profile_dir` (a `target/`) every unit whose hash +/// is not in `used`. +pub fn keep_used(profile_dir: &Path, used: &BTreeSet) -> anyhow::Result { + let gone: BTreeSet = units_in(profile_dir)?.into_iter().filter(|hash| !used.contains(hash)).collect(); + remove_units(profile_dir, &gone) +} + +/// Remove from every profile folder of `target` (`debug`, `release`, ...) +/// the units no up-to-date build uses. +pub fn drop_superseded(target: &Path) -> anyhow::Result { + let mut swept = Swept::default(); + for entry in std::fs::read_dir(target).map_err(|e| anyhow::anyhow!("read {}: {e}", target.display()))? { + let profile_dir = entry.map_err(|e| anyhow::anyhow!("read {}: {e}", target.display()))?.path(); + if profile_dir.join(".fingerprint").is_dir() { + swept += drop_superseded_in(&profile_dir)?; + } + } + Ok(swept) +} + +/// One unit's fingerprint folder, read. +struct Unit { + /// The package, from the folder's name (`-`). + package: String, + /// When cargo last compiled it. + compiled: std::time::SystemTime, + /// One entry per fingerprint file (`lib-`, `test-lib-`, + /// `run-build-script-...`). + builds: Vec, +} + +/// One fingerprint file of a unit: its kind, how it was built, and what +/// against (each dep's name and the fingerprint it had). +type Build = (String, String, Vec<(String, String)>); + +fn drop_superseded_in(profile_dir: &Path) -> anyhow::Result { + let fingerprints = profile_dir.join(".fingerprint"); + let mut units: BTreeMap = BTreeMap::new(); + // The fingerprint a unit's file records (as hex) -> the unit. + let mut by_fingerprint: HashMap = HashMap::new(); + for entry in std::fs::read_dir(&fingerprints).map_err(|e| anyhow::anyhow!("read {}: {e}", fingerprints.display()))? { + let entry = entry.map_err(|e| anyhow::anyhow!("read {}: {e}", fingerprints.display()))?; + let name = entry.file_name().to_string_lossy().to_string(); + let Some(hash) = unit_of(&name).map(str::to_string) else { continue }; + let package = name[..name.len() - hash.len() - 1].to_string(); + let dir = entry.path(); + let mut builds = Vec::new(); + // When cargo last wrote any of its fingerprint files: a build + // script's run has no `invoked.timestamp`. + let mut compiled = std::time::UNIX_EPOCH; + for file in std::fs::read_dir(&dir).map_err(|e| anyhow::anyhow!("read {}: {e}", dir.display()))? { + let file = file.map_err(|e| anyhow::anyhow!("read {}: {e}", dir.display()))?.path(); + if file.extension().is_none_or(|ext| ext != "json") { + continue; + } + let Ok(record) = std::fs::read(&file).map_err(anyhow::Error::from).and_then(|b| Ok(serde_json::from_slice::(&b)?)) else { + continue; + }; + if let Ok(fingerprint) = std::fs::read_to_string(file.with_extension("")) { + by_fingerprint.insert(fingerprint.trim().to_string(), hash.clone()); + } + if let Ok(written) = std::fs::metadata(&file).and_then(|m| m.modified()) { + compiled = compiled.max(written); + } + let kind = file.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_default(); + // Everything that says how it was built, except what changes + // with the sources or the lock: `path` (a registry crate's + // folder names its version), `local` and the deps' own hashes. + let how = ["rustc", "features", "target", "profile", "rustflags", "config", "compile_kind"] + .iter() + .map(|key| record[*key].to_string()) + .collect::>() + .join(" "); + let against = record["deps"] + .as_array() + .into_iter() + .flatten() + // cargo writes a fingerprint as the hex of its little-endian bytes. + .filter_map(|dep| Some((dep[1].as_str()?.to_string(), hex::encode(dep[3].as_u64()?.to_le_bytes())))) + .collect(); + builds.push((kind, how, against)); + } + // A folder cargo has not written a fingerprint into yet is left + // alone. + if builds.is_empty() { + continue; + } + builds.sort(); + units.insert(hash, Unit { package, compiled, builds }); + } + + // How a unit was built, its deps' included, as one key: two units with + // the same key are generations of one piece. A dep is found by the + // fingerprint the unit recorded for it; one recompiled since (the + // unit itself is then out of date, and rebuilt in place by its next + // build) counts by its name alone. + fn key_of(hash: &str, units: &BTreeMap, by_fingerprint: &HashMap, keys: &mut HashMap) -> String { + if let Some(key) = keys.get(hash) { + return key.clone(); + } + let unit = &units[hash]; + let mut key = unit.package.clone(); + for (kind, how, against) in &unit.builds { + key.push_str(&format!("|{kind} {how}")); + let mut deps: Vec = against + .iter() + .map(|(name, fingerprint)| match by_fingerprint.get(fingerprint) { + Some(dep) if dep != hash => format!("{name}={}", key_of(dep, units, by_fingerprint, keys)), + _ => format!("{name}=?"), + }) + .collect(); + deps.sort(); + key.push_str(&format!("[{}]", deps.join(","))); + } + let key = hex::encode(::digest(key.as_bytes())); + keys.insert(hash.to_string(), key.clone()); + key + } + + // The tops: units no other unit was built against (test and program + // binaries, the libraries a build asked for). Of the units built the + // same way only the newest is today's; every unit a top of today's was + // built against is used, and nothing else is. + let mut keys = HashMap::new(); + let referenced: HashSet<&String> = units + .values() + .flat_map(|unit| unit.builds.iter().flat_map(|(_, _, against)| against.iter())) + .filter_map(|(_, fingerprint)| by_fingerprint.get(fingerprint)) + .collect(); + let newest_by_key = |hashes: &mut dyn Iterator, keys: &mut HashMap| { + let mut newest: HashMap = HashMap::new(); + for hash in hashes { + let key = key_of(hash, &units, &by_fingerprint, keys); + if newest.get(&key).is_none_or(|other| units[*other].compiled < units[hash].compiled) { + newest.insert(key, hash); + } + } + newest.into_values().cloned().collect::>() + }; + let mut used: HashSet = HashSet::new(); + // A top counts only when no unit built the same way is newer: an old + // generation of a library nothing is built against any more is a top + // too, and its newer generation is not. + let newest = newest_by_key(&mut units.keys(), &mut keys); + let mut todo: Vec = newest.into_iter().filter(|hash| !referenced.contains(hash)).collect(); + while let Some(hash) = todo.pop() { + if !used.insert(hash.clone()) { + continue; + } + for (_, _, against) in &units[&hash].builds { + for (name, fingerprint) in against { + match by_fingerprint.get(fingerprint) { + Some(dep) => todo.push(dep.clone()), + // Built against a dep since recompiled in its place: + // the unit is out of date, and its next build uses the + // newest build of that dep, whichever way it is built. + None => todo.extend(newest_by_key( + &mut units.iter().filter(|(_, unit)| unit.package.replace('-', "_") == *name).map(|(hash, _)| hash), + &mut keys, + )), + } + } + } + } + let gone: BTreeSet = units.keys().filter(|hash| !used.contains(*hash)).cloned().collect(); + for hash in &gone { + units.remove(hash); + } + let mut swept = remove_units(profile_dir, &gone)?; + swept += trim_incremental(profile_dir, &units)?; + Ok(swept) +} + +/// rustc keeps a crate's incremental state in `incremental/-`, +/// a name cargo's hash is not in, so it is bounded by count: a crate keeps +/// as many folders as it has units left, the most recently written first. +fn trim_incremental(profile_dir: &Path, units: &BTreeMap) -> anyhow::Result { + let incremental = profile_dir.join("incremental"); + let Ok(entries) = std::fs::read_dir(&incremental) else { return Ok(Swept::default()) }; + let mut wanted: HashMap = HashMap::new(); + for unit in units.values() { + *wanted.entry(unit.package.replace('-', "_")).or_default() += 1; + } + let mut by_crate: HashMap> = HashMap::new(); + for entry in entries { + let path = entry.map_err(|e| anyhow::anyhow!("read {}: {e}", incremental.display()))?.path(); + let name = path.file_name().unwrap_or_default().to_string_lossy().to_string(); + let Some((krate, _)) = name.rsplit_once('-') else { continue }; + let written = std::fs::metadata(&path).and_then(|m| m.modified()).unwrap_or(std::time::UNIX_EPOCH); + by_crate.entry(krate.to_string()).or_default().push((written, path)); + } + let mut swept = Swept::default(); + for (krate, mut dirs) in by_crate { + dirs.sort_by_key(|dir| std::cmp::Reverse(dir.0)); + for (_, dir) in dirs.into_iter().skip(wanted.get(&krate).copied().unwrap_or(0)) { + swept.bytes += size_of(&dir); + std::fs::remove_dir_all(&dir).map_err(|e| anyhow::anyhow!("remove {}: {e}", dir.display()))?; + } + } + Ok(swept) +} + +/// Every unit hash with a file in `profile_dir`. +fn units_in(profile_dir: &Path) -> anyhow::Result> { + let mut units = BTreeSet::new(); + for dir in ["deps", "examples", ".fingerprint", "build"] { + let Ok(entries) = std::fs::read_dir(profile_dir.join(dir)) else { continue }; + for entry in entries { + let entry = entry.map_err(|e| anyhow::anyhow!("read {}: {e}", profile_dir.join(dir).display()))?; + if let Some(hash) = unit_of(&entry.file_name().to_string_lossy()) { + units.insert(hash.to_string()); + } + } + } + Ok(units) +} + +/// Remove every file of the units `gone` from `profile_dir`. +fn remove_units(profile_dir: &Path, gone: &BTreeSet) -> anyhow::Result { + let mut swept = Swept { units: gone.len(), bytes: 0 }; + if gone.is_empty() { + return Ok(swept); + } + let gone: HashSet<&str> = gone.iter().map(String::as_str).collect(); + for dir in ["deps", "examples", ".fingerprint", "build"] { + let dir = profile_dir.join(dir); + let Ok(entries) = std::fs::read_dir(&dir) else { continue }; + for entry in entries { + let entry = entry.map_err(|e| anyhow::anyhow!("read {}: {e}", dir.display()))?; + if !unit_of(&entry.file_name().to_string_lossy()).is_some_and(|hash| gone.contains(hash)) { + continue; + } + let path = entry.path(); + swept.bytes += size_of(&path); + let removed = if path.is_dir() { std::fs::remove_dir_all(&path) } else { std::fs::remove_file(&path) }; + removed.map_err(|e| anyhow::anyhow!("remove {}: {e}", path.display()))?; + } + } + Ok(swept) +} + +/// The bytes a file or folder takes, its contents included. +fn size_of(path: &Path) -> u64 { + let Ok(meta) = std::fs::symlink_metadata(path) else { return 0 }; + if !meta.is_dir() { + return meta.len(); + } + std::fs::read_dir(path).into_iter().flatten().flatten().map(|entry| size_of(&entry.path())).sum() +} + +#[cfg(test)] +mod tests { + use super::*; + + const A: &str = "00000000000000a1"; + const B: &str = "00000000000000b2"; + const C: &str = "00000000000000c3"; + const D: &str = "00000000000000d4"; + + fn touch(path: &Path, bytes: &[u8]) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, bytes).unwrap(); + } + + /// A unit `package-hash` built with `features` against `deps` (each a + /// dep's name and the fingerprint it recorded), whose own fingerprint + /// is `fingerprint`, compiled `age` seconds after the epoch. + fn unit(profile: &Path, package: &str, hash: &str, features: &str, fingerprint: u64, deps: &[(&str, u64)], age: u64) { + let dir = profile.join(".fingerprint").join(format!("{package}-{hash}")); + let krate = package.replace('-', "_"); + let record = serde_json::json!({ + "rustc": 1, "features": features, "target": 2, "profile": 3, "path": 4, "rustflags": [], "config": 5, "compile_kind": 0, + "deps": deps.iter().map(|(name, fp)| serde_json::json!([9, name, false, fp])).collect::>(), + "local": [], + }); + touch(&dir.join(format!("lib-{krate}.json")), record.to_string().as_bytes()); + touch(&dir.join(format!("lib-{krate}")), hex::encode(fingerprint.to_le_bytes()).as_bytes()); + touch(&dir.join("invoked.timestamp"), b""); + let at = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_000_000 + age); + std::fs::File::options().write(true).open(dir.join(format!("lib-{krate}.json"))).unwrap().set_modified(at).unwrap(); + touch(&profile.join("deps").join(format!("lib{krate}-{hash}.rlib")), b"rlib"); + touch(&profile.join("deps").join(format!("{krate}-{hash}.d")), b"d"); + } + + fn has(profile: &Path, package: &str, hash: &str) -> bool { + profile.join(".fingerprint").join(format!("{package}-{hash}")).exists() + && profile.join("deps").join(format!("lib{}-{hash}.rlib", package.replace('-', "_"))).exists() + } + + /// Of two units built the same way, the older goes with all its files; + /// one built with other features is another piece and stays, and so + /// does each dependent, told apart by how its dep was built. + #[test] + fn only_the_newest_build_of_each_piece_stays() { + let target = tempfile::tempdir().unwrap(); + let debug = target.path().join("debug"); + unit(&debug, "tokio", A, "[\"rt\"]", 0xa, &[], 1); + unit(&debug, "tokio", B, "[\"rt\"]", 0xb, &[], 2); + unit(&debug, "tokio", C, "[\"rt\", \"net\"]", 0xc, &[], 1); + unit(&debug, "weft-core", D, "[]", 0xd, &[("tokio", 0xb)], 3); + unit(&debug, "weft-core", "00000000000000e5", "[]", 0xe, &[("tokio", 0xc)], 3); + let swept = drop_superseded(target.path()).unwrap(); + assert_eq!(swept.units, 1); + assert!(!has(&debug, "tokio", A) && !debug.join("deps").join(format!("tokio-{A}.d")).exists(), "the older generation goes whole"); + assert!(has(&debug, "tokio", B) && has(&debug, "tokio", C)); + assert!(has(&debug, "weft-core", D) && has(&debug, "weft-core", "00000000000000e5"), "built against different tokios"); + } + + /// A unit built against a version of its dep since replaced is an + /// older generation of the unit built against the new one. + #[test] + fn a_unit_built_against_a_replaced_dep_is_an_older_generation() { + let target = tempfile::tempdir().unwrap(); + let release = target.path().join("release"); + unit(&release, "serde", A, "[]", 0xa, &[], 1); + unit(&release, "serde", B, "[]", 0xb, &[], 2); + unit(&release, "weft-core", C, "[]", 0xc, &[("serde", 0xa)], 1); + unit(&release, "weft-core", D, "[]", 0xd, &[("serde", 0xb)], 2); + drop_superseded(target.path()).unwrap(); + assert!(!has(&release, "serde", A) && !has(&release, "weft-core", C)); + assert!(has(&release, "serde", B) && has(&release, "weft-core", D)); + } + + /// Two versions of one crate a build uses side by side are both kept, + /// however alike they were built. + #[test] + fn two_versions_of_a_crate_in_use_both_stay() { + let target = tempfile::tempdir().unwrap(); + let debug = target.path().join("debug"); + unit(&debug, "hyper", A, "[]", 0xa, &[], 1); + unit(&debug, "hyper", B, "[]", 0xb, &[], 2); + unit(&debug, "weft-core", C, "[]", 0xc, &[("hyper", 0xa), ("hyper", 0xb)], 3); + drop_superseded(target.path()).unwrap(); + assert!(has(&debug, "hyper", A) && has(&debug, "hyper", B) && has(&debug, "weft-core", C)); + } + + /// The units a build used are read off cargo's messages, a binary + /// traced through its hard link to its unit, and everything else goes; + /// the shared folders its units came from are read off them too. + #[test] + fn a_cache_keeps_the_units_its_builds_used() { + let dir = tempfile::tempdir().unwrap(); + let release = dir.path().join("release"); + for hash in [A, B, C, D] { + unit(&release, "pkg", hash, "[]", 1, &[], 1); + } + let exe = release.join("deps").join(format!("worker-{C}")); + touch(&exe, b"\x7fELF"); + std::fs::hard_link(&exe, release.join("worker")).unwrap(); + let messages = [ + serde_json::json!({ + "reason": "compiler-artifact", "fresh": true, "filenames": [release.join(format!("deps/libpkg-{A}.rlib"))], + "manifest_path": dir.path().join("shared/pkg_x-1/Cargo.toml"), + }), + serde_json::json!({"reason": "build-script-executed", "out_dir": release.join(format!("build/pkg-{B}/out"))}), + serde_json::json!({"reason": "compiler-artifact", "filenames": [release.join("worker")]}), + ] + .iter() + .map(|m| format!("{m}\n")) + .collect::(); + let used = used_units(messages.as_bytes(), &release).unwrap(); + assert_eq!(used, [A, B, C].iter().map(|h| h.to_string()).collect()); + assert_eq!(source_folders(messages.as_bytes(), &dir.path().join("shared")), ["pkg_x-1".to_string()].into()); + keep_used(&release, &used).unwrap(); + assert!(has(&release, "pkg", A) && has(&release, "pkg", B) && exe.exists()); + assert!(!has(&release, "pkg", D) && !release.join("deps").join(format!("pkg-{D}.d")).exists()); + } +} diff --git a/crates/weft-platform-local/src/host_build.rs b/crates/weft-platform-local/src/host_build.rs index 51e52c76c..98c255352 100644 --- a/crates/weft-platform-local/src/host_build.rs +++ b/crates/weft-platform-local/src/host_build.rs @@ -12,6 +12,7 @@ //! The commands run behind [`Toolchain`], so the steps are tested against //! a fake that records them; only [`MachineToolchain`] runs anything. +use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use std::process::Stdio; @@ -93,7 +94,7 @@ pub(crate) fn mark_seed(machine: &MachineLayout) -> anyhow::Result<()> { /// copied whole into it, or a build finished in it. Written only once /// that is done, so a copy cut short leaves the lane unmarked and the next /// build copies the seed again over it. -fn mark_lane(machine: &MachineLayout, number: u32) -> anyhow::Result<()> { +pub(crate) fn mark_lane(machine: &MachineLayout, number: u32) -> anyhow::Result<()> { let mark = machine.lane_mark(number); std::fs::write(&mark, b"").map_err(|e| anyhow::anyhow!("mark compile lane {number} warm ({}): {e}", mark.display())) } @@ -221,6 +222,82 @@ pub const PROGRAM_FILE: &str = "worker"; /// The record of what the build left out, beside the program. pub const LEFT_OUT_FILE: &str = "left-out.json"; +/// What the program's build used ([`Used`]), beside the program. +pub const USED_FILE: &str = "used.json"; + +/// What a program's build used: everything here stays on the machine as +/// long as the program does, so building it again compiles nothing new. +#[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct Used { + /// The compiled units, by the hash cargo names their files with: kept + /// in every compile cache ([`sweep`]). + pub units: BTreeSet, + /// The folders of the shared crates it was compiled from + /// (`MachineLayout::shared_crates`): kept there ([`reclaim_shared_crates`]). + pub crates: BTreeSet, +} + +impl Used { + /// Everything the programs on `machine` were built from. A program from + /// before builds kept this record keeps nothing. + pub fn of_programs(machine: &MachineLayout) -> anyhow::Result { + let mut used = Self::default(); + for programs in [machine.workers(), machine.node_test_servers()] { + let Ok(entries) = std::fs::read_dir(&programs) else { continue }; + for entry in entries { + let dir = entry.map_err(|e| anyhow::anyhow!("read {}: {e}", programs.display()))?.path(); + if !dir.is_dir() { + continue; + } + let file = dir.join(USED_FILE); + let bytes = match std::fs::read(&file) { + Ok(bytes) => bytes, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, + Err(e) => anyhow::bail!("read {}: {e}", file.display()), + }; + let one: Self = serde_json::from_slice(&bytes).map_err(|e| anyhow::anyhow!("{} is not a record of what a build used: {e}", file.display()))?; + used.units.extend(one.units); + used.crates.extend(one.crates); + } + } + Ok(used) + } +} + +/// Remove from the machine's shared crates every folder no program on the +/// machine was built from, and anything a stopped build left +/// half-written; answers the names it removed. While any build holds the +/// folder (`weft_core::builds::shared_crates_lock`) it removes nothing and +/// answers `None`: that build's crates are in no program's record yet, and +/// a later reclaim takes what is left. +pub fn reclaim_shared_crates(machine: &MachineLayout) -> anyhow::Result>> { + let shared = machine.shared_crates(); + let lock = weft_core::builds::shared_crates_lock(&shared); + let Some(_alone) = weft_core::file_lock::try_hold(&lock).map_err(|e| anyhow::anyhow!("lock {}: {e}", lock.display()))? else { + return Ok(None); + }; + let kept = Used::of_programs(machine)?.crates; + let entries = match std::fs::read_dir(&shared) { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Some(Vec::new())), + Err(e) => anyhow::bail!("read {}: {e}", shared.display()), + }; + let mut removed = Vec::new(); + for entry in entries { + let entry = entry.map_err(|e| anyhow::anyhow!("read {}: {e}", shared.display()))?; + let name = entry.file_name().to_string_lossy().into_owned(); + if kept.contains(&name) { + continue; + } + let path = entry.path(); + let gone = if path.is_dir() { std::fs::remove_dir_all(&path) } else { std::fs::remove_file(&path) }; + gone.map_err(|e| anyhow::anyhow!("remove {}: {e}", path.display()))?; + removed.push(name); + } + removed.sort(); + Ok(Some(removed)) +} + /// What a host build needs to know besides its recipe and its cache. struct HostBuild<'a> { recipe: &'a HostRecipe, @@ -292,30 +369,43 @@ async fn build(toolchain: &dyn Toolchain, machine: &MachineLayout, held: &Held, let record = tokio::fs::read(&r.left_out_record) .await .map_err(|e| anyhow::anyhow!("the build ended without its record of what it left out at {}: {e}", r.left_out_record.display()))?; + let used = used(&messages, &release, &machine.shared_crates()).await?; + let used = serde_json::to_vec(&used).map_err(|e| anyhow::anyhow!("the record of what the build used: {e}"))?; + place_bytes(&b.out_dir, USED_FILE, &used).await?; place(&b.out_dir, LEFT_OUT_FILE, &r.left_out_record).await?; place(&b.out_dir, PROGRAM_FILE, &release.join(&r.binary_name)).await?; - let sweep = step( - "sh".into(), - vec![ - r.cache_gc_script.display().to_string(), - "aged".into(), - release.display().to_string(), - messages.display().to_string(), - r.binary_name.clone(), - r.cache_retention_days.to_string(), - r.cache_versions_kept.to_string(), - ], - None, - ); // The sweep only bounds the cache: the build is done whatever it did. - match toolchain.run(&sweep, b.log).await { - Ok(0) => {} - Ok(code) => tracing::warn!(target: "weft_platform_local::host_build", code, log = %b.log.display(), "the compile cache sweep failed; the build is unaffected"), - Err(e) => tracing::warn!(target: "weft_platform_local::host_build", error = %format!("{e:#}"), "the compile cache sweep could not run; the build is unaffected"), + if let Err(e) = sweep(machine, &release) { + tracing::warn!(target: "weft_platform_local::host_build", error = %format!("{e:#}"), "the compile cache sweep failed; the build is unaffected"); } Ok(HostBuilt::Built(record)) } +/// What a build used, off the cargo messages it wrote: its units in +/// `release`, and the folders of `shared` its units were compiled from. +async fn used(messages: &Path, release: &Path, shared: &Path) -> anyhow::Result { + let bytes = tokio::fs::read(messages).await.map_err(|e| anyhow::anyhow!("read the build's cargo messages {}: {e}", messages.display()))?; + Ok(Used { units: crate::cargo_cache::used_units(&bytes, release)?, crates: crate::cargo_cache::source_folders(&bytes, shared) }) +} + +/// Keep in the cache `release` just built in, and in the seed when no +/// build holds it, only the units some program on the machine was built +/// from ([`Used`]): what any of them would reuse when built again. A +/// program removed from the machine takes its units with it at the next +/// build. +fn sweep(machine: &MachineLayout, release: &Path) -> anyhow::Result<()> { + let kept = Used::of_programs(machine)?.units; + crate::cargo_cache::keep_used(release, &kept)?; + let lock_path = machine.seed_lock(); + if let Some(_seed_lock) = weft_core::file_lock::try_hold(&lock_path).map_err(|e| anyhow::anyhow!("lock {}: {e}", lock_path.display()))? { + let seed = machine.seed_cache().join("release"); + if seed.is_dir() { + crate::cargo_cache::keep_used(&seed, &kept)?; + } + } + Ok(()) +} + /// Give lane `number`'s cache the seed's compiled tree, once: when the /// lane is not warm yet ([`mark_lane`]) and the seed is whole /// ([`mark_seed`]). The copy holds the seed's lock, so it never copies a @@ -346,6 +436,23 @@ async fn seed(toolchain: &dyn Toolchain, machine: &MachineLayout, number: u32, l mark_lane(machine, number) } +/// Write `bytes` to `name` in `dir` the way [`place`] copies a file. +async fn place_bytes(dir: &Path, name: &str, bytes: &[u8]) -> anyhow::Result<()> { + tokio::fs::create_dir_all(dir).await.map_err(|e| anyhow::anyhow!("create {}: {e}", dir.display()))?; + let partial = dir.join(format!("{name}.{}.tmp", uuid::Uuid::new_v4().simple())); + let to = dir.join(name); + let written = async { + tokio::fs::write(&partial, bytes).await?; + tokio::fs::rename(&partial, &to).await + } + .await; + if let Err(e) = written { + let _ = tokio::fs::remove_file(&partial).await; + anyhow::bail!("write {}: {e}", to.display()); + } + Ok(()) +} + /// Copy `from` to `name` in `dir`: to a file beside it first, then renamed /// over it, so a reader never sees half of it. async fn place(dir: &Path, name: &str, from: &Path) -> anyhow::Result<()> { @@ -395,6 +502,9 @@ pub mod fake { async fn run(&self, cmd: &HostCommand, log: &Path) -> anyhow::Result { self.calls.lock().push(cmd.clone()); std::fs::write(log, format!("ran {}\n", cmd.program.display()))?; + if let Some(stdout) = &cmd.stdout { + std::fs::write(stdout, b"")?; + } for (program, file, bytes) in self.makes.lock().iter() { if *program == cmd.program { std::fs::create_dir_all(file.parent().unwrap())?; diff --git a/crates/weft-platform-local/src/images.rs b/crates/weft-platform-local/src/images.rs index 371cbaa26..cbef96d58 100644 --- a/crates/weft-platform-local/src/images.rs +++ b/crates/weft-platform-local/src/images.rs @@ -204,8 +204,18 @@ impl LocalImageBuilder { || tracing::info!(target: "weft_platform_local::images", "every compile lane of this machine is busy; a build waits for one"), ) .await; - // The context is read until the build ends. + // The context is read until the build ends, and with it goes + // the build's claim on the shared crates, which the reclaim + // waits for. drop(req.staging); + if matches!(built, Ok(HostBuilt::Built(_))) { + let reclaim = machine.clone(); + match tokio::task::spawn_blocking(move || host_build::reclaim_shared_crates(&reclaim)).await { + Ok(Ok(_)) => {} + Ok(Err(e)) => tracing::warn!(target: "weft_platform_local::images", error = %format!("{e:#}"), "could not remove the shared crates no program uses; the build is unaffected"), + Err(e) => tracing::warn!(target: "weft_platform_local::images", error = %e, "the shared crates reclaim panicked; the build is unaffected"), + } + } let status = match built { Ok(HostBuilt::Built(record)) => BuildStatus::Succeeded { result: Some(record) }, Ok(HostBuilt::Failed(why)) => { @@ -905,9 +915,6 @@ mod tests { left_out_record: ctx.join("crate/left-out.json"), seed_lock: ctx.join("seed.lock"), binary_name: "weft-worker".into(), - cache_gc_script: ctx.join("crate/weft-cache-gc.sh"), - cache_retention_days: 30, - cache_versions_kept: 3, }; for dir in [&recipe.crate_dir, &recipe.isolate_dir] { std::fs::create_dir_all(dir).unwrap(); @@ -936,7 +943,8 @@ mod tests { /// A worker is compiled on the machine: the helper first, then the /// worker through it, both in the worker crate with its lane's cache; /// the program and its record land in the machine's workers folder, the - /// record is the build's result, and the cache sweep runs after. + /// record is the build's result, with the units the build used beside + /// them. #[tokio::test] async fn a_worker_is_compiled_on_the_machine_into_its_folder() { let m = HostMachine::new(); @@ -950,26 +958,74 @@ mod tests { let calls = m.toolchain.calls.lock().clone(); let programs: Vec = calls.iter().map(|c| c.program.display().to_string()).collect(); let target = m.layout().lane_cache(0); - assert_eq!(programs, ["cargo".to_string(), target.join("release/weft-isolate").display().to_string(), "sh".to_string(), "cp".to_string()]); + assert_eq!(programs, ["cargo".to_string(), target.join("release/weft-isolate").display().to_string(), "cp".to_string()]); // With no seed on the machine (its standard worker came prebuilt), // the first lane to finish becomes it. - assert_eq!(calls[3].args, ["-a".to_string(), format!("{}/.", target.display()), m.layout().seed_cache().display().to_string()]); + assert_eq!(calls[2].args, ["-a".to_string(), format!("{}/.", target.display()), m.layout().seed_cache().display().to_string()]); assert!(m.layout().seed_mark().is_file() && m.layout().lane_mark(0).is_file()); assert!(!m.layout().seed_cache().join(".weft-seeded").exists() && !target.join(".weft-seeded").exists(), "the marks are beside the caches, never in them"); - for c in &calls[..3] { + for c in &calls[..2] { assert_eq!(c.cwd, ctx.join("crate")); assert_eq!(c.env, vec![("CARGO_TARGET_DIR".to_string(), target.display().to_string())]); } assert_eq!(calls[0].args[..3], ["build", "--release", "--manifest-path"]); assert_eq!(calls[0].stdout, Some(ctx.join("cargo-messages.json")), "cargo's messages are what the sweep reads"); - assert_eq!(calls[2].args[1..2], ["aged"]); - assert_eq!(calls[2].args[4..], ["weft-worker", "30", "3"]); assert_eq!(std::fs::read(m.program_dir("abc").join("worker")).unwrap(), b"\x7fELF"); assert_eq!(std::fs::read(m.program_dir("abc").join("left-out.json")).unwrap(), br#"{"packages":[]}"#); + assert!(m.program_dir("abc").join(host_build::USED_FILE).is_file()); assert!(b.image_exists(&image).await.unwrap()); gone(&ctx).await; } + /// After a build, its lane, the seed and the shared crates keep only + /// what some program on the machine was built from: another program's + /// stay, and what no program used goes. + #[tokio::test] + async fn a_build_leaves_its_caches_only_what_programs_on_the_machine_use() { + let m = HostMachine::new(); + let (kept, dropped) = ("00000000000000a1", "00000000000000b2"); + for cache in [m.layout().lane_cache(0), m.layout().seed_cache()] { + for hash in [kept, dropped] { + let rlib = cache.join(format!("release/deps/libpkg_x-{hash}.rlib")); + std::fs::create_dir_all(rlib.parent().unwrap()).unwrap(); + std::fs::write(rlib, b"rlib").unwrap(); + } + } + host_build::mark_seed(&m.layout()).unwrap(); + host_build::mark_lane(&m.layout(), 0).unwrap(); + std::fs::create_dir_all(m.program_dir("other")).unwrap(); + let used = host_build::Used { units: [kept.to_string()].into(), crates: ["pkg_kept-1".to_string()].into() }; + std::fs::write(m.program_dir("other").join(host_build::USED_FILE), serde_json::to_vec(&used).unwrap()).unwrap(); + for name in ["pkg_kept-1", "pkg_dropped-2"] { + std::fs::create_dir_all(m.layout().shared_crates().join(name)).unwrap(); + } + let b = m.builder("one"); + let h = b.start(m.request(&b, "abc", b"{}")).await.unwrap(); + assert_eq!(settled(&b, &h).await, BuildStatus::Succeeded { result: Some(b"{}".to_vec()) }); + for cache in [m.layout().lane_cache(0), m.layout().seed_cache()] { + let deps = cache.join("release/deps"); + assert!(deps.join(format!("libpkg_x-{kept}.rlib")).exists(), "{}: another program's piece stays", cache.display()); + assert!(!deps.join(format!("libpkg_x-{dropped}.rlib")).exists(), "{}: a piece no program used goes", cache.display()); + } + assert!(m.layout().shared_crates().join("pkg_kept-1").is_dir(), "another program's crate stays"); + assert!(!m.layout().shared_crates().join("pkg_dropped-2").exists(), "a crate no program used goes"); + } + + /// While a build holds the shared crates (it staged into them and is + /// not compiled yet), a reclaim removes nothing: its crates are in no + /// program's record yet. + #[test] + fn a_reclaim_waits_for_no_build_to_hold_the_shared_crates() { + let m = HostMachine::new(); + let crate_dir = m.layout().shared_crates().join("pkg_staged-1"); + std::fs::create_dir_all(&crate_dir).unwrap(); + let staging = weft_core::file_lock::hold_shared(&weft_core::builds::shared_crates_lock(&m.layout().shared_crates())).unwrap(); + assert_eq!(host_build::reclaim_shared_crates(&m.layout()).unwrap(), None); + assert!(crate_dir.is_dir()); + drop(staging); + assert_eq!(host_build::reclaim_shared_crates(&m.layout()).unwrap(), Some(vec!["pkg_staged-1".to_string()])); + } + /// A toolchain that, at every command, notes whether lane 0 and the /// seed are held by someone (the build that ran it). struct Watching { @@ -988,11 +1044,11 @@ mod tests { } } - /// A build holds its lane from the seed copy through the sweep, and + /// A build holds its lane from the seed copy through the compile, and /// the seed's lock while it copies the seed (never while it compiles); /// the lane is free once the build ended. #[tokio::test] - async fn a_host_build_holds_its_lane_through_the_seed_copy_the_compile_and_the_sweep() { + async fn a_host_build_holds_its_lane_through_the_seed_copy_and_the_compile() { let m = HostMachine::new(); std::fs::create_dir_all(m.layout().seed_cache()).unwrap(); host_build::mark_seed(&m.layout()).unwrap(); @@ -1002,9 +1058,9 @@ mod tests { assert_eq!(settled(&b, &h).await, BuildStatus::Succeeded { result: Some(b"{}".to_vec()) }); let seen = watching.seen.lock().clone(); let programs: Vec<&str> = seen.iter().map(|(p, _, _)| p.rsplit('/').next().unwrap()).collect(); - assert_eq!(programs, ["cp", "cargo", "weft-isolate", "sh"], "the cold lane is seeded first"); + assert_eq!(programs, ["cp", "cargo", "weft-isolate"], "the cold lane is seeded first"); assert!(seen.iter().all(|(_, lane, _)| *lane), "the lane is held at every step: {seen:?}"); - assert_eq!(seen.iter().map(|(_, _, seed)| *seed).collect::>(), [true, false, false, false], "the seed only while it is copied"); + assert_eq!(seen.iter().map(|(_, _, seed)| *seed).collect::>(), [true, false, false], "the seed only while it is copied"); assert!(crate::lanes::Lane::try_take(&m.layout(), 0).unwrap().is_some(), "let go once the build ended"); } @@ -1098,7 +1154,7 @@ mod tests { .unwrap(); assert_eq!(built, HostBuilt::Built(b"{}".to_vec())); let seen = watching.seen.lock().clone(); - assert_eq!(seen.iter().map(|(p, _, _)| p.rsplit('/').next().unwrap()).collect::>(), ["cargo", "weft-isolate", "sh"], "the seed copies nothing"); + assert_eq!(seen.iter().map(|(p, _, _)| p.rsplit('/').next().unwrap()).collect::>(), ["cargo", "weft-isolate"], "the seed copies nothing"); assert!(seen.iter().all(|(_, lane, seed)| !lane && *seed), "{seen:?}"); assert!(m.layout().seed_mark().is_file(), "a seed that left nothing out is whole"); } diff --git a/crates/weft-platform-local/src/lib.rs b/crates/weft-platform-local/src/lib.rs index e8cd3e6d3..c49e1622d 100644 --- a/crates/weft-platform-local/src/lib.rs +++ b/crates/weft-platform-local/src/lib.rs @@ -19,6 +19,7 @@ //! - `folders`: every folder working copy kept in sync with its folder. pub mod alarm; +pub mod cargo_cache; pub mod docker; pub mod domains; pub mod folders; diff --git a/docs/src/running/architecture.md b/docs/src/running/architecture.md index b65aff282..f0a0181b0 100644 --- a/docs/src/running/architecture.md +++ b/docs/src/running/architecture.md @@ -230,9 +230,13 @@ once, one compiles and the others wait for the lane, then compile only what the build before them did not already leave in it. A build of a program this machine already has takes no lane at all, and neither does a run, so running code is never held up. Two asks for the same build of one project -are one build, which both wait on. After each build in a lane, weft keeps -only the three newest compiled versions of each node package and of the -worker, and drops any of them no build has used for 30 days. +are one build, which both wait on. Each program weft compiles notes which +compiled pieces and which source crates its build used, and after each +build, weft keeps in that lane, in the seed and in the shared source crates +(`~/.local/share/weft/worker-crates/`) only what some program still on this +machine was built from. When a program goes (a project removed, a weft +update replacing the standard worker), what only it used goes at the next +build. On a cloud install the worker is an image instead, and the `[build.worker]` section of `weft.toml` says how that image is built: diff --git a/docs/src/running/cli.md b/docs/src/running/cli.md index b070c5875..fb6e84c51 100644 --- a/docs/src/running/cli.md +++ b/docs/src/running/cli.md @@ -530,7 +530,7 @@ what no install on this machine uses: runtime images, the builder bases, node-test images, standard worker images and image compile caches an earlier weft or a `weft build-images` run left, and the worker programs (an earlier weft's standard worker among them). It also removes the crates your machine -compiles workers from that no build used for 30 days +compiles workers from that no worker program on it was built from (`~/.local/share/weft/worker-crates/`), and the images and build caches of installs no longer on your machine, along with the caches an older weft made before each install had its own (`weft daemon remove` drops an install's diff --git a/setup.sh b/setup.sh index f879a532e..2f5351008 100755 --- a/setup.sh +++ b/setup.sh @@ -38,11 +38,10 @@ # declares a VOLUME, so a throwaway test database whose container # was removed without -v strands one forever; named volumes are # never touched), -# - a --cli install (the default includes it) bounds the two local -# build caches: the BuildKit cache to 20GB (LRU) and the workspace -# target/ (WEFT_TARGET_CAP_GB, default 30G; over it, the incremental -# dirs and the superseded executables go, and the tree is never -# wiped), +# - a --cli install (the default includes it) bounds the BuildKit +# cache to 20GB (LRU), and one that builds weft from source removes +# from the workspace target/ every compiled piece a newer build of +# the same piece replaced (`weft trim-cargo-target`), # - the daemon start removes the worker containers and images a weft # from before workers ran as processes left for this install (a # local install's workers are programs compiled on this machine), @@ -1679,104 +1678,11 @@ if [[ $build_cli -eq 1 ]]; then # ---- disk hygiene, BEFORE the build ------------------------------- # - # The two BUILD caches are bounded on every --cli install (the - # default includes it): - # - the cargo target/ dir (incremental artifacts accumulate across - # dep bumps and crate renames and are never evicted; once grew to - # ~100GB before anyone noticed), and - # - docker's BuildKit layer cache (every runtime and infra image - # build adds entries; nothing prunes them). - # (The anonymous-volume reap is NOT here: it runs on every install - # regardless of flags, in its own section above.) - # The target/ bound is a cap, and it never wipes the tree: a wiped - # tree makes the build that follows run cold. Under the cap nothing is - # touched. Over it, two things go, cheapest first, re-measuring after - # each: - # 1. the incremental dirs: they only speed up rebuilding a crate you - # just edited, and they are the biggest line item (27G of a 57G - # tree, measured), while deps/ is what makes a rebuild fast at all; - # 2. the superseded linked executables (test binaries above all): cargo - # keeps every one it ever linked, one per hash, so an edit that - # relinks a test binary leaves the old one behind for good. Of the - # executables in a deps/ dir sharing a name, the newest stays and - # the rest go with their dep-info, debug info and fingerprint. The - # worst a wrong pick costs is a relink (an older one can still be - # live, for another feature set): cargo rebuilds any unit whose - # output or fingerprint is missing. - # Still over after both: said loudly, and left alone. - # The cap measures the WHOLE of target/ (debug + release + tmp). A - # routine tree, the whole workspace's tests, clippy and this script's - # release build, measured about 22G (18G debug, half of it incremental, - # and 4G release), so the default leaves room for a few edits' worth of - # superseded builds before anything is trimmed. - # Override with WEFT_TARGET_CAP_GB. - target_cap_gb="${WEFT_TARGET_CAP_GB:-30}" - if [[ ! "${target_cap_gb}" =~ ^[0-9]+$ ]]; then - fail "WEFT_TARGET_CAP_GB must be a whole number of gigabytes, got '${target_cap_gb}'" - exit 1 - fi - # -L: a target/ symlinked onto a scratch disk (a normal cargo setup, and - # the case where the cap matters most) measures as 0 without it. -sk - # (POSIX kilobytes; GNU-only -BG never works on macOS, which would - # silently disable the cap there forever). Prints nothing on an - # unreadable measurement, which skips the cap (trimming on an unreadable - # measurement is the dangerous direction). - target_size_gb() { - local kb - kb="$(du -sk -L "${here}/target" 2>/dev/null | tail -n1 | cut -f1)" || true - if [[ "${kb}" =~ ^[0-9]+$ ]]; then - echo "$((kb / 1024 / 1024))" - fi - } - over_target_cap() { - [[ "${target_gb}" =~ ^[0-9]+$ && "${target_gb}" -gt "${target_cap_gb}" ]] - } - if [[ -d "${here}/target" ]]; then - target_gb="$(target_size_gb)" - if [[ ! "${target_gb}" =~ ^[0-9]+$ ]]; then - warn "could not measure target/; skipping the size cap" - elif over_target_cap; then - warn "target/ is ${target_gb}G (cap ${target_cap_gb}G); trimming before the build" - for inc in "${here}"/target/*/incremental; do - [[ -d "${inc}" ]] || continue - remove_dir_reporting "${inc}" "${C_DIM}${inc#"${here}"/}${C_RESET}" \ - "" "; the cap may stay exceeded" - done - target_gb="$(target_size_gb)" - if over_target_cap; then - dropped=0 - for deps_dir in "${here}"/target/*/deps; do - [[ -d "${deps_dir}" ]] || continue - fingerprint_dir="${deps_dir%/deps}/.fingerprint" - # Newest first, so the first of each name is the one that stays. - # A linked executable is a file with the execute bit and no - # extension, named `-<16 hex>` (the hash every file of its - # unit carries, its fingerprint dir included); a library has an - # extension (.rlib, .so, .dylib), even an executable one. - seen=" " - while IFS= read -r entry; do - case "${entry}" in *.*) continue ;; esac - [[ -f "${deps_dir}/${entry}" && -x "${deps_dir}/${entry}" ]] || continue - unit_hash="${entry##*-}" - [[ "${unit_hash}" =~ ^[0-9a-f]{16}$ ]] || continue - unit_name="${entry%-*}" - case "${seen}" in - *" ${unit_name} "*) ;; - *) seen="${seen}${unit_name} "; continue ;; - esac - rm -rf "${deps_dir:?}/${entry}" "${deps_dir:?}/${entry}".* "${fingerprint_dir:?}"/*-"${unit_hash}" - dropped=$((dropped + 1)) - done < <(ls -t "${deps_dir}") - done - target_gb="$(target_size_gb)" - ok "removed ${dropped} superseded executables from target/ (now ${target_gb:-?}G)" - fi - if over_target_cap; then - warn "target/ is still ${target_gb}G, over the ${target_cap_gb}G cap, after dropping the incremental dirs and the superseded executables; it is left as it is so this build stays warm" - hint "to clear it by hand: ${C_BOLD}cargo clean${C_RESET}${C_DIM} in ${here} (the next build then runs cold), or raise WEFT_TARGET_CAP_GB" - fi - fi - fi + # docker's BuildKit layer cache is bounded on every --cli install (the + # default includes it): every runtime and infra image build adds + # entries, and nothing prunes them. (The anonymous-volume reap is NOT + # here: it runs on every install regardless of flags, in its own + # section above. The cargo target/ is trimmed after the build, below.) # Gate on daemon REACHABILITY, not binary presence: a --cli install # on a machine whose docker isn't running must still produce a # binary. With a live daemon, a failed prune fails loud; with none, @@ -1842,6 +1748,16 @@ if [[ $build_cli -eq 1 ]]; then with_build_slot build_cli_from_source new_cli="${target_dir}/weft" new_runtime="${target_dir}/weft-runtime" + # cargo never deletes what a newer build replaced, so target/ grows + # by a copy of every piece each dependency move or update touches. + # Each piece kept is the newest of its kind (one per way it is built: + # tests, clippy, this release build), so no build that follows + # compiles more than it would have. A failed trim costs only disk. + if trimmed="$("${new_cli}" trim-cargo-target "${here}/target" 2>&1)"; then + ok "${trimmed} from target/" + else + warn "could not trim target/: ${trimmed}" + fi # An unplaced binary (one in target/) answers for the checkout it # runs in, which is how this run learns where to install before # anything is installed. From c8158a73d177f14fb58141d1ac3bd18eee66709a Mon Sep 17 00:00:00 2001 From: Quentin Date: Wed, 28 Oct 2026 10:26:03 +0100 Subject: [PATCH 2/2] Name the changelog entry after its release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bebd3912a..3702d749e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ What changed between versions of weft that you need to know about when you update. To update, run `weft maintenance on`, pull and run `./setup.sh` again, then run `weft maintenance off`. -## Unreleased +## 2026-10-28 (341192de) **The folders weft compiles in stop growing with every update.** Each program weft compiles for a project now notes which compiled pieces and